Media: fix encrypted media

2023-05-16 11:51:11 +02:00
parent 801e99a4db
commit ac1531390e
20 changed files with 73 additions and 54 deletions
--- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesFlowNode.kt
+++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesFlowNode.kt
@@ -40,7 +40,7 @@ import io.element.android.features.messages.impl.timeline.model.event.TimelineIt
 import io.element.android.libraries.architecture.BackstackNode
 import io.element.android.libraries.architecture.createNode
 import io.element.android.libraries.di.RoomScope
-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
+import io.element.android.libraries.matrix.api.media.MediaSource
 import kotlinx.android.parcel.Parcelize
 import kotlinx.collections.immutable.ImmutableList

@@ -64,7 +64,7 @@ class MessagesFlowNode @AssistedInject constructor(
        @Parcelize
        data class MediaViewer(
            val title: String,
-            val mediaSource: MatrixMediaSource,
+            val mediaSource: MediaSource,
            val mimeType: String?
        ) : NavTarget

--- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/media/viewer/MediaViewerNode.kt
+++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/media/viewer/MediaViewerNode.kt
@@ -27,7 +27,7 @@ import io.element.android.anvilannotations.ContributesNode
 import io.element.android.libraries.architecture.NodeInputs
 import io.element.android.libraries.architecture.inputs
 import io.element.android.libraries.di.RoomScope
-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
+import io.element.android.libraries.matrix.api.media.MediaSource

@ContributesNode(RoomScope::class)
 class MediaViewerNode @AssistedInject constructor(
@@ -38,7 +38,7 @@ class MediaViewerNode @AssistedInject constructor(

    data class Inputs(
        val name: String,
-        val mediaSource: MatrixMediaSource,
+        val mediaSource: MediaSource,
        val mimeType: String?
    ) : NodeInputs

--- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemImageContent.kt
+++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemImageContent.kt
@@ -16,11 +16,11 @@

 package io.element.android.features.messages.impl.timeline.model.event

-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
+import io.element.android.libraries.matrix.api.media.MediaSource

 data class TimelineItemImageContent(
    val body: String,
-    val mediaSource: MatrixMediaSource,
+    val mediaSource: MediaSource,
    val mimeType: String?,
    val blurhash: String?,
    val width: Int?,
--- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemImageContentProvider.kt
+++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemImageContentProvider.kt
@@ -17,8 +17,7 @@
 package io.element.android.features.messages.impl.timeline.model.event

 import androidx.compose.ui.tooling.preview.PreviewParameterProvider
-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
-import io.element.android.libraries.matrix.ui.media.MediaRequestData
+import io.element.android.libraries.matrix.api.media.MediaSource

 open class TimelineItemImageContentProvider : PreviewParameterProvider<TimelineItemImageContent> {
    override val values: Sequence<TimelineItemImageContent>
@@ -31,7 +30,7 @@ open class TimelineItemImageContentProvider : PreviewParameterProvider<TimelineI

 fun aTimelineItemImageContent() = TimelineItemImageContent(
    body = "a body",
-    mediaSource = MatrixMediaSource(""),
+    mediaSource = MediaSource(""),
    blurhash = "TQF5:I_NtRE4kXt7Z#MwkCIARPjr",
    aspectRatio = 0.5f,
    mimeType = "null",
--- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemVideoContent.kt
+++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemVideoContent.kt
@@ -16,13 +16,13 @@

 package io.element.android.features.messages.impl.timeline.model.event

-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
+import io.element.android.libraries.matrix.api.media.MediaSource

 data class TimelineItemVideoContent(
    val body: String,
    val duration: Long,
-    val videoSource: MatrixMediaSource,
-    val thumbnailSource: MatrixMediaSource?,
+    val videoSource: MediaSource,
+    val thumbnailSource: MediaSource?,
    val aspectRatio: Float,
    val blurhash: String?,
    val height: Int?,
--- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemVideoContentProvider.kt
+++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemVideoContentProvider.kt
@@ -17,7 +17,7 @@
 package io.element.android.features.messages.impl.timeline.model.event

 import androidx.compose.ui.tooling.preview.PreviewParameterProvider
-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
+import io.element.android.libraries.matrix.api.media.MediaSource

 open class TimelineItemVideoContentProvider : PreviewParameterProvider<TimelineItemVideoContent> {
    override val values: Sequence<TimelineItemVideoContent>
@@ -30,11 +30,11 @@ open class TimelineItemVideoContentProvider : PreviewParameterProvider<TimelineI

 fun aTimelineItemVideoContent() = TimelineItemVideoContent(
    body = "a video",
-    thumbnailSource = MatrixMediaSource(url = ""),
+    thumbnailSource = MediaSource(url = ""),
    blurhash = "TQF5:I_NtRE4kXt7Z#MwkCIARPjr",
    aspectRatio = 0.5f,
    duration = 0,
-    videoSource = MatrixMediaSource(""),
+    videoSource = MediaSource(""),
    height = null,
    width = null,
    mimeType = null
--- a/features/roomlist/impl/src/test/kotlin/io/element/android/features/roomlist/impl/DefaultRoomLastMessageFormatterTests.kt
+++ b/features/roomlist/impl/src/test/kotlin/io/element/android/features/roomlist/impl/DefaultRoomLastMessageFormatterTests.kt
@@ -21,7 +21,7 @@ import androidx.compose.ui.text.AnnotatedString
 import com.google.common.truth.Truth
 import io.element.android.libraries.matrix.api.core.UserId
 import io.element.android.libraries.matrix.api.media.ImageInfo
-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
+import io.element.android.libraries.matrix.api.media.MediaSource
 import io.element.android.libraries.matrix.api.timeline.item.event.AudioMessageType
 import io.element.android.libraries.matrix.api.timeline.item.event.EmoteMessageType
 import io.element.android.libraries.matrix.api.timeline.item.event.EventContent
@@ -149,10 +149,10 @@ class DefaultRoomLastMessageFormatterTests {

        val sharedContentMessagesTypes = arrayOf(
            TextMessageType(body, null),
-            VideoMessageType(body, MatrixMediaSource("url"), null),
-            AudioMessageType(body, MatrixMediaSource("url"), null),
-            ImageMessageType(body, MatrixMediaSource("url"), null),
-            FileMessageType(body, MatrixMediaSource("url"), null),
+            VideoMessageType(body, MediaSource("url"), null),
+            AudioMessageType(body, MediaSource("url"), null),
+            ImageMessageType(body, MediaSource("url"), null),
+            FileMessageType(body, MediaSource("url"), null),
            NoticeMessageType(body, null),
            EmoteMessageType(body, null),
        )
--- a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/FileInfo.kt
+++ b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/FileInfo.kt
@@ -20,5 +20,5 @@ data class FileInfo(
    val mimetype: String?,
    val size: Long?,
    val thumbnailInfo: ThumbnailInfo?,
-    val thumbnailSource: MatrixMediaSource?
+    val thumbnailSource: MediaSource?
 )
--- a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/ImageInfo.kt
+++ b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/ImageInfo.kt
@@ -22,6 +22,6 @@ data class ImageInfo(
    val mimetype: String?,
    val size: Long?,
    val thumbnailInfo: ThumbnailInfo?,
-    val thumbnailSource: MatrixMediaSource?,
+    val thumbnailSource: MediaSource?,
    val blurhash: String?
 )
--- a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/MatrixMediaLoader.kt
+++ b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/MatrixMediaLoader.kt
@@ -21,7 +21,7 @@ interface MatrixMediaLoader {
     * @param url to fetch the content for.
     * @return a [Result] of ByteArray. It contains the binary data for the media.
     */
-    suspend fun loadMediaContent(source: MatrixMediaSource): Result<ByteArray>
+    suspend fun loadMediaContent(source: MediaSource): Result<ByteArray>

    /**
     * @param url to fetch the data for.
@@ -29,12 +29,12 @@ interface MatrixMediaLoader {
     * @param height: the desired height for rescaling the media as thumbnail
     * @return a [Result] of ByteArray. It contains the binary data for the media.
     */
-    suspend fun loadMediaThumbnail(source: MatrixMediaSource, width: Long, height: Long): Result<ByteArray>
+    suspend fun loadMediaThumbnail(source: MediaSource, width: Long, height: Long): Result<ByteArray>

    /**
     * @param url to fetch the data for.
     * @param mimeType: optional mime type
     * @return a [Result] of [MediaFile]
     */
-    suspend fun loadMediaFile(source: MatrixMediaSource, mimeType: String?): Result<MediaFile>
+    suspend fun loadMediaFile(source: MediaSource, mimeType: String?): Result<MediaFile>
 }
--- a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/MatrixMediaSource.kt
+++ b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/MatrixMediaSource.kt
@@ -20,6 +20,13 @@ import android.os.Parcelable
 import kotlinx.parcelize.Parcelize

@Parcelize
-data class MatrixMediaSource(
-    val url: String
+data class MediaSource(
+    /**
+     * Url of the media
+     */
+    val url: String,
+    /**
+     * This is used to hold data for encrypted media
+     */
+    val json: String? = null,
 ) : Parcelable
--- a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/VideoInfo.kt
+++ b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/VideoInfo.kt
@@ -23,6 +23,6 @@ data class VideoInfo(
    val mimetype: String?,
    val size: Long?,
    val thumbnailInfo: ThumbnailInfo?,
-    val thumbnailSource: MatrixMediaSource?,
+    val thumbnailSource: MediaSource?,
    val blurhash: String?
 )
--- a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/timeline/item/event/EventContent.kt
+++ b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/timeline/item/event/EventContent.kt
@@ -21,7 +21,7 @@ import io.element.android.libraries.matrix.api.core.UserId
 import io.element.android.libraries.matrix.api.media.AudioInfo
 import io.element.android.libraries.matrix.api.media.FileInfo
 import io.element.android.libraries.matrix.api.media.ImageInfo
-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
+import io.element.android.libraries.matrix.api.media.MediaSource
 import io.element.android.libraries.matrix.api.media.VideoInfo

 sealed interface EventContent
@@ -107,25 +107,25 @@ data class EmoteMessageType(

 data class ImageMessageType(
    val body: String,
-    val source: MatrixMediaSource,
+    val source: MediaSource,
    val info: ImageInfo?
 ) : MessageType

 data class AudioMessageType(
    val body: String,
-    val source: MatrixMediaSource,
+    val source: MediaSource,
    val info: AudioInfo?
 ) : MessageType

 data class VideoMessageType(
    val body: String,
-    val source: MatrixMediaSource,
+    val source: MediaSource,
    val info: VideoInfo?
 ) : MessageType

 data class FileMessageType(
    val body: String,
-    val source: MatrixMediaSource,
+    val source: MediaSource,
    val info: FileInfo?
 ) : MessageType

--- a/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/MediaSource.kt
+++ b/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/MediaSource.kt
@@ -16,8 +16,10 @@

 package io.element.android.libraries.matrix.impl.media

-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
+import io.element.android.libraries.matrix.api.media.MediaSource
 import org.matrix.rustcomponents.sdk.use
 import org.matrix.rustcomponents.sdk.MediaSource as RustMediaSource

-fun RustMediaSource.map(): MatrixMediaSource = use { MatrixMediaSource(it.url()) }
+fun RustMediaSource.map(): MediaSource = use {
+    MediaSource(it.url(), it.toJson())
+}
--- a/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/RustMediaLoader.kt
+++ b/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/RustMediaLoader.kt
@@ -18,12 +18,13 @@ package io.element.android.libraries.matrix.impl.media

 import io.element.android.libraries.core.coroutine.CoroutineDispatchers
 import io.element.android.libraries.matrix.api.media.MatrixMediaLoader
-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
 import io.element.android.libraries.matrix.api.media.MediaFile
+import io.element.android.libraries.matrix.api.media.MediaSource
 import kotlinx.coroutines.withContext
 import org.matrix.rustcomponents.sdk.Client
 import org.matrix.rustcomponents.sdk.mediaSourceFromUrl
 import org.matrix.rustcomponents.sdk.use
+import org.matrix.rustcomponents.sdk.MediaSource as RustMediaSource

 class RustMediaLoader(
    private val dispatchers: CoroutineDispatchers,
@@ -31,10 +32,10 @@ class RustMediaLoader(
 ) : MatrixMediaLoader {

    @OptIn(ExperimentalUnsignedTypes::class)
-    override suspend fun loadMediaContent(source: MatrixMediaSource): Result<ByteArray> =
+    override suspend fun loadMediaContent(source: MediaSource): Result<ByteArray> =
        withContext(dispatchers.io) {
            runCatching {
-                mediaSourceFromUrl(source.url).use { source ->
+                source.toRustMediaSource().use { source ->
                    innerClient.getMediaContent(source).toUByteArray().toByteArray()
                }
            }
@@ -42,13 +43,13 @@ class RustMediaLoader(

    @OptIn(ExperimentalUnsignedTypes::class)
    override suspend fun loadMediaThumbnail(
-        source: MatrixMediaSource,
+        source: MediaSource,
        width: Long,
        height: Long
    ): Result<ByteArray> =
        withContext(dispatchers.io) {
            runCatching {
-                mediaSourceFromUrl(source.url).use { mediaSource ->
+                source.toRustMediaSource().use { mediaSource ->
                    innerClient.getMediaThumbnail(
                        mediaSource = mediaSource,
                        width = width.toULong(),
@@ -58,10 +59,10 @@ class RustMediaLoader(
            }
        }

-    override suspend fun loadMediaFile(source: MatrixMediaSource, mimeType: String?): Result<MediaFile> =
+    override suspend fun loadMediaFile(source: MediaSource, mimeType: String?): Result<MediaFile> =
        withContext(dispatchers.io) {
            runCatching {
-                mediaSourceFromUrl(source.url).use { mediaSource ->
+                source.toRustMediaSource().use { mediaSource ->
                    val mediaFile = innerClient.getMediaFile(
                        mediaSource = mediaSource,
                        body = null,
@@ -71,4 +72,13 @@ class RustMediaLoader(
                }
            }
        }
+
+    private fun MediaSource.toRustMediaSource(): RustMediaSource {
+        val json = this.json
+        return if (json != null) {
+            RustMediaSource.fromJson(json)
+        } else {
+            mediaSourceFromUrl(url)
+        }
+    }
 }
--- a/libraries/matrix/test/src/main/kotlin/io/element/android/libraries/matrix/test/media/FakeMediaLoader.kt
+++ b/libraries/matrix/test/src/main/kotlin/io/element/android/libraries/matrix/test/media/FakeMediaLoader.kt
@@ -17,14 +17,14 @@
 package io.element.android.libraries.matrix.test.media

 import io.element.android.libraries.matrix.api.media.MatrixMediaLoader
-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
+import io.element.android.libraries.matrix.api.media.MediaSource
 import io.element.android.libraries.matrix.api.media.MediaFile

 class FakeMediaLoader : MatrixMediaLoader {

    var shouldFail = false

-    override suspend fun loadMediaContent(source: MatrixMediaSource): Result<ByteArray> {
+    override suspend fun loadMediaContent(source: MediaSource): Result<ByteArray> {
        return if (shouldFail) {
            Result.failure(RuntimeException())
        } else {
@@ -32,7 +32,7 @@ class FakeMediaLoader : MatrixMediaLoader {
        }
    }

-    override suspend fun loadMediaThumbnail(source: MatrixMediaSource, width: Long, height: Long): Result<ByteArray> {
+    override suspend fun loadMediaThumbnail(source: MediaSource, width: Long, height: Long): Result<ByteArray> {
        return if (shouldFail) {
            Result.failure(RuntimeException())
        } else {
@@ -40,7 +40,7 @@ class FakeMediaLoader : MatrixMediaLoader {
        }
    }

-    override suspend fun loadMediaFile(source: MatrixMediaSource, mimeType: String?): Result<MediaFile> {
+    override suspend fun loadMediaFile(source: MediaSource, mimeType: String?): Result<MediaFile> {
        return if (shouldFail) {
            Result.failure(RuntimeException())
        } else {
--- a/libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/AvatatarDataExt.kt
+++ b/libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/AvatatarDataExt.kt
@@ -17,12 +17,12 @@
 package io.element.android.libraries.matrix.ui.media

 import io.element.android.libraries.designsystem.components.avatar.AvatarData
-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
+import io.element.android.libraries.matrix.api.media.MediaSource
 import kotlin.math.roundToLong

 fun AvatarData.toMediaRequestData(): MediaRequestData {
    return MediaRequestData(
-        source = url?.let { MatrixMediaSource(it) },
+        source = url?.let { MediaSource(it) },
        kind = MediaRequestData.Kind.Thumbnail(size.dp.value.roundToLong())
    )
 }
--- a/libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/MediaRequestData.kt
+++ b/libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/MediaRequestData.kt
@@ -16,10 +16,10 @@

 package io.element.android.libraries.matrix.ui.media

-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
+import io.element.android.libraries.matrix.api.media.MediaSource

 data class MediaRequestData(
-    val source: MatrixMediaSource?,
+    val source: MediaSource?,
    val kind: Kind
 ) {

--- a/libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/MediaPreProcessorImpl.kt
+++ b/libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/MediaPreProcessorImpl.kt
@@ -33,7 +33,7 @@ import io.element.android.libraries.di.ApplicationContext
 import io.element.android.libraries.matrix.api.media.AudioInfo
 import io.element.android.libraries.matrix.api.media.FileInfo
 import io.element.android.libraries.matrix.api.media.ImageInfo
-import io.element.android.libraries.matrix.api.media.MatrixMediaSource
+import io.element.android.libraries.matrix.api.media.MediaSource
 import io.element.android.libraries.matrix.api.media.ThumbnailInfo
 import io.element.android.libraries.matrix.api.media.VideoInfo
 import io.element.android.libraries.mediaupload.api.MediaPreProcessor
@@ -217,7 +217,7 @@ class MediaPreProcessorImpl @Inject constructor(
                mimetype = mimeType,
                size = file.length(),
                thumbnailInfo = thumbnailInfo?.info,
-                thumbnailSource = thumbnailUrl?.let { MatrixMediaSource(it) },
+                thumbnailSource = thumbnailUrl?.let { MediaSource(it) },
                blurhash = thumbnailInfo?.blurhash,
            )
        }
@@ -251,7 +251,7 @@ fun ImageCompressionResult.toImageInfo(mimeType: String, thumbnailUrl: String?,
    mimetype = mimeType,
    size = size,
    thumbnailInfo = thumbnailInfo,
-    thumbnailSource = thumbnailUrl?.let { MatrixMediaSource(it) },
+    thumbnailSource = thumbnailUrl?.let { MediaSource(it) },
    blurhash = blurhash,
 )

--- a/libraries/mediaupload/test/src/main/kotlin/io/element/android/libraries/mediaupload/test/FakeMediaPreProcessor.kt
+++ b/libraries/mediaupload/test/src/main/kotlin/io/element/android/libraries/mediaupload/test/FakeMediaPreProcessor.kt
@@ -32,10 +32,11 @@ class FakeMediaPreProcessor : MediaPreProcessor {
                mimetype = "*/*",
                size = 999L,
                thumbnailInfo = null,
-                thumbnailUrl = null,
+                thumbnailSource = null,
            )
        )
    )
+
    override suspend fun process(uri: Uri, mediaType: MediaType, deleteOriginal: Boolean): Result<MediaUploadInfo> = result

    fun givenResult(value: Result<MediaUploadInfo>) {