Merge pull request #599 from vector-im/feature/fga/timeline_media_improvements

Feature/fga/timeline media improvements

features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesFlowNode.kt

@@ -138,7 +138,7 @@ class MessagesFlowNode @AssistedInject constructor(
                         fileExtension = event.content.fileExtension
                     ),
                     mediaSource = event.content.mediaSource,
-                    thumbnailSource = event.content.mediaSource,
+                    thumbnailSource = event.content.thumbnailSource,
                 )
                 backstack.push(navTarget)
             }

features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/media/viewer/MediaViewerView.kt

@@ -54,9 +54,11 @@ import androidx.compose.ui.unit.dp
 import coil.compose.AsyncImage
 import io.element.android.features.messages.impl.media.local.LocalMedia
 import io.element.android.features.messages.impl.media.local.LocalMediaView
+import io.element.android.features.messages.impl.media.local.MediaInfo
 import io.element.android.features.messages.impl.media.local.rememberLocalMediaViewState
 import io.element.android.libraries.architecture.Async
 import io.element.android.libraries.architecture.isLoading
+import io.element.android.libraries.core.mimetype.MimeTypes
 import io.element.android.libraries.designsystem.components.button.BackButton
 import io.element.android.libraries.designsystem.components.dialogs.RetryDialog
 import io.element.android.libraries.designsystem.preview.ElementPreviewDark
@@ -140,6 +142,7 @@ fun MediaViewerView(
                     mediaInfo = state.mediaInfo,
                 )
                 ThumbnailView(
+                    mediaInfo = state.mediaInfo,
                     thumbnailSource = state.thumbnailSource,
                     showThumbnail = showThumbnail,
                 )
@@ -211,6 +214,7 @@ private fun MediaViewerTopBar(
 private fun ThumbnailView(
     thumbnailSource: MediaSource?,
     showThumbnail: Boolean,
+    mediaInfo: MediaInfo,
 ) {
     AnimatedVisibility(
         visible = showThumbnail,
@@ -223,7 +227,7 @@ private fun ThumbnailView(
         ) {
             val mediaRequestData = MediaRequestData(
                 source = thumbnailSource,
-                kind = MediaRequestData.Kind.Content
+                kind = MediaRequestData.Kind.File(mediaInfo.name, mediaInfo.mimeType)
             )
             AsyncImage(
                 modifier = Modifier.fillMaxSize(),

features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/components/event/TimelineItemImageView.kt

@@ -43,7 +43,7 @@ fun TimelineItemImageView(
         modifier = modifier
     ) {
         BlurHashAsyncImage(
-            model = MediaRequestData(content.mediaSource, MediaRequestData.Kind.Content),
+            model = MediaRequestData(content.preferredMediaSource, MediaRequestData.Kind.File(content.body, content.mimeType)),
             blurHash = content.blurhash,
             modifier = Modifier.fillMaxSize(),
             contentScale = ContentScale.Fit,

features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/components/event/TimelineItemVideoView.kt

@@ -49,7 +49,7 @@ fun TimelineItemVideoView(
         contentAlignment = Alignment.Center,
     ) {
         BlurHashAsyncImage(
-            model = MediaRequestData(content.thumbnailSource, MediaRequestData.Kind.Content),
+            model = MediaRequestData(content.thumbnailSource, MediaRequestData.Kind.File(content.body, content.mimeType)),
             blurHash = content.blurHash,
             modifier = Modifier.fillMaxSize(),
             contentScale = ContentScale.Fit,

features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt

@@ -54,6 +54,7 @@ class TimelineItemContentMessageFactory @Inject constructor(
                 TimelineItemImageContent(
                     body = messageType.body,
                     mediaSource = messageType.source,
+                    thumbnailSource  = messageType.info?.thumbnailSource,
                     mimeType = messageType.info?.mimetype ?: MimeTypes.OctetStream,
                     blurhash = messageType.info?.blurhash,
                     width = messageType.info?.width?.toInt(),

features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemImageContent.kt

@@ -16,11 +16,13 @@
 
 package io.element.android.features.messages.impl.timeline.model.event
 
+import io.element.android.libraries.core.mimetype.MimeTypes
 import io.element.android.libraries.matrix.api.media.MediaSource
 
 data class TimelineItemImageContent(
     val body: String,
     val mediaSource: MediaSource,
+    val thumbnailSource: MediaSource?,
     val formattedFileSize: String,
     val fileExtension: String,
     val mimeType: String,
@@ -30,4 +32,10 @@ data class TimelineItemImageContent(
     val aspectRatio: Float
 ) : TimelineItemEventContent {
     override val type: String = "TimelineItemImageContent"
+
+    val preferredMediaSource = if (mimeType == MimeTypes.Gif) {
+        mediaSource
+    } else {
+        thumbnailSource ?: mediaSource
+    }
 }

features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemImageContentProvider.kt

@@ -32,6 +32,7 @@ open class TimelineItemImageContentProvider : PreviewParameterProvider<TimelineI
 fun aTimelineItemImageContent() = TimelineItemImageContent(
     body = "a body",
     mediaSource = MediaSource(""),
+    thumbnailSource = null,
     mimeType = MimeTypes.IMAGE_JPEG,
     blurhash = "TQF5:I_NtRE4kXt7Z#MwkCIARPjr",
     width = null,

features/messages/impl/src/test/kotlin/io/element/android/features/messages/MessagesPresenterTest.kt

@@ -161,6 +161,7 @@ class MessagesPresenterTest {
                 content = TimelineItemImageContent(
                     body = "image.jpg",
                     mediaSource = MediaSource(AN_AVATAR_URL),
+                    thumbnailSource  = null,
                     mimeType = MimeTypes.Jpeg,
                     blurhash = null,
                     width = 20,

libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/CoilMediaFetcher.kt

@@ -17,42 +17,93 @@
 package io.element.android.libraries.matrix.ui.media
 
 import coil.ImageLoader
+import coil.decode.DataSource
+import coil.decode.ImageSource
 import coil.fetch.FetchResult
 import coil.fetch.Fetcher
+import coil.fetch.SourceResult
 import coil.request.Options
 import io.element.android.libraries.designsystem.components.avatar.AvatarData
 import io.element.android.libraries.matrix.api.MatrixClient
 import io.element.android.libraries.matrix.api.media.MatrixMediaLoader
+import io.element.android.libraries.matrix.api.media.MediaSource
+import io.element.android.libraries.matrix.api.media.toFile
+import okio.Buffer
+import okio.Path.Companion.toOkioPath
+import timber.log.Timber
 import java.nio.ByteBuffer
 
 internal class CoilMediaFetcher(
     private val mediaLoader: MatrixMediaLoader,
     private val mediaData: MediaRequestData?,
-    private val options: Options,
-    private val imageLoader: ImageLoader
+    private val options: Options
 ) : Fetcher {
 
     override suspend fun fetch(): FetchResult? {
-        return loadMedia()
-            .map { data ->
-                val byteBuffer = ByteBuffer.wrap(data)
-                imageLoader.components.newFetcher(byteBuffer, options, imageLoader)?.first?.fetch()
-            }.getOrThrow()
-    }
-
-    private suspend fun loadMedia(): Result<ByteArray> {
-        if (mediaData?.source == null) return Result.failure(IllegalStateException("No media data to fetch."))
+        if (mediaData?.source == null) return null
         return when (mediaData.kind) {
-            is MediaRequestData.Kind.Content -> mediaLoader.loadMediaContent(source = mediaData.source)
-            is MediaRequestData.Kind.Thumbnail -> mediaLoader.loadMediaThumbnail(
-                source = mediaData.source,
-                width = mediaData.kind.width,
-                height = mediaData.kind.height
-            )
+            is MediaRequestData.Kind.Content -> fetchContent(mediaData.source, options)
+            is MediaRequestData.Kind.Thumbnail -> fetchThumbnail(mediaData.source, mediaData.kind, options)
+            is MediaRequestData.Kind.File -> fetchFile(mediaData.source, mediaData.kind)
         }
     }
 
-    class MediaRequestDataFactory(private val client: MatrixClient) :
+    /**
+     * This method is here to avoid using [MatrixMediaLoader.loadMediaContent] as too many ByteArray allocations will flood the memory and cause lots of GC.
+     * The MediaFile will be closed (and so destroyed from disk) when the image source is closed.
+     *
+     */
+    private suspend fun fetchFile(mediaSource: MediaSource, kind: MediaRequestData.Kind.File): FetchResult? {
+        return mediaLoader.downloadMediaFile(mediaSource, kind.mimeType, kind.body)
+            .map { mediaFile ->
+                val file = mediaFile.toFile()
+                SourceResult(
+                    source = ImageSource(file = file.toOkioPath(), closeable = mediaFile),
+                    mimeType = null,
+                    dataSource = DataSource.DISK
+                )
+            }
+            .onFailure {
+                Timber.e(it)
+            }
+            .getOrNull()
+    }
+
+    private suspend fun fetchContent(mediaSource: MediaSource, options: Options): FetchResult? {
+        return mediaLoader.loadMediaContent(
+            source = mediaSource,
+        ).map { byteArray ->
+            byteArray.asSourceResult(options)
+        }.getOrNull()
+    }
+
+    private suspend fun fetchThumbnail(mediaSource: MediaSource, kind: MediaRequestData.Kind.Thumbnail, options: Options): FetchResult? {
+        return mediaLoader.loadMediaThumbnail(
+            source = mediaSource,
+            width = kind.width,
+            height = kind.height
+        ).map { byteArray ->
+            byteArray.asSourceResult(options)
+        }.getOrNull()
+    }
+
+    private fun ByteArray.asSourceResult(options: Options): SourceResult {
+        val byteBuffer = ByteBuffer.wrap(this)
+        val bufferedSource = try {
+            Buffer().apply { write(byteBuffer) }
+        } finally {
+            byteBuffer.position(0)
+        }
+        return SourceResult(
+            source = ImageSource(bufferedSource, options.context),
+            mimeType = null,
+            dataSource = DataSource.MEMORY
+        )
+    }
+
+    class MediaRequestDataFactory(
+        private val client: MatrixClient
+    ) :
         Fetcher.Factory<MediaRequestData> {
         override fun create(
             data: MediaRequestData,
@@ -62,13 +113,14 @@ internal class CoilMediaFetcher(
             return CoilMediaFetcher(
                 mediaLoader = client.mediaLoader,
                 mediaData = data,
-                options = options,
-                imageLoader = imageLoader
+                options = options
             )
         }
     }
 
-    class AvatarFactory(private val client: MatrixClient) :
+    class AvatarFactory(
+        private val client: MatrixClient
+    ) :
         Fetcher.Factory<AvatarData> {
 
         override fun create(
@@ -79,8 +131,7 @@ internal class CoilMediaFetcher(
             return CoilMediaFetcher(
                 mediaLoader = client.mediaLoader,
                 mediaData = data.toMediaRequestData(),
-                options = options,
-                imageLoader = imageLoader
+                options = options
             )
         }
     }

libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/MediaRequestData.kt

@@ -18,17 +18,28 @@ package io.element.android.libraries.matrix.ui.media
 
 import io.element.android.libraries.matrix.api.media.MediaSource
 
+/**
+ * Can be use with [coil.compose.AsyncImage] to load a [MediaSource].
+ * This will go internally through our [CoilMediaFetcher].
+ *
+ * Example of usage:
+ *  AsyncImage(
+ *      model = MediaRequestData(mediaSource, MediaRequestData.Kind.Content),
+ *      contentScale = ContentScale.Fit,
+ *  )
+ *
+ */
 data class MediaRequestData(
     val source: MediaSource?,
     val kind: Kind
 ) {
 
     sealed interface Kind {
+        object Content : Kind
+        data class File(val body: String?, val mimeType: String) : Kind
         data class Thumbnail(val width: Long, val height: Long) : Kind {
             constructor(size: Long) : this(size, size)
         }
-
-        object Content : Kind
     }
 }