Merge pull request #1659 from vector-im/langleyd/live_waveform

Live waveform

features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/composer/VoiceMessageComposerPresenter.kt

@@ -40,6 +40,7 @@ import io.element.android.libraries.textcomposer.model.VoiceMessageState
 import io.element.android.libraries.voicerecorder.api.VoiceRecorder
 import io.element.android.libraries.voicerecorder.api.VoiceRecorderState
 import io.element.android.services.analytics.api.AnalyticsService
+import kotlinx.collections.immutable.toPersistentList
 import kotlinx.collections.immutable.ImmutableList
 import kotlinx.collections.immutable.toImmutableList
 import kotlinx.coroutines.CoroutineScope
@@ -175,7 +176,7 @@ class VoiceMessageComposerPresenter @Inject constructor(
             voiceMessageState = when (val state = recorderState) {
                 is VoiceRecorderState.Recording -> VoiceMessageState.Recording(
                     duration = state.elapsedTime,
-                    level = state.level
+                    levels = state.levels.toPersistentList()
                 )
                 is VoiceRecorderState.Finished -> VoiceMessageState.Preview(
                     isSending = isSending,

features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/composer/VoiceMessageComposerStateProvider.kt

@@ -18,12 +18,13 @@ package io.element.android.features.messages.impl.voicemessages.composer
 
 import androidx.compose.ui.tooling.preview.PreviewParameterProvider
 import io.element.android.libraries.textcomposer.model.VoiceMessageState
+import kotlinx.collections.immutable.toPersistentList
 import kotlin.time.Duration.Companion.seconds
 
 internal open class VoiceMessageComposerStateProvider : PreviewParameterProvider<VoiceMessageComposerState> {
     override val values: Sequence<VoiceMessageComposerState>
         get() = sequenceOf(
-            aVoiceMessageComposerState(voiceMessageState = VoiceMessageState.Recording(duration = 61.seconds, level = 0.5f)),
+            aVoiceMessageComposerState(voiceMessageState = VoiceMessageState.Recording(duration = 61.seconds, levels = aWaveformLevels)),
         )
 }
 
@@ -35,3 +36,7 @@ internal fun aVoiceMessageComposerState(
     showPermissionRationaleDialog = showPermissionRationaleDialog,
     eventSink = {},
 )
+
+internal var aWaveformLevels = List(100) { it.toFloat() / 100 }.toPersistentList()
+
+

features/messages/impl/src/test/kotlin/io/element/android/features/messages/voicemessages/composer/VoiceMessageComposerPresenterTest.kt

@@ -44,6 +44,7 @@ import io.element.android.libraries.textcomposer.model.VoiceMessageState
 import io.element.android.libraries.voicerecorder.test.FakeVoiceRecorder
 import io.element.android.services.analytics.test.FakeAnalyticsService
 import io.element.android.tests.testutils.WarmUpRule
+import kotlinx.collections.immutable.toPersistentList
 import kotlinx.collections.immutable.toImmutableList
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.test.TestScope
@@ -67,7 +68,7 @@ class VoiceMessageComposerPresenterTest {
 
     companion object {
         private val RECORDING_DURATION = 1.seconds
-        private val RECORDING_STATE = VoiceMessageState.Recording(RECORDING_DURATION, 0.2f)
+        private val RECORDING_STATE = VoiceMessageState.Recording(RECORDING_DURATION, listOf(0.1f, 0.2f).toPersistentList())
     }
 
     @Test
@@ -146,7 +147,7 @@ class VoiceMessageComposerPresenterTest {
             }
 
             // Nothing should happen
-            assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Recording(RECORDING_DURATION, 0.2f))
+            assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Recording(RECORDING_DURATION, RECORDING_STATE.levels))
             voiceRecorder.assertCalls(started = 1, stopped = 0, deleted = 0)
 
             testPauseAndDestroy(finalState)

libraries/designsystem/src/main/kotlin/io/element/android/libraries/designsystem/components/media/DrawScopeWaveformExtensions.kt

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2023 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.element.android.libraries.designsystem.components.media
+
+import androidx.compose.ui.geometry.CornerRadius
+import androidx.compose.ui.geometry.Offset
+import androidx.compose.ui.geometry.Size
+import androidx.compose.ui.graphics.Brush
+import androidx.compose.ui.graphics.drawscope.DrawScope
+import androidx.compose.ui.graphics.drawscope.Fill
+import androidx.compose.ui.unit.Dp
+import androidx.compose.ui.unit.DpSize
+import androidx.compose.ui.unit.dp
+import kotlinx.collections.immutable.ImmutableList
+import kotlin.math.max
+
+fun DrawScope.drawWaveform(
+    waveformData: ImmutableList<Float>,
+    canvasSize: DpSize,
+    brush: Brush,
+    minimumGraphAmplitude: Float = 2F,
+    lineWidth: Dp = 2.dp,
+    linePadding: Dp = 2.dp,
+) {
+    val centerY = canvasSize.height.toPx() / 2
+    val cornerRadius = lineWidth / 2
+    waveformData.forEachIndexed { index, amplitude ->
+        val drawingAmplitude = max(minimumGraphAmplitude, amplitude * (canvasSize.height.toPx() - 2))
+        drawRoundRect(
+            brush = brush,
+            topLeft = Offset(
+                x = index * (linePadding + lineWidth).toPx(),
+                y = centerY - drawingAmplitude / 2
+            ),
+            size = Size(
+                width = lineWidth.toPx(),
+                height = drawingAmplitude
+            ),
+            cornerRadius = CornerRadius(cornerRadius.toPx(), cornerRadius.toPx()),
+            style = Fill
+        )
+    }
+}

libraries/designsystem/src/main/kotlin/io/element/android/libraries/designsystem/components/media/WaveformPlaybackView.kt

@@ -49,7 +49,6 @@ import io.element.android.libraries.theme.ElementTheme
 import kotlinx.collections.immutable.ImmutableList
 import kotlinx.collections.immutable.persistentListOf
 import kotlinx.collections.immutable.toPersistentList
-import kotlin.math.max
 import kotlin.math.roundToInt
 
 private const val DEFAULT_GRAPHICS_LAYER_ALPHA: Float = 0.99F
@@ -67,7 +66,6 @@ private const val DEFAULT_GRAPHICS_LAYER_ALPHA: Float = 0.99F
  * @param cursorBrush The brush to use to draw the cursor.
  * @param lineWidth The width of the waveform lines.
  * @param linePadding The padding between waveform lines.
- * @param minimumGraphAmplitude The minimum amplitude to display, regardless of waveform data.
  */
 @OptIn(ExperimentalComposeUiApi::class)
 @Composable
@@ -82,7 +80,6 @@ fun WaveformPlaybackView(
     cursorBrush: Brush = SolidColor(ElementTheme.colors.iconAccentTertiary),
     lineWidth: Dp = 2.dp,
     linePadding: Dp = 2.dp,
-    minimumGraphAmplitude: Float = 2F,
 ) {
     val seekProgress = remember { mutableStateOf<Float?>(null) }
     var canvasSize by remember { mutableStateOf(DpSize(0.dp, 0.dp)) }
@@ -139,22 +136,13 @@ fun WaveformPlaybackView(
         canvasSizePx = size
         val centerY = canvasSize.height.toPx() / 2
         val cornerRadius = lineWidth / 2
-        normalizedWaveformData.forEachIndexed { index, amplitude ->
-            val drawingAmplitude = max(minimumGraphAmplitude, amplitude * (canvasSize.height.toPx() - 2))
-            drawRoundRect(
-                brush = brush,
-                topLeft = Offset(
-                    x = index * (linePadding + lineWidth).toPx(),
-                    y = centerY - drawingAmplitude / 2
-                ),
-                size = Size(
-                    width = lineWidth.toPx(),
-                    height = drawingAmplitude
-                ),
-                cornerRadius = CornerRadius(cornerRadius.toPx(), cornerRadius.toPx()),
-                style = Fill
-            )
-        }
+        drawWaveform(
+            waveformData = normalizedWaveformData,
+            canvasSize = canvasSize,
+            brush = brush,
+            lineWidth = lineWidth,
+            linePadding = linePadding
+        )
         drawRect(
             brush = progressBrush,
             size = Size(

libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/TextComposer.kt

@@ -85,6 +85,7 @@ import io.element.android.wysiwyg.compose.RichTextEditor
 import io.element.android.wysiwyg.compose.RichTextEditorState
 import kotlinx.collections.immutable.ImmutableList
 import kotlinx.collections.immutable.persistentListOf
+import kotlinx.collections.immutable.toPersistentList
 import kotlin.time.Duration.Companion.seconds
 import uniffi.wysiwyg_composer.MenuAction
 
@@ -214,7 +215,7 @@ fun TextComposer(
                     onSeek = onSeekVoiceMessage,
                 )
             is VoiceMessageState.Recording ->
-                VoiceMessageRecording(voiceMessageState.level, voiceMessageState.duration)
+                VoiceMessageRecording(voiceMessageState.levels, voiceMessageState.duration)
             VoiceMessageState.Idle -> {}
         }
     }
@@ -814,7 +815,7 @@ internal fun TextComposerVoicePreview() = ElementPreview {
         enableVoiceMessages = true,
     )
     PreviewColumn(items = persistentListOf({
-        VoicePreview(voiceMessageState = VoiceMessageState.Recording(61.seconds, 0.5f))
+        VoicePreview(voiceMessageState = VoiceMessageState.Recording(61.seconds, List(100) { it.toFloat() / 100 }.toPersistentList()))
     }, {
         VoicePreview(voiceMessageState = VoiceMessageState.Preview(isSending = false, isPlaying = false, waveform = createFakeWaveform()))
     }, {

libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/LiveWaveformView.kt

@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2023 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.element.android.libraries.textcomposer.components
+
+import androidx.compose.foundation.Canvas
+import androidx.compose.foundation.layout.Box
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.fillMaxWidth
+import androidx.compose.foundation.layout.height
+import androidx.compose.foundation.layout.width
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.derivedStateOf
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableIntStateOf
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.remember
+import androidx.compose.runtime.setValue
+import androidx.compose.ui.Alignment
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.graphics.Brush
+import androidx.compose.ui.graphics.SolidColor
+import androidx.compose.ui.graphics.graphicsLayer
+import androidx.compose.ui.layout.onSizeChanged
+import androidx.compose.ui.unit.Dp
+import androidx.compose.ui.unit.DpSize
+import androidx.compose.ui.unit.dp
+import io.element.android.libraries.designsystem.components.media.drawWaveform
+import io.element.android.libraries.designsystem.preview.ElementPreview
+import io.element.android.libraries.designsystem.preview.PreviewsDayNight
+import io.element.android.libraries.theme.ElementTheme
+import kotlinx.collections.immutable.ImmutableList
+import kotlinx.collections.immutable.toPersistentList
+import java.lang.Float.min
+
+private const val DEFAULT_GRAPHICS_LAYER_ALPHA: Float = 0.99F
+private val waveFormHeight = 26.dp
+@Composable
+fun LiveWaveformView(
+    levels: ImmutableList<Float>,
+    modifier: Modifier = Modifier,
+    brush: Brush = SolidColor(ElementTheme.colors.iconQuaternary),
+    lineWidth: Dp = 2.dp,
+    linePadding: Dp = 2.dp,
+) {
+    var canvasSize by remember { mutableStateOf(DpSize(0.dp, 0.dp)) }
+
+    var parentWidth by remember { mutableIntStateOf(0) }
+
+    val waveformWidth by remember(levels, lineWidth, linePadding) {
+        derivedStateOf {
+            levels.size * (lineWidth.value + linePadding.value)
+        }
+    }
+
+
+    Box(contentAlignment = Alignment.CenterEnd,
+        modifier = modifier
+            .fillMaxWidth()
+            .height(waveFormHeight)
+            .onSizeChanged { parentWidth = it.width }
+    ) {
+        Canvas(
+            modifier = Modifier
+                .width(Dp(waveformWidth))
+                .graphicsLayer(alpha = DEFAULT_GRAPHICS_LAYER_ALPHA)
+                .then(modifier)
+        ) {
+            canvasSize = DpSize(Dp(min(waveformWidth, parentWidth.toFloat())), size.height.toDp())
+            val countThatFitsWidth = (parentWidth.toFloat() / (lineWidth.toPx() + linePadding.toPx())).toInt()
+            drawWaveform(
+                waveformData = levels.takeLast(countThatFitsWidth).toPersistentList(),
+                canvasSize = canvasSize,
+                brush = brush,
+                lineWidth = lineWidth,
+                linePadding = linePadding,
+            )
+        }
+    }
+}
+
+@PreviewsDayNight
+@Composable
+internal fun LiveWaveformViewPreview() = ElementPreview {
+    Column {
+
+        LiveWaveformView(
+            levels = List(100) { it.toFloat() / 100 }.toPersistentList(),
+            modifier = Modifier.height(34.dp),
+        )
+        LiveWaveformView(
+            levels = List(40) { it.toFloat() / 40 }.toPersistentList(),
+            modifier = Modifier.height(34.dp),
+        )
+    }
+}

libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/VoiceMessageRecording.kt

@@ -20,7 +20,6 @@ import androidx.compose.foundation.background
 import androidx.compose.foundation.layout.Box
 import androidx.compose.foundation.layout.Row
 import androidx.compose.foundation.layout.Spacer
-import androidx.compose.foundation.layout.fillMaxHeight
 import androidx.compose.foundation.layout.fillMaxWidth
 import androidx.compose.foundation.layout.height
 import androidx.compose.foundation.layout.heightIn
@@ -37,12 +36,14 @@ import io.element.android.libraries.designsystem.preview.PreviewsDayNight
 import io.element.android.libraries.designsystem.theme.components.Text
 import io.element.android.libraries.theme.ElementTheme
 import io.element.android.libraries.ui.utils.time.formatShort
+import kotlinx.collections.immutable.ImmutableList
+import kotlinx.collections.immutable.toPersistentList
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.seconds
 
 @Composable
 internal fun VoiceMessageRecording(
-    level: Float,
+    levels: ImmutableList<Float>,
     duration: Duration,
     modifier: Modifier = Modifier,
 ) {
@@ -70,28 +71,11 @@ internal fun VoiceMessageRecording(
 
         Spacer(Modifier.size(20.dp))
 
-        // TODO Replace with waveform UI
-        DebugAudioLevel(
-            modifier = Modifier.weight(1f), level = level
-        )
-    }
-}
-
-@Composable
-private fun DebugAudioLevel(
-    level: Float,
-    modifier: Modifier = Modifier,
-) {
-    Box(
-        modifier = modifier
-            .height(26.dp)
-    ) {
-        Box(
+        LiveWaveformView(
             modifier = Modifier
-                .align(Alignment.CenterEnd)
-                .fillMaxWidth(level)
-                .background(ElementTheme.colors.iconQuaternary, shape = MaterialTheme.shapes.small)
-                .fillMaxHeight()
+                .height(26.dp)
+                .weight(1f),
+            levels = levels
         )
     }
 }
@@ -108,5 +92,5 @@ private fun RedRecordingDot(
 @PreviewsDayNight
 @Composable
 internal fun VoiceMessageRecordingPreview() = ElementPreview {
-    VoiceMessageRecording(0.5f, 0.seconds)
+    VoiceMessageRecording(List(100) { it.toFloat() / 100 }.toPersistentList(), 0.seconds)
 }

libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/model/VoiceMessageState.kt

@@ -30,6 +30,6 @@ sealed class VoiceMessageState {
 
     data class Recording(
         val duration: Duration,
-        val level: Float,
+        val levels: ImmutableList<Float>,
     ): VoiceMessageState()
 }

libraries/voicerecorder/api/src/main/kotlin/io/element/android/libraries/voicerecorder/api/VoiceRecorderState.kt

@@ -29,9 +29,9 @@ sealed class VoiceRecorderState {
      * The recorder is currently recording.
      *
      * @property elapsedTime The elapsed time since the recording started.
-     * @property level The current audio level of the recording as a fraction of 1.
+     * @property levels The current audio levels of the recording as a fraction of 1.
      */
-    data class Recording(val elapsedTime: Duration, val level: Float) : VoiceRecorderState()
+    data class Recording(val elapsedTime: Duration, val levels: List<Float>) : VoiceRecorderState()
 
     /**
      * The recorder has finished recording.

libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/VoiceRecorderImpl.kt

@@ -38,6 +38,8 @@ import kotlinx.coroutines.Job
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.StateFlow
 import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
 import kotlinx.coroutines.yield
 import timber.log.Timber
 import java.io.File
@@ -67,6 +69,7 @@ class VoiceRecorderImpl @Inject constructor(
     private var audioReader: AudioReader? = null
     private var recordingJob: Job? = null
     private val levels: MutableList<Float> = mutableListOf()
+    private val lock = Mutex()
 
     private val _state = MutableStateFlow<VoiceRecorderState>(VoiceRecorderState.Idle)
     override val state: StateFlow<VoiceRecorderState> = _state
@@ -76,7 +79,10 @@ class VoiceRecorderImpl @Inject constructor(
         Timber.i("Voice recorder started recording")
         outputFile = fileManager.createFile()
             .also(encoder::init)
-        levels.clear()
+
+        lock.withLock {
+            levels.clear()
+        }
 
         val audioRecorder = audioReaderFactory.create(config, dispatchers).also { audioReader = it }
 
@@ -96,13 +102,16 @@ class VoiceRecorderImpl @Inject constructor(
                 when (audio) {
                     is Audio.Data -> {
                         val audioLevel = audioLevelCalculator.calculateAudioLevel(audio.buffer)
-                        _state.emit(VoiceRecorderState.Recording(elapsedTime, audioLevel))
-                        levels.add(audioLevel)
+
+                        lock.withLock{
+                            levels.add(audioLevel)
+                            _state.emit(VoiceRecorderState.Recording(elapsedTime, levels.toList()))
+                        }
                         encoder.encode(audio.buffer, audio.readSize)
                     }
                     is Audio.Error -> {
                         Timber.e("Voice message error: code=${audio.audioRecordErrorCode}")
-                        _state.emit(VoiceRecorderState.Recording(elapsedTime, 0.0f))
+                        _state.emit(VoiceRecorderState.Recording(elapsedTime, listOf()))
                     }
                 }
             }
@@ -126,21 +135,24 @@ class VoiceRecorderImpl @Inject constructor(
         audioReader = null
         encoder.release()
 
-        if (cancelled) {
-            deleteRecording()
-            levels.clear()
-        }
 
-        _state.emit(
-            when (val file = outputFile) {
-                null -> VoiceRecorderState.Idle
-                else -> VoiceRecorderState.Finished(
-                    file = file,
-                    mimeType = fileConfig.mimeType,
-                    waveform = levels.resample(100),
-                )
+        lock.withLock {
+            if (cancelled) {
+                deleteRecording()
+                levels.clear()
             }
-        )
+
+            _state.emit(
+                when (val file = outputFile) {
+                    null -> VoiceRecorderState.Idle
+                    else -> VoiceRecorderState.Finished(
+                        file = file,
+                        mimeType = fileConfig.mimeType,
+                        waveform = levels.resample(100),
+                    )
+                }
+            )
+        }
     }
 
     /**

libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/impl/VoiceRecorderImplTest.kt

@@ -60,11 +60,11 @@ class VoiceRecorderImplTest {
             assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Idle)
 
             voiceRecorder.startRecord()
-            assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(0.seconds, 1.0f))
+            assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(0.seconds, listOf(1.0f)))
             timeSource += 1.seconds
-            assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(1.seconds, 0.0f))
+            assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(1.seconds, listOf()))
             timeSource += 1.seconds
-            assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(2.seconds, 1.0f))
+            assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(2.seconds, listOf(1.0f, 1.0f)))
         }
     }
 
@@ -75,9 +75,9 @@ class VoiceRecorderImplTest {
             assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Idle)
 
             voiceRecorder.startRecord()
-            assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(0.minutes, 1.0f))
+            assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(0.minutes, listOf(1.0f)))
             timeSource += 29.minutes
-            assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(29.minutes, 0.0f))
+            assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(29.minutes, listOf()))
             timeSource += 1.minutes
 
             assertThat(awaitItem()).isEqualTo(

libraries/voicerecorder/test/src/main/kotlin/io/element/android/libraries/voicerecorder/test/FakeVoiceRecorder.kt

@@ -54,8 +54,8 @@ class FakeVoiceRecorder(
         curRecording = File("file.ogg")
 
         timeSource += recordingDuration
-        levels.forEach {
-            _state.emit(VoiceRecorderState.Recording(startedAt.elapsedNow(), it))
+        for(i in 1..levels.size) {
+            _state.emit(VoiceRecorderState.Recording(startedAt.elapsedNow(), levels.take(i)))
         }
     }