Fix rageshakes not uploading if they are too long (#6075)

* Fix rageshakes not uploading if they are too long. The max log line size in the default rageshake server is `1_000_000`.

* Change upload order to ensure the current logs and the push rules are always sent.

* Add 'buffer' for unexpected log lines.

appconfig/src/main/kotlin/io/element/android/appconfig/RageshakeConfig.kt

@@ -30,4 +30,9 @@ object RageshakeConfig {
      * The maximum size of a single log file.
      */
     const val MAX_LOG_CONTENT_SIZE = 100 * 1024 * 1024L
+
+    /**
+     * The maximum number of log lines a rageshake can contain.
+     */
+    const val MAX_LOG_LINES_SIZE = 1_000_000
 }

features/rageshake/impl/src/main/kotlin/io/element/android/features/rageshake/impl/reporter/DefaultBugReporter.kt

@@ -135,6 +135,10 @@ class DefaultBugReporter(
         // enumerate files to delete
         val bugReportFiles: MutableList<File> = ArrayList()
         var response: Response? = null
+
+        // Start at something like 1000 lines to have some 'buffer' in case unexpected lines were added
+        var totalLogLines = 1000L
+
         try {
             var serverError: String? = null
             withContext(coroutineDispatchers.io) {
@@ -147,23 +151,12 @@ class DefaultBugReporter(
                     }
                 }
                 val gzippedFiles = mutableListOf<File>()
-                var filesTooBig = 0
+                var filesTooBig = emptyList<String>()
-                if (withDevicesLogs) {
+
-                    val files = getLogFiles().sortedByDescending { it.lastModified() }
-                    val filesBySize = files.groupBy {
-                        it.length() < RageshakeConfig.MAX_LOG_CONTENT_SIZE
-                    }
-                    filesBySize[true].orEmpty().mapNotNullTo(gzippedFiles) { file ->
-                        when {
-                            file.extension == "gz" -> file
-                            else -> compressFile(file)
-                        }
-                    }
-                    filesTooBig = filesBySize[false].orEmpty().size
-                }
                 if (withCrashLogs || withDevicesLogs) {
                     saveLogCat()
                         ?.takeIf { it.length() < RageshakeConfig.MAX_LOG_CONTENT_SIZE }
+                        ?.takeIf { countLogLines(it) + totalLogLines < RageshakeConfig.MAX_LOG_LINES_SIZE }
                         ?.let { logCatFile ->
                             compressFile(logCatFile).also {
                                 logCatFile.safeDelete()
@@ -197,9 +190,7 @@ class DefaultBugReporter(
                     .addFormDataPart("label", buildMeta.versionName)
                     .addFormDataPart("label", buildMeta.flavorDescription)
                     .addFormDataPart("branch_name", buildMeta.gitBranchName)
-                if (filesTooBig > 0) {
+
-                    builder.addFormDataPart("omitted_logs", filesTooBig.toString())
-                }
                 userId?.let {
                     matrixClientProvider.getOrNull(it)?.let { client ->
                         val curveKey = client.encryptionService.deviceCurve25519()
@@ -210,15 +201,57 @@ class DefaultBugReporter(
 
                         if (sendPushRules) {
                             client.notificationSettingsService.getRawPushRules().getOrNull()?.let { pushRules ->
-                                builder.addFormDataPart(
+                                val logLines = pushRules.lineSequence().count()
-                                    name = "file",
+
-                                    filename = "push_rules.json",
+                                if (totalLogLines + logLines < RageshakeConfig.MAX_LOG_LINES_SIZE) {
-                                    body = pushRules.toByteArray().toRequestBody(MimeTypes.Json.toMediaTypeOrNull())
+                                    builder.addFormDataPart(
-                                )
+                                        name = "file",
+                                        filename = "push_rules.json",
+                                        body = pushRules.toByteArray().toRequestBody(MimeTypes.Json.toMediaTypeOrNull())
+                                    )
+                                } else {
+                                    Timber.w("Could not upload push rules because it would exceed the max log lines size")
+                                }
                             }
                         }
                     }
                 }
+
+                if (withDevicesLogs) {
+                    val files = getLogFiles().sortedByDescending { it.lastModified() }
+                    val filesBySize = files.groupBy {
+                        it.length() < RageshakeConfig.MAX_LOG_CONTENT_SIZE
+                    }.toMutableMap()
+
+                    filesBySize[true].orEmpty().mapNotNullTo(gzippedFiles) { file ->
+                        val logLines = countLogLines(file)
+                        totalLogLines += logLines
+
+                        when {
+                            totalLogLines > RageshakeConfig.MAX_LOG_LINES_SIZE -> {
+                                // Add it to the list of omitted files too
+                                (filesBySize.getOrPut(false) { mutableListOf() } as MutableList<File>).add(file)
+
+                                Timber.e(
+                                    "Could not upload file ${file.name} because it would exceed the max log lines size " +
+                                        "($totalLogLines/${RageshakeConfig.MAX_LOG_LINES_SIZE}"
+                                )
+
+                                totalLogLines -= logLines
+
+                                null
+                            }
+                            file.extension == "gz" -> file
+                            else -> compressFile(file)
+                        }
+                    }
+                    filesTooBig = filesBySize[false].orEmpty().map { it.name }
+                }
+
+                if (filesTooBig.isNotEmpty()) {
+                    builder.addFormDataPart("omitted_logs", filesTooBig.toString())
+                }
+
                 if (crashCallStack.isNotEmpty() && withCrashLogs) {
                     builder.addFormDataPart("label", "crash")
                 }
@@ -453,4 +486,8 @@ class DefaultBugReporter(
             Timber.e(e, "getLogCatContent fails")
         }
     }
+
+    private fun countLogLines(file: File): Int {
+        return file.reader().useLines { it.count() }
+    }
 }

features/rageshake/impl/src/test/kotlin/io/element/android/features/rageshake/impl/reporter/DefaultBugReporterTest.kt

@@ -536,6 +536,6 @@ class DefaultBugReporterTest {
     }
 
     companion object {
-        private const val EXPECTED_NUMBER_OF_PROGRESS_VALUE = 18
+        private const val EXPECTED_NUMBER_OF_PROGRESS_VALUE = 17
     }
 }