#748 - Implement methods for processing various media types and prepa… (#772)

* #748 - Implement methods for processing various media types and preparing them for upload

* Various tweaks following code review:
- added blurhash generation
- changed thumbnails to jpeg
- throwing if file size cannot be retrieved
- move MediaProvider files to separate folder so they can cleanly be imported in the NSE
- added audio file processing
- switched all image resizing methods to ImageIO
- various renames

* Fix blurhash formatting issues, `swift-format-ignore-file` doesn't work

* Processing all media within unique folders to avoid conflicts

* Fix various warnings

* Rename `sizeForItemAt(_ url: URL)` to `sizeForItem(at url: URL)`

* Remove unnecessary resizeImage(UIImage) method, start by copying not moving the file to the unique location

* Add back warning for roomDetails.avatarURL

ElementX/Sources/Application/Navigation/NavigationCoordinators.swift

@@ -579,7 +579,7 @@ class NavigationStackCoordinator: ObservableObject, CoordinatorProtocol, CustomS
         transaction.disablesAnimations = !animated
 
         withTransaction(transaction) {
-            stackModules.popLast()
+            _ = stackModules.popLast()
         }
     }
     

ElementX/Sources/Mocks/Generated/GeneratedMocks.swift

@@ -576,23 +576,23 @@ class RoomProxyMock: RoomProxyProtocol {
     }
     //MARK: - sendImage
 
-    var sendImageUrlCallsCount = 0
-    var sendImageUrlCalled: Bool {
-        return sendImageUrlCallsCount > 0
+    var sendImageUrlThumbnailURLImageInfoCallsCount = 0
+    var sendImageUrlThumbnailURLImageInfoCalled: Bool {
+        return sendImageUrlThumbnailURLImageInfoCallsCount > 0
     }
-    var sendImageUrlReceivedUrl: URL?
-    var sendImageUrlReceivedInvocations: [URL] = []
-    var sendImageUrlReturnValue: Result<Void, RoomProxyError>!
-    var sendImageUrlClosure: ((URL) async -> Result<Void, RoomProxyError>)?
+    var sendImageUrlThumbnailURLImageInfoReceivedArguments: (url: URL, thumbnailURL: URL, imageInfo: ImageInfo)?
+    var sendImageUrlThumbnailURLImageInfoReceivedInvocations: [(url: URL, thumbnailURL: URL, imageInfo: ImageInfo)] = []
+    var sendImageUrlThumbnailURLImageInfoReturnValue: Result<Void, RoomProxyError>!
+    var sendImageUrlThumbnailURLImageInfoClosure: ((URL, URL, ImageInfo) async -> Result<Void, RoomProxyError>)?
 
-    func sendImage(url: URL) async -> Result<Void, RoomProxyError> {
-        sendImageUrlCallsCount += 1
-        sendImageUrlReceivedUrl = url
-        sendImageUrlReceivedInvocations.append(url)
-        if let sendImageUrlClosure = sendImageUrlClosure {
-            return await sendImageUrlClosure(url)
+    func sendImage(url: URL, thumbnailURL: URL, imageInfo: ImageInfo) async -> Result<Void, RoomProxyError> {
+        sendImageUrlThumbnailURLImageInfoCallsCount += 1
+        sendImageUrlThumbnailURLImageInfoReceivedArguments = (url: url, thumbnailURL: thumbnailURL, imageInfo: imageInfo)
+        sendImageUrlThumbnailURLImageInfoReceivedInvocations.append((url: url, thumbnailURL: thumbnailURL, imageInfo: imageInfo))
+        if let sendImageUrlThumbnailURLImageInfoClosure = sendImageUrlThumbnailURLImageInfoClosure {
+            return await sendImageUrlThumbnailURLImageInfoClosure(url, thumbnailURL, imageInfo)
         } else {
-            return sendImageUrlReturnValue
+            return sendImageUrlThumbnailURLImageInfoReturnValue
         }
     }
     //MARK: - editMessage

ElementX/Sources/Other/BlurHashDecode.swift

@@ -1,7 +1,7 @@
-import UIKit
-
 // swiftlint:disable all
 
+import UIKit
+
 public extension UIImage {
     convenience init?(blurHash: String, size: CGSize, punch: Float = 1) {
         guard blurHash.count >= 6 else { return nil }

ElementX/Sources/Other/BlurHashEncode.swift

@@ -0,0 +1,145 @@
+// swiftlint:disable all
+
+import UIKit
+
+extension UIImage {
+    public func blurHash(numberOfComponents components: (Int, Int)) -> String? {
+        let pixelWidth = Int(round(size.width * scale))
+        let pixelHeight = Int(round(size.height * scale))
+        
+        let context = CGContext(data: nil,
+                                width: pixelWidth,
+                                height: pixelHeight,
+                                bitsPerComponent: 8,
+                                bytesPerRow: pixelWidth * 4,
+                                space: CGColorSpace(name: CGColorSpace.sRGB)!,
+                                bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)!
+        context.scaleBy(x: scale, y: -scale)
+        context.translateBy(x: 0, y: -size.height)
+        
+        UIGraphicsPushContext(context)
+        draw(at: .zero)
+        UIGraphicsPopContext()
+        
+        guard let cgImage = context.makeImage(),
+              let dataProvider = cgImage.dataProvider,
+              let data = dataProvider.data,
+              let pixels = CFDataGetBytePtr(data) else {
+            assertionFailure("Unexpected error!")
+            return nil
+        }
+        
+        let width = cgImage.width
+        let height = cgImage.height
+        let bytesPerRow = cgImage.bytesPerRow
+        
+        var factors: [(Float, Float, Float)] = []
+        for y in 0..<components.1 {
+            for x in 0..<components.0 {
+                let normalisation: Float = (x == 0 && y == 0) ? 1 : 2
+                let factor = multiplyBasisFunction(pixels: pixels, width: width, height: height, bytesPerRow: bytesPerRow, bytesPerPixel: cgImage.bitsPerPixel / 8, pixelOffset: 0) {
+                    normalisation * cos(Float.pi * Float(x) * $0 / Float(width)) as Float * cos(Float.pi * Float(y) * $1 / Float(height)) as Float
+                }
+                factors.append(factor)
+            }
+        }
+        
+        let dc = factors.first!
+        let ac = factors.dropFirst()
+        
+        var hash = ""
+        
+        let sizeFlag = (components.0 - 1) + (components.1 - 1) * 9
+        hash += sizeFlag.encode83(length: 1)
+        
+        let maximumValue: Float
+        if ac.count > 0 {
+            let actualMaximumValue = ac.map { max(abs($0.0), abs($0.1), abs($0.2)) }.max()!
+            let quantisedMaximumValue = Int(max(0, min(82, floor(actualMaximumValue * 166 - 0.5))))
+            maximumValue = Float(quantisedMaximumValue + 1) / 166
+            hash += quantisedMaximumValue.encode83(length: 1)
+        } else {
+            maximumValue = 1
+            hash += 0.encode83(length: 1)
+        }
+        
+        hash += encodeDC(dc).encode83(length: 4)
+        
+        for factor in ac {
+            hash += encodeAC(factor, maximumValue: maximumValue).encode83(length: 2)
+        }
+        
+        return hash
+    }
+    
+    private func multiplyBasisFunction(pixels: UnsafePointer<UInt8>, width: Int, height: Int, bytesPerRow: Int, bytesPerPixel: Int, pixelOffset: Int, basisFunction: (Float, Float) -> Float) -> (Float, Float, Float) {
+        var r: Float = 0
+        var g: Float = 0
+        var b: Float = 0
+        
+        let buffer = UnsafeBufferPointer(start: pixels, count: height * bytesPerRow)
+        
+        for x in 0..<width {
+            for y in 0..<height {
+                let basis = basisFunction(Float(x), Float(y))
+                r += basis * sRGBToLinear(buffer[bytesPerPixel * x + pixelOffset + 0 + y * bytesPerRow])
+                g += basis * sRGBToLinear(buffer[bytesPerPixel * x + pixelOffset + 1 + y * bytesPerRow])
+                b += basis * sRGBToLinear(buffer[bytesPerPixel * x + pixelOffset + 2 + y * bytesPerRow])
+            }
+        }
+        
+        let scale = 1 / Float(width * height)
+        
+        return (r * scale, g * scale, b * scale)
+    }
+}
+
+private func encodeDC(_ value: (Float, Float, Float)) -> Int {
+    let roundedR = linearTosRGB(value.0)
+    let roundedG = linearTosRGB(value.1)
+    let roundedB = linearTosRGB(value.2)
+    return (roundedR << 16) + (roundedG << 8) + roundedB
+}
+
+private func encodeAC(_ value: (Float, Float, Float), maximumValue: Float) -> Int {
+    let quantR = Int(max(0, min(18, floor(signPow(value.0 / maximumValue, 0.5) * 9 + 9.5))))
+    let quantG = Int(max(0, min(18, floor(signPow(value.1 / maximumValue, 0.5) * 9 + 9.5))))
+    let quantB = Int(max(0, min(18, floor(signPow(value.2 / maximumValue, 0.5) * 9 + 9.5))))
+    
+    return quantR * 19 * 19 + quantG * 19 + quantB
+}
+
+private func signPow(_ value: Float, _ exp: Float) -> Float {
+    copysign(pow(abs(value), exp), value)
+}
+
+private func linearTosRGB(_ value: Float) -> Int {
+    let v = max(0, min(1, value))
+    if v <= 0.0031308 { return Int(v * 12.92 * 255 + 0.5) }
+    else { return Int((1.055 * pow(v, 1 / 2.4) - 0.055) * 255 + 0.5) }
+}
+
+private func sRGBToLinear<Type: BinaryInteger>(_ value: Type) -> Float {
+    let v = Float(Int64(value)) / 255
+    if v <= 0.04045 { return v / 12.92 }
+    else { return pow((v + 0.055) / 1.055, 2.4) }
+}
+
+private let encodeCharacters: [String] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~".map { String($0) }
+
+extension BinaryInteger {
+    func encode83(length: Int) -> String {
+        var result = ""
+        for i in 1...length {
+            let digit = (Int(self) / pow(83, length - i)) % 83
+            result += encodeCharacters[Int(digit)]
+        }
+        return result
+    }
+}
+
+private func pow(_ base: Int, _ exponent: Int) -> Int {
+    (0..<exponent).reduce(1) { value, _ in value * base }
+}
+
+// swiftlint:enable all

ElementX/Sources/Other/Extensions/FileManager.swift

@@ -16,6 +16,10 @@
 
 import Foundation
 
+enum FileManagerError: Error {
+    case invalidFileSize
+}
+
 extension FileManager {
     func directoryExists(at url: URL) -> Bool {
         var isDirectory: ObjCBool = false
@@ -32,7 +36,7 @@ extension FileManager {
         try createDirectory(at: url, withIntermediateDirectories: withIntermediateDirectories)
     }
     
-    func copyFileToTemporaryDirectory(url: URL) throws -> URL {
+    func copyFileToTemporaryDirectory(file url: URL) throws -> URL {
         let newURL = URL.temporaryDirectory.appendingPathComponent(url.lastPathComponent)
         
         try? removeItem(at: newURL)
@@ -48,4 +52,17 @@ extension FileManager {
         
         return newURL
     }
+    
+    /// Retrieve a file's disk size
+    /// - Parameter url: the file URL
+    /// - Returns: the size in bytes
+    func sizeForItem(at url: URL) throws -> Double {
+        let attributes = try attributesOfItem(atPath: url.path())
+        
+        guard let size = attributes[FileAttributeKey.size] as? Double else {
+            throw FileManagerError.invalidFileSize
+        }
+        
+        return size
+    }
 }

ElementX/Sources/Screens/HomeScreen/HomeScreenViewModel.swift

@@ -219,7 +219,8 @@ class HomeScreenViewModel: HomeScreenViewModelType, HomeScreenViewModelProtocol
         var rooms = [HomeScreenRoom]()
         var createdRoomIdentifiers = [String: Bool]()
         
-        #warning("This works around duplicated room list items coming out of the SDK, remove once fixed")
+        // This works around duplicated room list items which happens because the 2 different ss lists used
+        // update at different times. That will be fixed once we move this logic to the Rust side
         func appendRoom(_ room: HomeScreenRoom, allRoomsProvider: Bool) {
             guard createdRoomIdentifiers[room.id] == nil else {
                 MXLog.error("Built duplicated room for identifier: \(room.id). AllRoomsSummaryProvider: \(allRoomsProvider). Ignoring")

ElementX/Sources/Screens/MediaPicker/DocumentPicker.swift

@@ -19,7 +19,7 @@ import SwiftUI
 enum DocumentPickerAction {
     case selectFile(URL)
     case cancel
-    case error(DocumentPickerError)
+    case error(Error)
 }
 
 enum DocumentPickerError: Error {
@@ -62,11 +62,15 @@ struct DocumentPicker: UIViewControllerRepresentable {
         
         func documentPicker(_ controller: UIDocumentPickerViewController, didPickDocumentsAt urls: [URL]) {
             guard let url = urls.first else {
-                documentPicker.callback(.error(.unknown))
+                documentPicker.callback(.error(DocumentPickerError.unknown))
                 return
             }
-            
-            documentPicker.callback(.selectFile(url))
+            do {
+                let newURL = try FileManager.default.copyFileToTemporaryDirectory(file: url)
+                documentPicker.callback(.selectFile(newURL))
+            } catch {
+                documentPicker.callback(.error(error))
+            }
         }
     }
 }

ElementX/Sources/Screens/MediaPicker/PhotoLibraryPicker.swift

@@ -69,7 +69,7 @@ struct PhotoLibraryPicker: UIViewControllerRepresentable {
                 
                 do {
                     let _ = url.startAccessingSecurityScopedResource()
-                    let newURL = try FileManager.default.copyFileToTemporaryDirectory(url: url)
+                    let newURL = try FileManager.default.copyFileToTemporaryDirectory(file: url)
                     url.stopAccessingSecurityScopedResource()
                     
                     Task { @MainActor in

ElementX/Sources/Screens/RoomScreen/RoomScreenCoordinator.swift

@@ -94,7 +94,24 @@ final class RoomScreenCoordinator: CoordinatorProtocol {
                 let mediaPickerPreviewScreenCoordinator = MediaPickerPreviewScreenCoordinator(parameters: .init(url: url, title: url.lastPathComponent)) { action in
                     switch action {
                     case .send:
-                        self?.navigationStackCoordinator.setSheetCoordinator(nil)
+                        Task {
+                            let preprocessor = MediaUploadingPreprocessor()
+                            switch await preprocessor.processMedia(at: url) {
+                            case .success(let mediaInfo):
+                                MXLog.info(mediaInfo)
+                                
+                                switch mediaInfo {
+                                case let .image(imageURL, thumbnailURL, imageInfo):
+                                    let _ = await self?.parameters.roomProxy.sendImage(url: imageURL, thumbnailURL: thumbnailURL, imageInfo: imageInfo)
+                                default:
+                                    break
+                                }
+                            case .failure(let error):
+                                MXLog.error("Failed processing media to upload with error: \(error)")
+                            }
+                            
+                            self?.navigationStackCoordinator.setSheetCoordinator(nil)
+                        }
                     case .cancel:
                         self?.navigationStackCoordinator.setSheetCoordinator(nil)
                     }

ElementX/Sources/Services/Media/MediaUploadingPreprocessor.swift

@@ -0,0 +1,405 @@
+//
+// Copyright 2023 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import AVFoundation
+import MatrixRustSDK
+import UIKit
+import UniformTypeIdentifiers
+
+indirect enum MediaUploadingPreprocessorError: Error {
+    case failedProcessingMedia(Error)
+    
+    case failedProcessingImage(MediaUploadingPreprocessorError)
+    case failedProcessingVideo(MediaUploadingPreprocessorError)
+    case failedProcessingAudio
+    case failedProcessingFile
+    
+    case failedGeneratingVideoThumbnail(Error?)
+    case failedGeneratingImageThumbnail(Error?)
+    
+    case failedStrippingLocationData
+    case failedResizingImage
+    
+    case failedConvertingVideo
+}
+
+enum MediaInfo {
+    case image(imageURL: URL, thumbnailURL: URL, imageInfo: ImageInfo)
+    case video(videoURL: URL, thumbnailURL: URL, videoInfo: VideoInfo)
+    case audio(audioURL: URL, audioInfo: AudioInfo)
+    case file(FileInfo)
+}
+
+private struct ImageProcessingInfo {
+    let url: URL
+    let height: Double
+    let width: Double
+    let mimeType: String
+    let blurhash: String?
+}
+
+private struct VideoProcessingInfo {
+    let url: URL
+    let height: Double
+    let width: Double
+    let duration: Double
+    let mimeType: String
+}
+
+struct MediaUploadingPreprocessor {
+    enum Constants {
+        static let maximumThumbnailSize = CGSize(width: 800, height: 600)
+        static let thumbnailCompressionQuality = 0.8
+        static let videoThumbnailTime = 5.0 // seconds
+    }
+    
+    /// Processes media at a given URL. It will generate thumbnails for images and videos, convert videos to 1080p mp4, strip GPS locations
+    /// from images and retrieve associated media information
+    /// - Parameter url: the file URL
+    /// - Returns: a specific type of `MediaInfo` depending on the file type and its associated details
+    func processMedia(at url: URL) async -> Result<MediaInfo, MediaUploadingPreprocessorError> {
+        // Start by copying the file to a unique temporary location in order to avoid conflicts if processing it multiple times
+        // All the other operations will be made relative to it
+        let uniqueFolder = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+        let newURL = uniqueFolder.appendingPathComponent(url.lastPathComponent)
+        do {
+            try FileManager.default.createDirectory(at: uniqueFolder, withIntermediateDirectories: true)
+            try FileManager.default.copyItem(at: url, to: newURL)
+        } catch {
+            return .failure(.failedProcessingMedia(error))
+        }
+        
+        // Process unknown types as plain files
+        guard let type = UTType(filenameExtension: newURL.pathExtension),
+              let mimeType = type.preferredMIMEType else {
+            return await processFile(at: newURL, mimeType: nil)
+        }
+        
+        if type.conforms(to: .image) {
+            return await processImage(at: newURL, type: type, mimeType: mimeType)
+        } else if type.conforms(to: .movie) || type.conforms(to: .video) {
+            return await processVideo(at: newURL)
+        } else if type.conforms(to: .audio) {
+            return await processAudio(at: newURL, mimeType: mimeType)
+        } else {
+            return await processFile(at: newURL, mimeType: mimeType)
+        }
+    }
+    
+    // MARK: - Private
+    
+    /// Prepares an image for upload. Strips location data from it and generates a thumbnail
+    /// - Parameters:
+    ///   - url: The image URL
+    ///   - type: its UTType
+    ///   - mimeType: the mimeType extracted from the UTType
+    /// - Returns: Returns a `MediaInfo.image` containing the URLs for the modified image and its thumbnail plus the corresponding `ImageInfo`
+    private func processImage(at url: URL, type: UTType, mimeType: String) async -> Result<MediaInfo, MediaUploadingPreprocessorError> {
+        switch await stripLocationFromImage(at: url, type: type, mimeType: mimeType) {
+        case .success(let result):
+            switch await generateThumbnailForImage(at: url) {
+            case .success(let thumbnailResult):
+                let imageSize = try? UInt64(FileManager.default.sizeForItem(at: result.url))
+                let thumbnailSize = try? UInt64(FileManager.default.sizeForItem(at: thumbnailResult.url))
+                
+                let thumbnailInfo = ThumbnailInfo(height: UInt64(thumbnailResult.height),
+                                                  width: UInt64(thumbnailResult.width),
+                                                  mimetype: thumbnailResult.mimeType,
+                                                  size: thumbnailSize)
+                
+                let imageInfo = ImageInfo(height: UInt64(result.height),
+                                          width: UInt64(result.width),
+                                          mimetype: result.mimeType,
+                                          size: imageSize,
+                                          thumbnailInfo: thumbnailInfo,
+                                          thumbnailSource: nil,
+                                          blurhash: thumbnailResult.blurhash)
+                
+                let mediaInfo = MediaInfo.image(imageURL: result.url, thumbnailURL: thumbnailResult.url, imageInfo: imageInfo)
+                
+                return .success(mediaInfo)
+            case .failure(let error):
+                return .failure(.failedProcessingImage(error))
+            }
+        case .failure(let error):
+            return .failure(.failedProcessingImage(error))
+        }
+    }
+    
+    /// Prepares a video for upload. Converts it to an 1080p mp4 and generates a thumbnail
+    /// - Parameters:
+    ///   - url: The video URL
+    ///   - type: its UTType
+    ///   - mimeType: the mimeType extracted from the UTType
+    /// - Returns: Returns a `MediaInfo.video` containing the URLs for the modified video and its thumbnail plus the corresponding `VideoInfo`
+    private func processVideo(at url: URL) async -> Result<MediaInfo, MediaUploadingPreprocessorError> {
+        switch await convertVideoToMP4(url) {
+        case .success(let result):
+            switch await generateThumbnailForVideoAt(result.url) {
+            case .success(let thumbnailResult):
+                let videoSize = try? UInt64(FileManager.default.sizeForItem(at: result.url))
+                let thumbnailSize = try? UInt64(FileManager.default.sizeForItem(at: thumbnailResult.url))
+                
+                let thumbnailInfo = ThumbnailInfo(height: UInt64(thumbnailResult.height),
+                                                  width: UInt64(thumbnailResult.width),
+                                                  mimetype: thumbnailResult.mimeType,
+                                                  size: thumbnailSize)
+                
+                let videoInfo = VideoInfo(duration: UInt64(result.duration),
+                                          height: UInt64(result.height),
+                                          width: UInt64(result.width),
+                                          mimetype: result.mimeType,
+                                          size: videoSize,
+                                          thumbnailInfo: thumbnailInfo,
+                                          thumbnailSource: nil,
+                                          blurhash: thumbnailResult.blurhash)
+                
+                let mediaInfo = MediaInfo.video(videoURL: result.url, thumbnailURL: thumbnailResult.url, videoInfo: videoInfo)
+                
+                return .success(mediaInfo)
+            case .failure(let error):
+                return .failure(.failedProcessingVideo(error))
+            }
+        case .failure(let error):
+            return .failure(.failedProcessingVideo(error))
+        }
+    }
+    
+    /// Prepares a file for upload.
+    /// - Parameters:
+    ///   - url: The audio URL
+    ///   - mimeType: the mimeType extracted from the UTType
+    /// - Returns: Returns a `MediaInfo.audio` containing the file URL plus the corresponding `AudioInfo`
+    private func processAudio(at url: URL, mimeType: String?) async -> Result<MediaInfo, MediaUploadingPreprocessorError> {
+        let fileSize = try? UInt64(FileManager.default.sizeForItem(at: url))
+        
+        let asset = AVURLAsset(url: url)
+        guard let durationInSeconds = try? await asset.load(.duration).seconds else {
+            return .failure(.failedProcessingAudio)
+        }
+        
+        let audioInfo = AudioInfo(duration: UInt64(durationInSeconds * 1000), size: fileSize, mimetype: mimeType)
+        return .success(.audio(audioURL: url, audioInfo: audioInfo))
+    }
+    
+    /// Prepares a file for upload.
+    /// - Parameters:
+    ///   - url: The file URL
+    ///   - type: its UTType
+    ///   - mimeType: the mimeType extracted from the UTType
+    /// - Returns: Returns a `MediaInfo.file` containing the file URL plus the corresponding `FileInfo`
+    private func processFile(at url: URL, mimeType: String?) async -> Result<MediaInfo, MediaUploadingPreprocessorError> {
+        let fileSize = try? UInt64(FileManager.default.sizeForItem(at: url))
+        
+        let fileInfo = FileInfo(mimetype: mimeType, size: fileSize, thumbnailInfo: nil, thumbnailSource: nil)
+        return .success(.file(fileInfo))
+    }
+    
+    // MARK: Images
+    
+    /// Removes the GPS dictionary from an image's metadata
+    /// - Parameters:
+    ///   - url: the URL for the original image
+    ///   - type: its UTType
+    /// - Returns: the URL for the modified image and its size as an `ImageProcessingResult`
+    private func stripLocationFromImage(at url: URL, type: UTType, mimeType: String) async -> Result<ImageProcessingInfo, MediaUploadingPreprocessorError> {
+        guard let originalData = NSData(contentsOf: url),
+              let originalCGImage = UIImage(data: originalData as Data)?.cgImage,
+              let imageSource = CGImageSourceCreateWithData(originalData, nil) else {
+            return .failure(.failedStrippingLocationData)
+        }
+        
+        guard let originalMetadata = CGImageSourceCopyPropertiesAtIndex(imageSource, 0, nil) else {
+            MXLog.info("No metadata found. Returning original image")
+            return .success(.init(url: url, height: Double(originalCGImage.height), width: Double(originalCGImage.width), mimeType: mimeType, blurhash: nil))
+        }
+        
+        guard let adjustedMetadata = (originalMetadata as NSDictionary).mutableCopy() as? NSMutableDictionary else {
+            return .failure(.failedStrippingLocationData)
+        }
+        
+        adjustedMetadata.setValue(nil, forKeyPath: "\(kCGImagePropertyGPSDictionary)")
+        
+        let data = NSMutableData()
+        guard let destination = CGImageDestinationCreateWithData(data as CFMutableData, type.identifier as CFString, 1, nil) else {
+            return .failure(.failedStrippingLocationData)
+        }
+        CGImageDestinationAddImage(destination, originalCGImage, adjustedMetadata)
+        CGImageDestinationFinalize(destination)
+        
+        do {
+            try data.write(to: url)
+            return .success(.init(url: url, height: Double(originalCGImage.height), width: Double(originalCGImage.width), mimeType: mimeType, blurhash: nil))
+        } catch {
+            return .failure(.failedStrippingLocationData)
+        }
+    }
+    
+    /// Generates a thumbnail for an image
+    /// - Parameter url: the original image URL
+    /// - Returns: the URL for the resulting thumbnail and its sizing info as an `ImageProcessingResult`
+    private func generateThumbnailForImage(at url: URL) async -> Result<ImageProcessingInfo, MediaUploadingPreprocessorError> {
+        switch await resizeImage(at: url, targetSize: Constants.maximumThumbnailSize) {
+        case .success(let thumbnail):
+            guard let data = thumbnail.jpegData(compressionQuality: Constants.thumbnailCompressionQuality) else {
+                return .failure(.failedGeneratingImageThumbnail(nil))
+            }
+            
+            let blurhash = thumbnail.blurHash(numberOfComponents: (3, 3))
+            
+            do {
+                let fileName = "thumbnail-\((url.lastPathComponent as NSString).deletingPathExtension).jpeg"
+                let thumbnailURL = url.deletingLastPathComponent().appendingPathComponent(fileName)
+                try data.write(to: thumbnailURL)
+                return .success(.init(url: thumbnailURL, height: thumbnail.size.height, width: thumbnail.size.width, mimeType: "image/jpeg", blurhash: blurhash))
+            } catch {
+                return .failure(.failedGeneratingImageThumbnail(error))
+            }
+            
+        case .failure(let error):
+            return .failure(.failedGeneratingImageThumbnail(error))
+        }
+    }
+        
+    private func resizeImage(at url: URL, targetSize: CGSize) async -> Result<UIImage, MediaUploadingPreprocessorError> {
+        let imageSource = CGImageSourceCreateWithURL(url as NSURL, nil)
+        guard let imageSource else {
+            return .failure(.failedResizingImage)
+        }
+        
+        return await resizeImage(withSource: imageSource, targetSize: targetSize)
+    }
+    
+    /// Aspect ratio resizes an image so it fits in the given size. This is useful for resizing images without loading them directly into memory
+    /// - Parameters:
+    ///   - imageSource: the original image `CGImageSource`
+    ///   - targetSize: maximum resulting size
+    /// - Returns: the resized image
+    private func resizeImage(withSource imageSource: CGImageSource, targetSize: CGSize) async -> Result<UIImage, MediaUploadingPreprocessorError> {
+        let maximumSize = min(targetSize.height, targetSize.width)
+        
+        let options: [NSString: Any] = [
+            // The maximum width and height in pixels of a thumbnail.
+            kCGImageSourceThumbnailMaxPixelSize: maximumSize,
+            kCGImageSourceCreateThumbnailFromImageAlways: true,
+            // Should include kCGImageSourceCreateThumbnailWithTransform: true in the options dictionary. Otherwise, the image result will appear rotated when an image is taken from camera in the portrait orientation.
+            kCGImageSourceCreateThumbnailWithTransform: true
+        ]
+        
+        guard let scaledImage = CGImageSourceCreateThumbnailAtIndex(imageSource, 0, options as CFDictionary) else {
+            return .failure(.failedResizingImage)
+        }
+
+        return .success(UIImage(cgImage: scaledImage))
+    }
+    
+    // MARK: Videos
+    
+    /// Generates a thumbnail for the video at the given URL
+    /// - Parameter url: the video URL
+    /// - Returns: the URL for the resulting thumbnail and its sizing info as an `ImageProcessingResult`
+    private func generateThumbnailForVideoAt(_ url: URL) async -> Result<ImageProcessingInfo, MediaUploadingPreprocessorError> {
+        let assetImageGenerator = AVAssetImageGenerator(asset: AVAsset(url: url))
+        assetImageGenerator.appliesPreferredTrackTransform = true
+        assetImageGenerator.maximumSize = Constants.maximumThumbnailSize
+        
+        do {
+            // Avoid the first frames as on a lot of videos they're black.
+            // If the specified seconds are longer than the actual video a frame close to the end of the video will be used, at AVFoundation's discretion
+            let location = CMTime(seconds: Constants.videoThumbnailTime, preferredTimescale: 1)
+            let cgImage = try await assetImageGenerator.image(at: location).image
+            
+            let thumbnail = UIImage(cgImage: cgImage)
+            
+            guard let data = thumbnail.jpegData(compressionQuality: Constants.thumbnailCompressionQuality) else {
+                return .failure(.failedGeneratingVideoThumbnail(nil))
+            }
+            
+            let blurhash = thumbnail.blurHash(numberOfComponents: (3, 3))
+            
+            let fileName = "\((url.lastPathComponent as NSString).deletingPathExtension).jpeg"
+            let thumbnailURL = url.deletingLastPathComponent().appendingPathComponent(fileName)
+            try data.write(to: thumbnailURL)
+            
+            return .success(.init(url: thumbnailURL, height: thumbnail.size.height, width: thumbnail.size.width, mimeType: "image/jpeg", blurhash: blurhash))
+            
+        } catch {
+            return .failure(.failedGeneratingVideoThumbnail(error))
+        }
+    }
+    
+    /// Converts the given video to an 1080p mp4
+    /// - Parameters:
+    ///   - url: the original video URL
+    ///   - targetFileSize: the maximum resulting file size. 90% of this will be used
+    /// - Returns: the URL for the resulting video and its media info as a `VideoProcessingResult`
+    private func convertVideoToMP4(_ url: URL, targetFileSize: UInt = 0) async -> Result<VideoProcessingInfo, MediaUploadingPreprocessorError> {
+        let asset = AVURLAsset(url: url)
+
+        guard let exportSession = AVAssetExportSession(asset: asset, presetName: AVAssetExportPreset1920x1080) else {
+            return .failure(.failedConvertingVideo)
+        }
+        
+        // AVAssetExportSession will fail if the output URL already exists
+        let uuid = UUID().uuidString
+        let originalFilenameWithoutExtension = url.deletingPathExtension().lastPathComponent
+        let outputURL = url.deletingLastPathComponent().appendingPathComponent("\(uuid)-\(originalFilenameWithoutExtension).mp4")
+        
+        try? FileManager.default.removeItem(at: outputURL)
+        
+        exportSession.outputURL = outputURL
+        exportSession.outputFileType = AVFileType.mp4
+        
+        guard exportSession.supportedFileTypes.contains(AVFileType.mp4) else {
+            return .failure(.failedConvertingVideo)
+        }
+        
+        if targetFileSize > 0 {
+            // Reduce the target file size by 10% as fileLengthLimit isn't a hard limit
+            exportSession.fileLengthLimit = Int64(Double(targetFileSize) * 0.9)
+        }
+        
+        await exportSession.export()
+        
+        switch exportSession.status {
+        case .completed:
+            do {
+                // Delete the original
+                try? FileManager.default.removeItem(at: url)
+                // Strip the UUID from the new version
+                let newOutputURL = url.deletingLastPathComponent().appendingPathComponent("\(originalFilenameWithoutExtension).mp4")
+                try FileManager.default.moveItem(at: outputURL, to: newOutputURL)
+                
+                let newAsset = AVURLAsset(url: newOutputURL)
+                guard let track = try? await newAsset.loadTracks(withMediaType: .video).first,
+                      let durationInSeconds = try? await newAsset.load(.duration).seconds,
+                      let naturalSize = try? await track.load(.naturalSize) else {
+                    return .failure(.failedConvertingVideo)
+                }
+                
+                return .success(.init(url: newOutputURL,
+                                      height: naturalSize.height,
+                                      width: naturalSize.width,
+                                      duration: durationInSeconds * 1000,
+                                      mimeType: "video/mp4"))
+            } catch {
+                return .failure(.failedConvertingVideo)
+            }
+        default:
+            return .failure(.failedConvertingVideo)
+        }
+    }
+}

ElementX/Sources/Services/Room/RoomProxy.swift

@@ -226,8 +226,21 @@ class RoomProxy: RoomProxyProtocol {
         }
     }
     
-    func sendImage(url: URL) async -> Result<Void, RoomProxyError> {
+    func sendImage(url: URL, thumbnailURL: URL, imageInfo: ImageInfo) async -> Result<Void, RoomProxyError> {
         .failure(.failedSendingMedia)
+//        sendMessageBackgroundTask = backgroundTaskService.startBackgroundTask(withName: backgroundTaskName, isReusable: true)
+//        defer {
+//            sendMessageBackgroundTask?.stop()
+//        }
+//
+//        return await Task.dispatch(on: userInitiatedDispatchQueue) {
+//            do {
+//                try self.room.sendImage(url: url.path(), thumbnailUrl: thumbnailURL.path(), imageInfo: imageInfo)
+//                return .success(())
+//            } catch {
+//                return .failure(.failedEditingMessage)
+//            }
+//        }
     }
 
     func editMessage(_ newMessage: String, original eventID: String) async -> Result<Void, RoomProxyError> {

ElementX/Sources/Services/Room/RoomProxyProtocol.swift

@@ -75,7 +75,7 @@ protocol RoomProxyProtocol {
     
     func sendReaction(_ reaction: String, to eventID: String) async -> Result<Void, RoomProxyError>
     
-    func sendImage(url: URL) async -> Result<Void, RoomProxyError>
+    func sendImage(url: URL, thumbnailURL: URL, imageInfo: ImageInfo) async -> Result<Void, RoomProxyError>
 
     func editMessage(_ newMessage: String, original eventID: String) async -> Result<Void, RoomProxyError>
     

NSE/SupportingFiles/target.yml

@@ -76,7 +76,7 @@ targets:
     - path: ../../ElementX/Sources/Services/Notification/Proxy/NotificationServiceProxy.swift
     - path: ../../ElementX/Sources/Services/Notification/Proxy/NotificationItemProxy.swift
     - path: ../../ElementX/Sources/Services/Notification/NotificationConstants.swift
-    - path: ../../ElementX/Sources/Services/Media
+    - path: ../../ElementX/Sources/Services/Media/Provider
     - path: ../../ElementX/Sources/Services/Background/BackgroundTaskServiceProtocol.swift
     - path: ../../ElementX/Sources/Services/Background/BackgroundTaskProtocol.swift
     - path: ../../ElementX/Sources/Other/Logging