From b391d7f698319ea659ba270c2516644e84779c23 Mon Sep 17 00:00:00 2001 From: Stefan Ceriu Date: Fri, 21 Apr 2023 12:13:36 +0300 Subject: [PATCH] =?UTF-8?q?#748=20-=20Implement=20methods=20for=20processi?= =?UTF-8?q?ng=20various=20media=20types=20and=20prepa=E2=80=A6=20(#772)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * #748 - Implement methods for processing various media types and preparing them for upload * Various tweaks following code review: - added blurhash generation - changed thumbnails to jpeg - throwing if file size cannot be retrieved - move MediaProvider files to separate folder so they can cleanly be imported in the NSE - added audio file processing - switched all image resizing methods to ImageIO - various renames * Fix blurhash formatting issues, `swift-format-ignore-file` doesn't work * Processing all media within unique folders to avoid conflicts * Fix various warnings * Rename `sizeForItemAt(_ url: URL)` to `sizeForItem(at url: URL)` * Remove unnecessary resizeImage(UIImage) method, start by copying not moving the file to the unique location * Add back warning for roomDetails.avatarURL --- .../Navigation/NavigationCoordinators.swift | 2 +- .../Mocks/Generated/GeneratedMocks.swift | 28 +- ElementX/Sources/Other/BlurHashDecode.swift | 4 +- ElementX/Sources/Other/BlurHashEncode.swift | 145 +++++++ .../Other/Extensions/FileManager.swift | 19 +- .../HomeScreen/HomeScreenViewModel.swift | 3 +- .../Screens/MediaPicker/DocumentPicker.swift | 12 +- .../MediaPicker/PhotoLibraryPicker.swift | 2 +- .../RoomScreen/RoomScreenCoordinator.swift | 19 +- .../Media/MediaUploadingPreprocessor.swift | 405 ++++++++++++++++++ .../ImageProviderProtocol.swift | 0 .../{ => Provider}/MediaFileHandleProxy.swift | 0 .../Media/{ => Provider}/MediaLoader.swift | 0 .../{ => Provider}/MediaLoaderProtocol.swift | 0 .../Media/{ => Provider}/MediaProvider.swift | 0 .../MediaProviderProtocol.swift | 0 .../{ => Provider}/MediaSourceProxy.swift | 0 .../{ => Provider}/MockMediaProvider.swift | 0 .../Sources/Services/Room/RoomProxy.swift | 15 +- .../Services/Room/RoomProxyProtocol.swift | 2 +- NSE/SupportingFiles/target.yml | 2 +- 21 files changed, 630 insertions(+), 28 deletions(-) create mode 100644 ElementX/Sources/Other/BlurHashEncode.swift create mode 100644 ElementX/Sources/Services/Media/MediaUploadingPreprocessor.swift rename ElementX/Sources/Services/Media/{ => Provider}/ImageProviderProtocol.swift (100%) rename ElementX/Sources/Services/Media/{ => Provider}/MediaFileHandleProxy.swift (100%) rename ElementX/Sources/Services/Media/{ => Provider}/MediaLoader.swift (100%) rename ElementX/Sources/Services/Media/{ => Provider}/MediaLoaderProtocol.swift (100%) rename ElementX/Sources/Services/Media/{ => Provider}/MediaProvider.swift (100%) rename ElementX/Sources/Services/Media/{ => Provider}/MediaProviderProtocol.swift (100%) rename ElementX/Sources/Services/Media/{ => Provider}/MediaSourceProxy.swift (100%) rename ElementX/Sources/Services/Media/{ => Provider}/MockMediaProvider.swift (100%) diff --git a/ElementX/Sources/Application/Navigation/NavigationCoordinators.swift b/ElementX/Sources/Application/Navigation/NavigationCoordinators.swift index afcf09223..5b6462745 100644 --- a/ElementX/Sources/Application/Navigation/NavigationCoordinators.swift +++ b/ElementX/Sources/Application/Navigation/NavigationCoordinators.swift @@ -579,7 +579,7 @@ class NavigationStackCoordinator: ObservableObject, CoordinatorProtocol, CustomS transaction.disablesAnimations = !animated withTransaction(transaction) { - stackModules.popLast() + _ = stackModules.popLast() } } diff --git a/ElementX/Sources/Mocks/Generated/GeneratedMocks.swift b/ElementX/Sources/Mocks/Generated/GeneratedMocks.swift index e0641f637..ac2a498e5 100644 --- a/ElementX/Sources/Mocks/Generated/GeneratedMocks.swift +++ b/ElementX/Sources/Mocks/Generated/GeneratedMocks.swift @@ -576,23 +576,23 @@ class RoomProxyMock: RoomProxyProtocol { } //MARK: - sendImage - var sendImageUrlCallsCount = 0 - var sendImageUrlCalled: Bool { - return sendImageUrlCallsCount > 0 + var sendImageUrlThumbnailURLImageInfoCallsCount = 0 + var sendImageUrlThumbnailURLImageInfoCalled: Bool { + return sendImageUrlThumbnailURLImageInfoCallsCount > 0 } - var sendImageUrlReceivedUrl: URL? - var sendImageUrlReceivedInvocations: [URL] = [] - var sendImageUrlReturnValue: Result! - var sendImageUrlClosure: ((URL) async -> Result)? + var sendImageUrlThumbnailURLImageInfoReceivedArguments: (url: URL, thumbnailURL: URL, imageInfo: ImageInfo)? + var sendImageUrlThumbnailURLImageInfoReceivedInvocations: [(url: URL, thumbnailURL: URL, imageInfo: ImageInfo)] = [] + var sendImageUrlThumbnailURLImageInfoReturnValue: Result! + var sendImageUrlThumbnailURLImageInfoClosure: ((URL, URL, ImageInfo) async -> Result)? - func sendImage(url: URL) async -> Result { - sendImageUrlCallsCount += 1 - sendImageUrlReceivedUrl = url - sendImageUrlReceivedInvocations.append(url) - if let sendImageUrlClosure = sendImageUrlClosure { - return await sendImageUrlClosure(url) + func sendImage(url: URL, thumbnailURL: URL, imageInfo: ImageInfo) async -> Result { + sendImageUrlThumbnailURLImageInfoCallsCount += 1 + sendImageUrlThumbnailURLImageInfoReceivedArguments = (url: url, thumbnailURL: thumbnailURL, imageInfo: imageInfo) + sendImageUrlThumbnailURLImageInfoReceivedInvocations.append((url: url, thumbnailURL: thumbnailURL, imageInfo: imageInfo)) + if let sendImageUrlThumbnailURLImageInfoClosure = sendImageUrlThumbnailURLImageInfoClosure { + return await sendImageUrlThumbnailURLImageInfoClosure(url, thumbnailURL, imageInfo) } else { - return sendImageUrlReturnValue + return sendImageUrlThumbnailURLImageInfoReturnValue } } //MARK: - editMessage diff --git a/ElementX/Sources/Other/BlurHashDecode.swift b/ElementX/Sources/Other/BlurHashDecode.swift index 0a90d2ae2..c1127d918 100644 --- a/ElementX/Sources/Other/BlurHashDecode.swift +++ b/ElementX/Sources/Other/BlurHashDecode.swift @@ -1,7 +1,7 @@ -import UIKit - // swiftlint:disable all +import UIKit + public extension UIImage { convenience init?(blurHash: String, size: CGSize, punch: Float = 1) { guard blurHash.count >= 6 else { return nil } diff --git a/ElementX/Sources/Other/BlurHashEncode.swift b/ElementX/Sources/Other/BlurHashEncode.swift new file mode 100644 index 000000000..f483a162b --- /dev/null +++ b/ElementX/Sources/Other/BlurHashEncode.swift @@ -0,0 +1,145 @@ +// swiftlint:disable all + +import UIKit + +extension UIImage { + public func blurHash(numberOfComponents components: (Int, Int)) -> String? { + let pixelWidth = Int(round(size.width * scale)) + let pixelHeight = Int(round(size.height * scale)) + + let context = CGContext(data: nil, + width: pixelWidth, + height: pixelHeight, + bitsPerComponent: 8, + bytesPerRow: pixelWidth * 4, + space: CGColorSpace(name: CGColorSpace.sRGB)!, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)! + context.scaleBy(x: scale, y: -scale) + context.translateBy(x: 0, y: -size.height) + + UIGraphicsPushContext(context) + draw(at: .zero) + UIGraphicsPopContext() + + guard let cgImage = context.makeImage(), + let dataProvider = cgImage.dataProvider, + let data = dataProvider.data, + let pixels = CFDataGetBytePtr(data) else { + assertionFailure("Unexpected error!") + return nil + } + + let width = cgImage.width + let height = cgImage.height + let bytesPerRow = cgImage.bytesPerRow + + var factors: [(Float, Float, Float)] = [] + for y in 0.. 0 { + let actualMaximumValue = ac.map { max(abs($0.0), abs($0.1), abs($0.2)) }.max()! + let quantisedMaximumValue = Int(max(0, min(82, floor(actualMaximumValue * 166 - 0.5)))) + maximumValue = Float(quantisedMaximumValue + 1) / 166 + hash += quantisedMaximumValue.encode83(length: 1) + } else { + maximumValue = 1 + hash += 0.encode83(length: 1) + } + + hash += encodeDC(dc).encode83(length: 4) + + for factor in ac { + hash += encodeAC(factor, maximumValue: maximumValue).encode83(length: 2) + } + + return hash + } + + private func multiplyBasisFunction(pixels: UnsafePointer, width: Int, height: Int, bytesPerRow: Int, bytesPerPixel: Int, pixelOffset: Int, basisFunction: (Float, Float) -> Float) -> (Float, Float, Float) { + var r: Float = 0 + var g: Float = 0 + var b: Float = 0 + + let buffer = UnsafeBufferPointer(start: pixels, count: height * bytesPerRow) + + for x in 0.. Int { + let roundedR = linearTosRGB(value.0) + let roundedG = linearTosRGB(value.1) + let roundedB = linearTosRGB(value.2) + return (roundedR << 16) + (roundedG << 8) + roundedB +} + +private func encodeAC(_ value: (Float, Float, Float), maximumValue: Float) -> Int { + let quantR = Int(max(0, min(18, floor(signPow(value.0 / maximumValue, 0.5) * 9 + 9.5)))) + let quantG = Int(max(0, min(18, floor(signPow(value.1 / maximumValue, 0.5) * 9 + 9.5)))) + let quantB = Int(max(0, min(18, floor(signPow(value.2 / maximumValue, 0.5) * 9 + 9.5)))) + + return quantR * 19 * 19 + quantG * 19 + quantB +} + +private func signPow(_ value: Float, _ exp: Float) -> Float { + copysign(pow(abs(value), exp), value) +} + +private func linearTosRGB(_ value: Float) -> Int { + let v = max(0, min(1, value)) + if v <= 0.0031308 { return Int(v * 12.92 * 255 + 0.5) } + else { return Int((1.055 * pow(v, 1 / 2.4) - 0.055) * 255 + 0.5) } +} + +private func sRGBToLinear(_ value: Type) -> Float { + let v = Float(Int64(value)) / 255 + if v <= 0.04045 { return v / 12.92 } + else { return pow((v + 0.055) / 1.055, 2.4) } +} + +private let encodeCharacters: [String] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~".map { String($0) } + +extension BinaryInteger { + func encode83(length: Int) -> String { + var result = "" + for i in 1...length { + let digit = (Int(self) / pow(83, length - i)) % 83 + result += encodeCharacters[Int(digit)] + } + return result + } +} + +private func pow(_ base: Int, _ exponent: Int) -> Int { + (0.. Bool { var isDirectory: ObjCBool = false @@ -32,7 +36,7 @@ extension FileManager { try createDirectory(at: url, withIntermediateDirectories: withIntermediateDirectories) } - func copyFileToTemporaryDirectory(url: URL) throws -> URL { + func copyFileToTemporaryDirectory(file url: URL) throws -> URL { let newURL = URL.temporaryDirectory.appendingPathComponent(url.lastPathComponent) try? removeItem(at: newURL) @@ -48,4 +52,17 @@ extension FileManager { return newURL } + + /// Retrieve a file's disk size + /// - Parameter url: the file URL + /// - Returns: the size in bytes + func sizeForItem(at url: URL) throws -> Double { + let attributes = try attributesOfItem(atPath: url.path()) + + guard let size = attributes[FileAttributeKey.size] as? Double else { + throw FileManagerError.invalidFileSize + } + + return size + } } diff --git a/ElementX/Sources/Screens/HomeScreen/HomeScreenViewModel.swift b/ElementX/Sources/Screens/HomeScreen/HomeScreenViewModel.swift index c795cf9da..575504d1f 100644 --- a/ElementX/Sources/Screens/HomeScreen/HomeScreenViewModel.swift +++ b/ElementX/Sources/Screens/HomeScreen/HomeScreenViewModel.swift @@ -219,7 +219,8 @@ class HomeScreenViewModel: HomeScreenViewModelType, HomeScreenViewModelProtocol var rooms = [HomeScreenRoom]() var createdRoomIdentifiers = [String: Bool]() - #warning("This works around duplicated room list items coming out of the SDK, remove once fixed") + // This works around duplicated room list items which happens because the 2 different ss lists used + // update at different times. That will be fixed once we move this logic to the Rust side func appendRoom(_ room: HomeScreenRoom, allRoomsProvider: Bool) { guard createdRoomIdentifiers[room.id] == nil else { MXLog.error("Built duplicated room for identifier: \(room.id). AllRoomsSummaryProvider: \(allRoomsProvider). Ignoring") diff --git a/ElementX/Sources/Screens/MediaPicker/DocumentPicker.swift b/ElementX/Sources/Screens/MediaPicker/DocumentPicker.swift index 4b93c3a06..9d53aa351 100644 --- a/ElementX/Sources/Screens/MediaPicker/DocumentPicker.swift +++ b/ElementX/Sources/Screens/MediaPicker/DocumentPicker.swift @@ -19,7 +19,7 @@ import SwiftUI enum DocumentPickerAction { case selectFile(URL) case cancel - case error(DocumentPickerError) + case error(Error) } enum DocumentPickerError: Error { @@ -62,11 +62,15 @@ struct DocumentPicker: UIViewControllerRepresentable { func documentPicker(_ controller: UIDocumentPickerViewController, didPickDocumentsAt urls: [URL]) { guard let url = urls.first else { - documentPicker.callback(.error(.unknown)) + documentPicker.callback(.error(DocumentPickerError.unknown)) return } - - documentPicker.callback(.selectFile(url)) + do { + let newURL = try FileManager.default.copyFileToTemporaryDirectory(file: url) + documentPicker.callback(.selectFile(newURL)) + } catch { + documentPicker.callback(.error(error)) + } } } } diff --git a/ElementX/Sources/Screens/MediaPicker/PhotoLibraryPicker.swift b/ElementX/Sources/Screens/MediaPicker/PhotoLibraryPicker.swift index 51a91dbb1..b069de75d 100644 --- a/ElementX/Sources/Screens/MediaPicker/PhotoLibraryPicker.swift +++ b/ElementX/Sources/Screens/MediaPicker/PhotoLibraryPicker.swift @@ -69,7 +69,7 @@ struct PhotoLibraryPicker: UIViewControllerRepresentable { do { let _ = url.startAccessingSecurityScopedResource() - let newURL = try FileManager.default.copyFileToTemporaryDirectory(url: url) + let newURL = try FileManager.default.copyFileToTemporaryDirectory(file: url) url.stopAccessingSecurityScopedResource() Task { @MainActor in diff --git a/ElementX/Sources/Screens/RoomScreen/RoomScreenCoordinator.swift b/ElementX/Sources/Screens/RoomScreen/RoomScreenCoordinator.swift index 7bdd86c3c..96f756c26 100644 --- a/ElementX/Sources/Screens/RoomScreen/RoomScreenCoordinator.swift +++ b/ElementX/Sources/Screens/RoomScreen/RoomScreenCoordinator.swift @@ -94,7 +94,24 @@ final class RoomScreenCoordinator: CoordinatorProtocol { let mediaPickerPreviewScreenCoordinator = MediaPickerPreviewScreenCoordinator(parameters: .init(url: url, title: url.lastPathComponent)) { action in switch action { case .send: - self?.navigationStackCoordinator.setSheetCoordinator(nil) + Task { + let preprocessor = MediaUploadingPreprocessor() + switch await preprocessor.processMedia(at: url) { + case .success(let mediaInfo): + MXLog.info(mediaInfo) + + switch mediaInfo { + case let .image(imageURL, thumbnailURL, imageInfo): + let _ = await self?.parameters.roomProxy.sendImage(url: imageURL, thumbnailURL: thumbnailURL, imageInfo: imageInfo) + default: + break + } + case .failure(let error): + MXLog.error("Failed processing media to upload with error: \(error)") + } + + self?.navigationStackCoordinator.setSheetCoordinator(nil) + } case .cancel: self?.navigationStackCoordinator.setSheetCoordinator(nil) } diff --git a/ElementX/Sources/Services/Media/MediaUploadingPreprocessor.swift b/ElementX/Sources/Services/Media/MediaUploadingPreprocessor.swift new file mode 100644 index 000000000..47b084008 --- /dev/null +++ b/ElementX/Sources/Services/Media/MediaUploadingPreprocessor.swift @@ -0,0 +1,405 @@ +// +// Copyright 2023 New Vector Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import AVFoundation +import MatrixRustSDK +import UIKit +import UniformTypeIdentifiers + +indirect enum MediaUploadingPreprocessorError: Error { + case failedProcessingMedia(Error) + + case failedProcessingImage(MediaUploadingPreprocessorError) + case failedProcessingVideo(MediaUploadingPreprocessorError) + case failedProcessingAudio + case failedProcessingFile + + case failedGeneratingVideoThumbnail(Error?) + case failedGeneratingImageThumbnail(Error?) + + case failedStrippingLocationData + case failedResizingImage + + case failedConvertingVideo +} + +enum MediaInfo { + case image(imageURL: URL, thumbnailURL: URL, imageInfo: ImageInfo) + case video(videoURL: URL, thumbnailURL: URL, videoInfo: VideoInfo) + case audio(audioURL: URL, audioInfo: AudioInfo) + case file(FileInfo) +} + +private struct ImageProcessingInfo { + let url: URL + let height: Double + let width: Double + let mimeType: String + let blurhash: String? +} + +private struct VideoProcessingInfo { + let url: URL + let height: Double + let width: Double + let duration: Double + let mimeType: String +} + +struct MediaUploadingPreprocessor { + enum Constants { + static let maximumThumbnailSize = CGSize(width: 800, height: 600) + static let thumbnailCompressionQuality = 0.8 + static let videoThumbnailTime = 5.0 // seconds + } + + /// Processes media at a given URL. It will generate thumbnails for images and videos, convert videos to 1080p mp4, strip GPS locations + /// from images and retrieve associated media information + /// - Parameter url: the file URL + /// - Returns: a specific type of `MediaInfo` depending on the file type and its associated details + func processMedia(at url: URL) async -> Result { + // Start by copying the file to a unique temporary location in order to avoid conflicts if processing it multiple times + // All the other operations will be made relative to it + let uniqueFolder = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let newURL = uniqueFolder.appendingPathComponent(url.lastPathComponent) + do { + try FileManager.default.createDirectory(at: uniqueFolder, withIntermediateDirectories: true) + try FileManager.default.copyItem(at: url, to: newURL) + } catch { + return .failure(.failedProcessingMedia(error)) + } + + // Process unknown types as plain files + guard let type = UTType(filenameExtension: newURL.pathExtension), + let mimeType = type.preferredMIMEType else { + return await processFile(at: newURL, mimeType: nil) + } + + if type.conforms(to: .image) { + return await processImage(at: newURL, type: type, mimeType: mimeType) + } else if type.conforms(to: .movie) || type.conforms(to: .video) { + return await processVideo(at: newURL) + } else if type.conforms(to: .audio) { + return await processAudio(at: newURL, mimeType: mimeType) + } else { + return await processFile(at: newURL, mimeType: mimeType) + } + } + + // MARK: - Private + + /// Prepares an image for upload. Strips location data from it and generates a thumbnail + /// - Parameters: + /// - url: The image URL + /// - type: its UTType + /// - mimeType: the mimeType extracted from the UTType + /// - Returns: Returns a `MediaInfo.image` containing the URLs for the modified image and its thumbnail plus the corresponding `ImageInfo` + private func processImage(at url: URL, type: UTType, mimeType: String) async -> Result { + switch await stripLocationFromImage(at: url, type: type, mimeType: mimeType) { + case .success(let result): + switch await generateThumbnailForImage(at: url) { + case .success(let thumbnailResult): + let imageSize = try? UInt64(FileManager.default.sizeForItem(at: result.url)) + let thumbnailSize = try? UInt64(FileManager.default.sizeForItem(at: thumbnailResult.url)) + + let thumbnailInfo = ThumbnailInfo(height: UInt64(thumbnailResult.height), + width: UInt64(thumbnailResult.width), + mimetype: thumbnailResult.mimeType, + size: thumbnailSize) + + let imageInfo = ImageInfo(height: UInt64(result.height), + width: UInt64(result.width), + mimetype: result.mimeType, + size: imageSize, + thumbnailInfo: thumbnailInfo, + thumbnailSource: nil, + blurhash: thumbnailResult.blurhash) + + let mediaInfo = MediaInfo.image(imageURL: result.url, thumbnailURL: thumbnailResult.url, imageInfo: imageInfo) + + return .success(mediaInfo) + case .failure(let error): + return .failure(.failedProcessingImage(error)) + } + case .failure(let error): + return .failure(.failedProcessingImage(error)) + } + } + + /// Prepares a video for upload. Converts it to an 1080p mp4 and generates a thumbnail + /// - Parameters: + /// - url: The video URL + /// - type: its UTType + /// - mimeType: the mimeType extracted from the UTType + /// - Returns: Returns a `MediaInfo.video` containing the URLs for the modified video and its thumbnail plus the corresponding `VideoInfo` + private func processVideo(at url: URL) async -> Result { + switch await convertVideoToMP4(url) { + case .success(let result): + switch await generateThumbnailForVideoAt(result.url) { + case .success(let thumbnailResult): + let videoSize = try? UInt64(FileManager.default.sizeForItem(at: result.url)) + let thumbnailSize = try? UInt64(FileManager.default.sizeForItem(at: thumbnailResult.url)) + + let thumbnailInfo = ThumbnailInfo(height: UInt64(thumbnailResult.height), + width: UInt64(thumbnailResult.width), + mimetype: thumbnailResult.mimeType, + size: thumbnailSize) + + let videoInfo = VideoInfo(duration: UInt64(result.duration), + height: UInt64(result.height), + width: UInt64(result.width), + mimetype: result.mimeType, + size: videoSize, + thumbnailInfo: thumbnailInfo, + thumbnailSource: nil, + blurhash: thumbnailResult.blurhash) + + let mediaInfo = MediaInfo.video(videoURL: result.url, thumbnailURL: thumbnailResult.url, videoInfo: videoInfo) + + return .success(mediaInfo) + case .failure(let error): + return .failure(.failedProcessingVideo(error)) + } + case .failure(let error): + return .failure(.failedProcessingVideo(error)) + } + } + + /// Prepares a file for upload. + /// - Parameters: + /// - url: The audio URL + /// - mimeType: the mimeType extracted from the UTType + /// - Returns: Returns a `MediaInfo.audio` containing the file URL plus the corresponding `AudioInfo` + private func processAudio(at url: URL, mimeType: String?) async -> Result { + let fileSize = try? UInt64(FileManager.default.sizeForItem(at: url)) + + let asset = AVURLAsset(url: url) + guard let durationInSeconds = try? await asset.load(.duration).seconds else { + return .failure(.failedProcessingAudio) + } + + let audioInfo = AudioInfo(duration: UInt64(durationInSeconds * 1000), size: fileSize, mimetype: mimeType) + return .success(.audio(audioURL: url, audioInfo: audioInfo)) + } + + /// Prepares a file for upload. + /// - Parameters: + /// - url: The file URL + /// - type: its UTType + /// - mimeType: the mimeType extracted from the UTType + /// - Returns: Returns a `MediaInfo.file` containing the file URL plus the corresponding `FileInfo` + private func processFile(at url: URL, mimeType: String?) async -> Result { + let fileSize = try? UInt64(FileManager.default.sizeForItem(at: url)) + + let fileInfo = FileInfo(mimetype: mimeType, size: fileSize, thumbnailInfo: nil, thumbnailSource: nil) + return .success(.file(fileInfo)) + } + + // MARK: Images + + /// Removes the GPS dictionary from an image's metadata + /// - Parameters: + /// - url: the URL for the original image + /// - type: its UTType + /// - Returns: the URL for the modified image and its size as an `ImageProcessingResult` + private func stripLocationFromImage(at url: URL, type: UTType, mimeType: String) async -> Result { + guard let originalData = NSData(contentsOf: url), + let originalCGImage = UIImage(data: originalData as Data)?.cgImage, + let imageSource = CGImageSourceCreateWithData(originalData, nil) else { + return .failure(.failedStrippingLocationData) + } + + guard let originalMetadata = CGImageSourceCopyPropertiesAtIndex(imageSource, 0, nil) else { + MXLog.info("No metadata found. Returning original image") + return .success(.init(url: url, height: Double(originalCGImage.height), width: Double(originalCGImage.width), mimeType: mimeType, blurhash: nil)) + } + + guard let adjustedMetadata = (originalMetadata as NSDictionary).mutableCopy() as? NSMutableDictionary else { + return .failure(.failedStrippingLocationData) + } + + adjustedMetadata.setValue(nil, forKeyPath: "\(kCGImagePropertyGPSDictionary)") + + let data = NSMutableData() + guard let destination = CGImageDestinationCreateWithData(data as CFMutableData, type.identifier as CFString, 1, nil) else { + return .failure(.failedStrippingLocationData) + } + CGImageDestinationAddImage(destination, originalCGImage, adjustedMetadata) + CGImageDestinationFinalize(destination) + + do { + try data.write(to: url) + return .success(.init(url: url, height: Double(originalCGImage.height), width: Double(originalCGImage.width), mimeType: mimeType, blurhash: nil)) + } catch { + return .failure(.failedStrippingLocationData) + } + } + + /// Generates a thumbnail for an image + /// - Parameter url: the original image URL + /// - Returns: the URL for the resulting thumbnail and its sizing info as an `ImageProcessingResult` + private func generateThumbnailForImage(at url: URL) async -> Result { + switch await resizeImage(at: url, targetSize: Constants.maximumThumbnailSize) { + case .success(let thumbnail): + guard let data = thumbnail.jpegData(compressionQuality: Constants.thumbnailCompressionQuality) else { + return .failure(.failedGeneratingImageThumbnail(nil)) + } + + let blurhash = thumbnail.blurHash(numberOfComponents: (3, 3)) + + do { + let fileName = "thumbnail-\((url.lastPathComponent as NSString).deletingPathExtension).jpeg" + let thumbnailURL = url.deletingLastPathComponent().appendingPathComponent(fileName) + try data.write(to: thumbnailURL) + return .success(.init(url: thumbnailURL, height: thumbnail.size.height, width: thumbnail.size.width, mimeType: "image/jpeg", blurhash: blurhash)) + } catch { + return .failure(.failedGeneratingImageThumbnail(error)) + } + + case .failure(let error): + return .failure(.failedGeneratingImageThumbnail(error)) + } + } + + private func resizeImage(at url: URL, targetSize: CGSize) async -> Result { + let imageSource = CGImageSourceCreateWithURL(url as NSURL, nil) + guard let imageSource else { + return .failure(.failedResizingImage) + } + + return await resizeImage(withSource: imageSource, targetSize: targetSize) + } + + /// Aspect ratio resizes an image so it fits in the given size. This is useful for resizing images without loading them directly into memory + /// - Parameters: + /// - imageSource: the original image `CGImageSource` + /// - targetSize: maximum resulting size + /// - Returns: the resized image + private func resizeImage(withSource imageSource: CGImageSource, targetSize: CGSize) async -> Result { + let maximumSize = min(targetSize.height, targetSize.width) + + let options: [NSString: Any] = [ + // The maximum width and height in pixels of a thumbnail. + kCGImageSourceThumbnailMaxPixelSize: maximumSize, + kCGImageSourceCreateThumbnailFromImageAlways: true, + // Should include kCGImageSourceCreateThumbnailWithTransform: true in the options dictionary. Otherwise, the image result will appear rotated when an image is taken from camera in the portrait orientation. + kCGImageSourceCreateThumbnailWithTransform: true + ] + + guard let scaledImage = CGImageSourceCreateThumbnailAtIndex(imageSource, 0, options as CFDictionary) else { + return .failure(.failedResizingImage) + } + + return .success(UIImage(cgImage: scaledImage)) + } + + // MARK: Videos + + /// Generates a thumbnail for the video at the given URL + /// - Parameter url: the video URL + /// - Returns: the URL for the resulting thumbnail and its sizing info as an `ImageProcessingResult` + private func generateThumbnailForVideoAt(_ url: URL) async -> Result { + let assetImageGenerator = AVAssetImageGenerator(asset: AVAsset(url: url)) + assetImageGenerator.appliesPreferredTrackTransform = true + assetImageGenerator.maximumSize = Constants.maximumThumbnailSize + + do { + // Avoid the first frames as on a lot of videos they're black. + // If the specified seconds are longer than the actual video a frame close to the end of the video will be used, at AVFoundation's discretion + let location = CMTime(seconds: Constants.videoThumbnailTime, preferredTimescale: 1) + let cgImage = try await assetImageGenerator.image(at: location).image + + let thumbnail = UIImage(cgImage: cgImage) + + guard let data = thumbnail.jpegData(compressionQuality: Constants.thumbnailCompressionQuality) else { + return .failure(.failedGeneratingVideoThumbnail(nil)) + } + + let blurhash = thumbnail.blurHash(numberOfComponents: (3, 3)) + + let fileName = "\((url.lastPathComponent as NSString).deletingPathExtension).jpeg" + let thumbnailURL = url.deletingLastPathComponent().appendingPathComponent(fileName) + try data.write(to: thumbnailURL) + + return .success(.init(url: thumbnailURL, height: thumbnail.size.height, width: thumbnail.size.width, mimeType: "image/jpeg", blurhash: blurhash)) + + } catch { + return .failure(.failedGeneratingVideoThumbnail(error)) + } + } + + /// Converts the given video to an 1080p mp4 + /// - Parameters: + /// - url: the original video URL + /// - targetFileSize: the maximum resulting file size. 90% of this will be used + /// - Returns: the URL for the resulting video and its media info as a `VideoProcessingResult` + private func convertVideoToMP4(_ url: URL, targetFileSize: UInt = 0) async -> Result { + let asset = AVURLAsset(url: url) + + guard let exportSession = AVAssetExportSession(asset: asset, presetName: AVAssetExportPreset1920x1080) else { + return .failure(.failedConvertingVideo) + } + + // AVAssetExportSession will fail if the output URL already exists + let uuid = UUID().uuidString + let originalFilenameWithoutExtension = url.deletingPathExtension().lastPathComponent + let outputURL = url.deletingLastPathComponent().appendingPathComponent("\(uuid)-\(originalFilenameWithoutExtension).mp4") + + try? FileManager.default.removeItem(at: outputURL) + + exportSession.outputURL = outputURL + exportSession.outputFileType = AVFileType.mp4 + + guard exportSession.supportedFileTypes.contains(AVFileType.mp4) else { + return .failure(.failedConvertingVideo) + } + + if targetFileSize > 0 { + // Reduce the target file size by 10% as fileLengthLimit isn't a hard limit + exportSession.fileLengthLimit = Int64(Double(targetFileSize) * 0.9) + } + + await exportSession.export() + + switch exportSession.status { + case .completed: + do { + // Delete the original + try? FileManager.default.removeItem(at: url) + // Strip the UUID from the new version + let newOutputURL = url.deletingLastPathComponent().appendingPathComponent("\(originalFilenameWithoutExtension).mp4") + try FileManager.default.moveItem(at: outputURL, to: newOutputURL) + + let newAsset = AVURLAsset(url: newOutputURL) + guard let track = try? await newAsset.loadTracks(withMediaType: .video).first, + let durationInSeconds = try? await newAsset.load(.duration).seconds, + let naturalSize = try? await track.load(.naturalSize) else { + return .failure(.failedConvertingVideo) + } + + return .success(.init(url: newOutputURL, + height: naturalSize.height, + width: naturalSize.width, + duration: durationInSeconds * 1000, + mimeType: "video/mp4")) + } catch { + return .failure(.failedConvertingVideo) + } + default: + return .failure(.failedConvertingVideo) + } + } +} diff --git a/ElementX/Sources/Services/Media/ImageProviderProtocol.swift b/ElementX/Sources/Services/Media/Provider/ImageProviderProtocol.swift similarity index 100% rename from ElementX/Sources/Services/Media/ImageProviderProtocol.swift rename to ElementX/Sources/Services/Media/Provider/ImageProviderProtocol.swift diff --git a/ElementX/Sources/Services/Media/MediaFileHandleProxy.swift b/ElementX/Sources/Services/Media/Provider/MediaFileHandleProxy.swift similarity index 100% rename from ElementX/Sources/Services/Media/MediaFileHandleProxy.swift rename to ElementX/Sources/Services/Media/Provider/MediaFileHandleProxy.swift diff --git a/ElementX/Sources/Services/Media/MediaLoader.swift b/ElementX/Sources/Services/Media/Provider/MediaLoader.swift similarity index 100% rename from ElementX/Sources/Services/Media/MediaLoader.swift rename to ElementX/Sources/Services/Media/Provider/MediaLoader.swift diff --git a/ElementX/Sources/Services/Media/MediaLoaderProtocol.swift b/ElementX/Sources/Services/Media/Provider/MediaLoaderProtocol.swift similarity index 100% rename from ElementX/Sources/Services/Media/MediaLoaderProtocol.swift rename to ElementX/Sources/Services/Media/Provider/MediaLoaderProtocol.swift diff --git a/ElementX/Sources/Services/Media/MediaProvider.swift b/ElementX/Sources/Services/Media/Provider/MediaProvider.swift similarity index 100% rename from ElementX/Sources/Services/Media/MediaProvider.swift rename to ElementX/Sources/Services/Media/Provider/MediaProvider.swift diff --git a/ElementX/Sources/Services/Media/MediaProviderProtocol.swift b/ElementX/Sources/Services/Media/Provider/MediaProviderProtocol.swift similarity index 100% rename from ElementX/Sources/Services/Media/MediaProviderProtocol.swift rename to ElementX/Sources/Services/Media/Provider/MediaProviderProtocol.swift diff --git a/ElementX/Sources/Services/Media/MediaSourceProxy.swift b/ElementX/Sources/Services/Media/Provider/MediaSourceProxy.swift similarity index 100% rename from ElementX/Sources/Services/Media/MediaSourceProxy.swift rename to ElementX/Sources/Services/Media/Provider/MediaSourceProxy.swift diff --git a/ElementX/Sources/Services/Media/MockMediaProvider.swift b/ElementX/Sources/Services/Media/Provider/MockMediaProvider.swift similarity index 100% rename from ElementX/Sources/Services/Media/MockMediaProvider.swift rename to ElementX/Sources/Services/Media/Provider/MockMediaProvider.swift diff --git a/ElementX/Sources/Services/Room/RoomProxy.swift b/ElementX/Sources/Services/Room/RoomProxy.swift index 23340c179..422bfe760 100644 --- a/ElementX/Sources/Services/Room/RoomProxy.swift +++ b/ElementX/Sources/Services/Room/RoomProxy.swift @@ -226,8 +226,21 @@ class RoomProxy: RoomProxyProtocol { } } - func sendImage(url: URL) async -> Result { + func sendImage(url: URL, thumbnailURL: URL, imageInfo: ImageInfo) async -> Result { .failure(.failedSendingMedia) +// sendMessageBackgroundTask = backgroundTaskService.startBackgroundTask(withName: backgroundTaskName, isReusable: true) +// defer { +// sendMessageBackgroundTask?.stop() +// } +// +// return await Task.dispatch(on: userInitiatedDispatchQueue) { +// do { +// try self.room.sendImage(url: url.path(), thumbnailUrl: thumbnailURL.path(), imageInfo: imageInfo) +// return .success(()) +// } catch { +// return .failure(.failedEditingMessage) +// } +// } } func editMessage(_ newMessage: String, original eventID: String) async -> Result { diff --git a/ElementX/Sources/Services/Room/RoomProxyProtocol.swift b/ElementX/Sources/Services/Room/RoomProxyProtocol.swift index 7e78e076e..0fb7f9811 100644 --- a/ElementX/Sources/Services/Room/RoomProxyProtocol.swift +++ b/ElementX/Sources/Services/Room/RoomProxyProtocol.swift @@ -75,7 +75,7 @@ protocol RoomProxyProtocol { func sendReaction(_ reaction: String, to eventID: String) async -> Result - func sendImage(url: URL) async -> Result + func sendImage(url: URL, thumbnailURL: URL, imageInfo: ImageInfo) async -> Result func editMessage(_ newMessage: String, original eventID: String) async -> Result diff --git a/NSE/SupportingFiles/target.yml b/NSE/SupportingFiles/target.yml index f5195730a..f423f9bb6 100644 --- a/NSE/SupportingFiles/target.yml +++ b/NSE/SupportingFiles/target.yml @@ -76,7 +76,7 @@ targets: - path: ../../ElementX/Sources/Services/Notification/Proxy/NotificationServiceProxy.swift - path: ../../ElementX/Sources/Services/Notification/Proxy/NotificationItemProxy.swift - path: ../../ElementX/Sources/Services/Notification/NotificationConstants.swift - - path: ../../ElementX/Sources/Services/Media + - path: ../../ElementX/Sources/Services/Media/Provider - path: ../../ElementX/Sources/Services/Background/BackgroundTaskServiceProtocol.swift - path: ../../ElementX/Sources/Services/Background/BackgroundTaskProtocol.swift - path: ../../ElementX/Sources/Other/Logging