From d7a924ba1b79b8465de3f39316dbfe539f190938 Mon Sep 17 00:00:00 2001 From: fanyu Date: Thu, 1 Jun 2023 13:15:42 +0800 Subject: [PATCH 01/23] Asynchronously receive and parse data --- Mixin.xcodeproj/project.pbxproj | 8 + Mixin/Resources/en.lproj/Localizable.strings | 1 + Mixin/Resources/es.lproj/Localizable.strings | 1 + Mixin/Resources/ja.lproj/Localizable.strings | 1 + Mixin/Resources/ru.lproj/Localizable.strings | 1 + .../zh-Hans.lproj/Localizable.strings | 1 + .../zh-Hant.lproj/Localizable.strings | 1 + .../DeviceTransfer/DeviceTransferClient.swift | 106 +++---- .../DeviceTransferClosedReason.swift | 3 +- .../DeviceTransfer/DeviceTransferData.swift | 36 +++ .../DeviceTransferDataWriter.swift | 270 ++++++++++++++++++ .../DeviceTransfer/DeviceTransferError.swift | 1 + .../DeviceTransferFileStream.swift | 77 ++--- .../DeviceTransfer/DeviceTransferServer.swift | 8 +- ...DeviceTransferProgressViewController.swift | 25 +- .../RestoreFromDesktopViewController.swift | 2 +- .../TransferToDesktopViewController.swift | 2 +- .../TransferToPhoneQRCodeViewController.swift | 2 +- 18 files changed, 407 insertions(+), 139 deletions(-) create mode 100644 Mixin/Service/DeviceTransfer/DeviceTransferData.swift create mode 100644 Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift diff --git a/Mixin.xcodeproj/project.pbxproj b/Mixin.xcodeproj/project.pbxproj index 14b76c8486..5439afbf31 100644 --- a/Mixin.xcodeproj/project.pbxproj +++ b/Mixin.xcodeproj/project.pbxproj @@ -663,6 +663,8 @@ 7CE3A25C2771A8AB006BE765 /* DeleteAccountVerifyCodeViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CE3A25B2771A8AB006BE765 /* DeleteAccountVerifyCodeViewController.swift */; }; 7CE5E7A8269BDA29000B7904 /* HomeAppsPinTipsViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CE5E7A6269BDA29000B7904 /* HomeAppsPinTipsViewController.swift */; }; 7CE5E7A9269BDA29000B7904 /* HomeAppsPinTipsView.xib in Resources */ = {isa = PBXBuildFile; fileRef = 7CE5E7A7269BDA29000B7904 /* HomeAppsPinTipsView.xib */; }; + 7CE7ADD72A24983800A6259F /* DeviceTransferDataWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CE7ADD62A24983800A6259F /* DeviceTransferDataWriter.swift */; }; + 7CE7ADD92A24A83A00A6259F /* DeviceTransferData.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CE7ADD82A24A83A00A6259F /* DeviceTransferData.swift */; }; 7CEB735429DB24F3006FB5B2 /* RestoreChatViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CEB735229DB24F3006FB5B2 /* RestoreChatViewController.swift */; }; 7CEB735529DB24F3006FB5B2 /* RestoreChatView.xib in Resources */ = {isa = PBXBuildFile; fileRef = 7CEB735329DB24F3006FB5B2 /* RestoreChatView.xib */; }; 7CEB735829DB272F006FB5B2 /* RestoreChatTableViewCell.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CEB735629DB272F006FB5B2 /* RestoreChatTableViewCell.swift */; }; @@ -1736,6 +1738,8 @@ 7CE3A25B2771A8AB006BE765 /* DeleteAccountVerifyCodeViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeleteAccountVerifyCodeViewController.swift; sourceTree = ""; }; 7CE5E7A6269BDA29000B7904 /* HomeAppsPinTipsViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HomeAppsPinTipsViewController.swift; sourceTree = ""; }; 7CE5E7A7269BDA29000B7904 /* HomeAppsPinTipsView.xib */ = {isa = PBXFileReference; lastKnownFileType = file.xib; path = HomeAppsPinTipsView.xib; sourceTree = ""; }; + 7CE7ADD62A24983800A6259F /* DeviceTransferDataWriter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeviceTransferDataWriter.swift; sourceTree = ""; }; + 7CE7ADD82A24A83A00A6259F /* DeviceTransferData.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeviceTransferData.swift; sourceTree = ""; }; 7CEB735229DB24F3006FB5B2 /* RestoreChatViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RestoreChatViewController.swift; sourceTree = ""; }; 7CEB735329DB24F3006FB5B2 /* RestoreChatView.xib */ = {isa = PBXFileReference; lastKnownFileType = file.xib; path = RestoreChatView.xib; sourceTree = ""; }; 7CEB735629DB272F006FB5B2 /* RestoreChatTableViewCell.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RestoreChatTableViewCell.swift; sourceTree = ""; }; @@ -2766,6 +2770,8 @@ 94E1D44D29E9393C00511267 /* DeviceTransferServer.swift */, 940B304A2A160BAB00B45D26 /* DeviceTransferServerDataSource.swift */, 94396F2929EBE52400A57833 /* DeviceTransferClient.swift */, + 7CE7ADD62A24983800A6259F /* DeviceTransferDataWriter.swift */, + 7CE7ADD82A24A83A00A6259F /* DeviceTransferData.swift */, 94396F2F29ED005200A57833 /* DeviceTransferFileStream.swift */, 94396F2529EB11E300A57833 /* DeviceTransferProtocol.swift */, 94396F2729EB475500A57833 /* NWParameters+DeviceTransfer.swift */, @@ -4771,6 +4777,7 @@ 7C7635B826A13461006101DB /* HomeAppsConstants.swift in Sources */, DFB190002330F6A40021CAF3 /* BiographyViewController.swift in Sources */, 7B7F7E391FD43F2500A1C91F /* DetailInfoMessageCell.swift in Sources */, + 7CE7ADD72A24983800A6259F /* DeviceTransferDataWriter.swift in Sources */, 7B8BFC8A1FDD77F9004E19DB /* UnknownMessageViewModel.swift in Sources */, 7CEB736C29DBD1C0006FB5B2 /* DeviceTransferMessage.swift in Sources */, 7B35AF7C228AA6CD00E8101D /* MessagesWithinConversationSearchResult.swift in Sources */, @@ -4967,6 +4974,7 @@ DFB19002233219650021CAF3 /* LogViewController.swift in Sources */, 7B054052243CCCD500C1FCB6 /* HomeTitleButton.swift in Sources */, 7BD127FC22D8892200BB5316 /* GalleryViewController.swift in Sources */, + 7CE7ADD92A24A83A00A6259F /* DeviceTransferData.swift in Sources */, DFD1FBC72302CB7A00C570D4 /* DatabaseUpgradeViewController.swift in Sources */, 944C656125D9BA0A008BDDD3 /* OggOpusWriter.swift in Sources */, 7BB8417C2230C63000FB88B9 /* ConversationAccessible.swift in Sources */, diff --git a/Mixin/Resources/en.lproj/Localizable.strings b/Mixin/Resources/en.lproj/Localizable.strings index 2b435608ab..ca7ed15394 100644 --- a/Mixin/Resources/en.lproj/Localizable.strings +++ b/Mixin/Resources/en.lproj/Localizable.strings @@ -429,6 +429,7 @@ "i_am_good" = "I’m good."; "image" = "image"; "immediately" = "Immediately"; +"importing_chat_progress" = "Importing chat (%@%%)"; "in_connecting" = "Connecting..."; "include_files" = "Include Files"; "include_videos" = "Include Videos"; diff --git a/Mixin/Resources/es.lproj/Localizable.strings b/Mixin/Resources/es.lproj/Localizable.strings index b69dddb4e6..7ecf78b531 100644 --- a/Mixin/Resources/es.lproj/Localizable.strings +++ b/Mixin/Resources/es.lproj/Localizable.strings @@ -429,6 +429,7 @@ "i_am_good" = "Estoy bien."; "image" = "imagen"; "immediately" = "Inmediatamente"; +"importing_chat_progress" = "Importing chat (%@%%)"; "in_connecting" = "Conectando..."; "include_files" = "Incluir archivos"; "include_videos" = "Incluir vídeos"; diff --git a/Mixin/Resources/ja.lproj/Localizable.strings b/Mixin/Resources/ja.lproj/Localizable.strings index cd48d98549..af729c8c16 100644 --- a/Mixin/Resources/ja.lproj/Localizable.strings +++ b/Mixin/Resources/ja.lproj/Localizable.strings @@ -429,6 +429,7 @@ "i_am_good" = "いい気分"; "image" = "画像"; "immediately" = "すぐに"; +"importing_chat_progress" = "Importing chat (%@%%)"; "in_connecting" = "接続中..."; "include_files" = "ファイルが含まれています"; "include_videos" = "動画が含まれています"; diff --git a/Mixin/Resources/ru.lproj/Localizable.strings b/Mixin/Resources/ru.lproj/Localizable.strings index 8e718abc42..27d08c1fdf 100644 --- a/Mixin/Resources/ru.lproj/Localizable.strings +++ b/Mixin/Resources/ru.lproj/Localizable.strings @@ -429,6 +429,7 @@ "i_am_good" = "Я в порядке."; "image" = "изображение"; "immediately" = "Немедленно"; +"importing_chat_progress" = "Importing chat (%@%%)"; "in_connecting" = "Подключение..."; "include_files" = "Включить файлы"; "include_videos" = "Включить видео"; diff --git a/Mixin/Resources/zh-Hans.lproj/Localizable.strings b/Mixin/Resources/zh-Hans.lproj/Localizable.strings index bb95de0fd3..9ccd94e816 100644 --- a/Mixin/Resources/zh-Hans.lproj/Localizable.strings +++ b/Mixin/Resources/zh-Hans.lproj/Localizable.strings @@ -429,6 +429,7 @@ "i_am_good" = "我很好。"; "image" = "图片"; "immediately" = "立刻"; +"importing_chat_progress" = "导入聊天记录（%@%%）"; "in_connecting" = "正在连接..."; "include_files" = "包括文件"; "include_videos" = "包括视频"; diff --git a/Mixin/Resources/zh-Hant.lproj/Localizable.strings b/Mixin/Resources/zh-Hant.lproj/Localizable.strings index f7f9d2463b..277ff785c3 100644 --- a/Mixin/Resources/zh-Hant.lproj/Localizable.strings +++ b/Mixin/Resources/zh-Hant.lproj/Localizable.strings @@ -429,6 +429,7 @@ "i_am_good" = "我很好。"; "image" = "圖片"; "immediately" = "立刻"; +"importing_chat_progress" = "匯入聊天記錄（%@%%）"; "in_connecting" = "正在連線..."; "include_files" = "包括檔案"; "include_videos" = "包括影片"; diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index 83224385bf..446401041f 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -8,6 +8,7 @@ final class DeviceTransferClient { case idle case transfer(progress: Double, speed: String) case closed(DeviceTransferClosedReason) + case importing(progress: Double) } @Published private(set) var state: State = .idle @@ -20,6 +21,7 @@ final class DeviceTransferClient { private let connection: NWConnection private let queue = Queue(label: "one.mixin.messenger.DeviceTransferClient") private let speedInspector = NetworkSpeedInspector() + private let dataWriter: DeviceTransferDataWriter private weak var statisticsTimer: Timer? @@ -45,6 +47,7 @@ final class DeviceTransferClient { let endpoint = NWEndpoint.hostPort(host: host, port: port) return NWConnection(to: endpoint, using: .deviceTransfer) }() + self.dataWriter = DeviceTransferDataWriter(remotePlatform: remotePlatform) Logger.general.info(category: "DeviceTransferClient", message: "\(opaquePointer) init") } @@ -93,7 +96,7 @@ final class DeviceTransferClient { connection.start(queue: queue.dispatchQueue) } - private func stop(reason: DeviceTransferClosedReason) { + func stop(reason: DeviceTransferClosedReason) { assert(queue.isCurrent) Logger.general.info(category: "DeviceTransferClient", message: "Stop: \(reason) Processed: \(processedCount) Total: \(totalCount)") DispatchQueue.main.sync { @@ -101,9 +104,15 @@ final class DeviceTransferClient { } connection.cancel() switch reason { - case .finished: - state = .closed(.finished) + case .transferFinished: + state = .closed(.transferFinished) + case .importFinished: + break case .exception(let error): + DispatchQueue.main.async { + self.dataWriter.delegate = nil + self.dataWriter.canProcessData = false + } state = .closed(.exception(error)) } } @@ -209,12 +218,12 @@ extension DeviceTransferClient { Logger.general.info(category: "DeviceTransferClient", message: "Total count: \(count)") self.state = .transfer(progress: 0, speed: "") DispatchQueue.main.async { + self.dataWriter.canProcessData = true self.totalCount = count self.startUpdatingProgressAndSpeed() } case .finish: Logger.general.info(category: "DeviceTransferClient", message: "Received finish command") - ConversationDAO.shared.updateLastMessageIdAndCreatedAt() do { let command = DeviceTransferCommand(action: .finish) let content = try DeviceTransferProtocol.output(command: command, key: key) @@ -223,7 +232,11 @@ extension DeviceTransferClient { } catch { Logger.general.error(category: "DeviceTransferClient", message: "Failed to finish command: \(error)") } - self.stop(reason: .finished) + DispatchQueue.main.async { + self.dataWriter.delegate = self + } + dataWriter.transferFinished() + self.stop(reason: .transferFinished) default: break } @@ -244,72 +257,15 @@ extension DeviceTransferClient { stop(reason: .exception(.mismatchedHMAC(local: localHMAC, remote: remoteHMAC))) return } - - let decryptedData: Data do { - decryptedData = try AESCryptor.decrypt(encryptedData, with: key.aes) + let decryptedData = try AESCryptor.decrypt(encryptedData, with: key.aes) + if !dataWriter.write(data: decryptedData) { + stop(reason: .exception(.unableSaveData)) + } } catch { Logger.general.error(category: "DeviceTransferClient", message: "Unable to decrypt: \(error)") return } - - do { - struct TypeWrapper: Decodable { - let type: DeviceTransferRecordType - } - - let decoder = JSONDecoder.default - let wrapper = try decoder.decode(TypeWrapper.self, from: decryptedData) - switch wrapper.type { - case .conversation: - let conversation = try decoder.decode(DeviceTransferTypedRecord.self, from: decryptedData).data - ConversationDAO.shared.save(conversation: conversation.toConversation(from: remotePlatform)) - case .participant: - let participant = try decoder.decode(DeviceTransferTypedRecord.self, from: decryptedData).data - ParticipantDAO.shared.save(participant: participant.toParticipant()) - case .user: - let user = try decoder.decode(DeviceTransferTypedRecord.self, from: decryptedData).data - UserDAO.shared.save(user: user.toUser()) - case .app: - let app = try decoder.decode(DeviceTransferTypedRecord.self, from: decryptedData).data - AppDAO.shared.save(app: app.toApp()) - case .asset: - let asset = try decoder.decode(DeviceTransferTypedRecord.self, from: decryptedData).data - AssetDAO.shared.save(asset: asset.toAsset()) - case .snapshot: - let snapshot = try decoder.decode(DeviceTransferTypedRecord.self, from: decryptedData).data - SnapshotDAO.shared.save(snapshot: snapshot.toSnapshot()) - case .sticker: - let sticker = try decoder.decode(DeviceTransferTypedRecord.self, from: decryptedData).data - StickerDAO.shared.save(sticker: sticker.toSticker()) - case .pinMessage: - let pinMessage = try decoder.decode(DeviceTransferTypedRecord.self, from: decryptedData).data - PinMessageDAO.shared.save(pinMessage: pinMessage.toPinMessage()) - case .transcriptMessage: - let transcriptMessage = try decoder.decode(DeviceTransferTypedRecord.self, from: decryptedData).data - TranscriptMessageDAO.shared.save(transcriptMessage: transcriptMessage.toTranscriptMessage()) - case .message: - let message = try decoder.decode(DeviceTransferTypedRecord.self, from: decryptedData).data - if MessageCategory.isLegal(category: message.category) { - MessageDAO.shared.save(message: message.toMessage()) - } else { - Logger.general.warn(category: "DeviceTransferClient", message: "Message is illegal: \(message)") - } - case .messageMention: - let messageMention = try decoder.decode(DeviceTransferTypedRecord.self, from: decryptedData).data - if let mention = messageMention.toMessageMention() { - MessageMentionDAO.shared.save(messageMention: mention) - } else { - Logger.general.warn(category: "DeviceTransferClient", message: "Message Mention does not exist: \(messageMention)") - } - case .expiredMessage: - let expiredMessage = try decoder.decode(DeviceTransferTypedRecord.self, from: decryptedData).data - ExpiredMessageDAO.shared.save(expiredMessage: expiredMessage.toExpiredMessage()) - } - } catch { - let content = String(data: decryptedData, encoding: .utf8) ?? "Data(\(decryptedData.count))" - Logger.general.error(category: "DeviceTransferClient", message: "Error: \(error) Content: \(content)") - } } private func receiveFile(context: DeviceTransferProtocol.FileContext, content: Data) { @@ -325,11 +281,11 @@ extension DeviceTransferClient { } else { assertionFailure("Should be closed by the end of previous call") currentStream.close() - stream = DeviceTransferFileStream(context: context, key: key) + stream = DeviceTransferFileStream(context: context, key: key, client: self) isReceivingNewFile = true } } else { - stream = DeviceTransferFileStream(context: context, key: key) + stream = DeviceTransferFileStream(context: context, key: key, client: self) isReceivingNewFile = true } if isReceivingNewFile { @@ -356,3 +312,17 @@ extension DeviceTransferClient { } } + +extension DeviceTransferClient: DeviceTransferDataWriterDelegate { + + func deviceTransferDataWriter(_ writer: DeviceTransferDataWriter, update progress: Double) { + if progress >= 1 { + Logger.general.info(category: "DeviceTransferClient", message: "Import finished") + ConversationDAO.shared.updateLastMessageIdAndCreatedAt() + state = .closed(.importFinished) + } else { + state = .importing(progress: progress) + } + } + +} diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClosedReason.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClosedReason.swift index 21540c9f4a..291fcf0925 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClosedReason.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClosedReason.swift @@ -1,6 +1,7 @@ import Foundation enum DeviceTransferClosedReason { - case finished + case transferFinished + case importFinished case exception(DeviceTransferError) } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferData.swift b/Mixin/Service/DeviceTransfer/DeviceTransferData.swift new file mode 100644 index 0000000000..9f5f144de4 --- /dev/null +++ b/Mixin/Service/DeviceTransfer/DeviceTransferData.swift @@ -0,0 +1,36 @@ +import Foundation +import MixinServices + +enum DeviceTransferData: String { + + case record + case file + + static let payloadLength: UInt64 = 4 + static let maxSizePerFile = 10 * Int(bytesPerMegaByte) + + static func url() -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("DeviceTransfer", isDirectory: true) + _ = try? FileManager.default.createDirectoryIfNotExists(at: url) + return url + } + + func url(index: Int?) -> URL { + if let index { + return url(name: "\(index)") + } else { + return url(name: nil) + } + } + + func url(name: String?) -> URL { + let url = Self.url().appendingPathComponent(self.rawValue, isDirectory: true) + _ = try? FileManager.default.createDirectoryIfNotExists(at: url) + if let name { + return url.appendingPathComponent("\(name).bin") + } else { + return url + } + } + +} diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift b/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift new file mode 100644 index 0000000000..9453381a77 --- /dev/null +++ b/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift @@ -0,0 +1,270 @@ +import Foundation +import MixinServices + +protocol DeviceTransferDataWriterDelegate: AnyObject { + + func deviceTransferDataWriter(_ writer: DeviceTransferDataWriter, update progress: Double) + +} + +final class DeviceTransferDataWriter { + + weak var delegate: DeviceTransferDataWriterDelegate? + + var canProcessData: Bool = true { + didSet { + fileIndex = 0 + parsedRecordCount = 0 + totalRecordCount = 0 + fileHandle = nil + cleanUpIfNeeded() + } + } + + private let remotePlatform: DeviceTransferPlatform + private let queue = Queue(label: "one.mixin.messenger.DeviceTransferClient.parse") + + private var fileHandle: FileHandle? + private var fileIndex = 0 + private var totalRecordCount = 0 + private var parsedRecordCount = 0 + private var pendingParsedRecordPath = [URL]() + + init(remotePlatform: DeviceTransferPlatform) { + self.remotePlatform = remotePlatform + } + + func write(data: Data) -> Bool { + if let fileHandle { + let fileSize = fileHandle.seekToEndOfFile() + let maxSizeExceeded = fileSize + UInt64(data.count) > DeviceTransferData.maxSizePerFile + if maxSizeExceeded { + fileHandle.closeFile() + let filePath = DeviceTransferData.record.url(index: fileIndex) + Logger.general.info(category: "DeviceTransferDataWriter", message: "Close file: \(filePath)") + queue.async { + self.pendingParsedRecordPath.append(filePath) + self.readAndParseRecordData() + } + fileIndex += 1 + openNextFile() + } + } else { + openNextFile() + } + guard let fileHandle else { + Logger.general.warn(category: "DeviceTransferDataWriter", message: "FileHandle is nil") + return false + } + let lenghtData = UInt32(data.count).data(endianness: .big) + fileHandle.write(lenghtData) + fileHandle.write(data) + DispatchQueue.main.sync { + self.totalRecordCount += 1 + } + return true + } + + func transferFinished() { + fileHandle?.closeFile() + let filePath = DeviceTransferData.record.url(index: fileIndex) + Logger.general.info(category: "DeviceTransferDataWriter", message: "Close file: \(filePath)") + queue.async { + self.pendingParsedRecordPath.append(filePath) + self.readAndParseRecordData() + } + } + +} + +extension DeviceTransferDataWriter { + + private func cleanUpIfNeeded() { + let path = DeviceTransferData.url().path + if FileManager.default.fileExists(atPath: path) { + try? FileManager.default.removeItem(atPath: path) + Logger.general.info(category: "DeviceTransferDataWriter", message: "Clean folder: \(path)") + } + } + + private func openNextFile() { + let filePath = DeviceTransferData.record.url(index: fileIndex).path + if FileManager.default.fileExists(atPath: filePath) { + try? FileManager.default.removeItem(atPath: filePath) + } + FileManager.default.createFile(atPath: filePath, contents: nil, attributes: nil) + fileHandle = FileHandle(forUpdatingAtPath: filePath) + Logger.general.info(category: "DeviceTransferDataWriter", message: "Open file: \(filePath)") + } + + private func readAndParseRecordData() { + assert(queue.isCurrent) + guard !pendingParsedRecordPath.isEmpty else { + return + } + let filePath = pendingParsedRecordPath.removeFirst() + let fileHandle: FileHandle + do { + fileHandle = try FileHandle(forReadingFrom: filePath) + } catch { + Logger.general.error(category: "DeviceTransferDataWriter", message: "Reading from \(filePath) failed: \(error)") + return + } + Logger.general.info(category: "DeviceTransferDataWriter", message: "Parse file: \(filePath.path)") + let fileSize = fileHandle.seekToEndOfFile() + var offset: UInt64 = 0 + while offset < fileSize, canProcessData { + autoreleasepool { + fileHandle.seek(toFileOffset: offset) + let lengthData = fileHandle.readData(ofLength: Int(DeviceTransferData.payloadLength)) + let length = Int(Int32(data: lengthData, endianess: .big)) + offset += DeviceTransferData.payloadLength + fileHandle.seek(toFileOffset: offset) + let data = fileHandle.readData(ofLength: Int(length)) + offset += UInt64(length) + parseRecord(data: data) + DispatchQueue.main.async { + self.parsedRecordCount += 1 + guard let delegate = self.delegate else { + return + } + if self.parsedRecordCount >= self.totalRecordCount { + guard self.canProcessData else { + return + } + self.queue.async { + self.processFiles() + DispatchQueue.main.async { + delegate.deviceTransferDataWriter(self, update: 1) + } + } + } else { + let progress = Double(self.parsedRecordCount) / Double(self.totalRecordCount + 1) + delegate.deviceTransferDataWriter(self, update: progress) + } + } + } + } + fileHandle.closeFile() + queue.async { + self.readAndParseRecordData() + } + } + +} + +extension DeviceTransferDataWriter { + + private func parseRecord(data: Data) { + assert(queue.isCurrent) + do { + struct TypeWrapper: Decodable { + let type: DeviceTransferRecordType + } + + let decoder = JSONDecoder.default + let wrapper = try decoder.decode(TypeWrapper.self, from: data) + switch wrapper.type { + case .conversation: + let conversation = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data + ConversationDAO.shared.save(conversation: conversation.toConversation(from: remotePlatform)) + case .participant: + let participant = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data + ParticipantDAO.shared.save(participant: participant.toParticipant()) + case .user: + let user = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data + UserDAO.shared.save(user: user.toUser()) + case .app: + let app = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data + AppDAO.shared.save(app: app.toApp()) + case .asset: + let asset = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data + AssetDAO.shared.save(asset: asset.toAsset()) + case .snapshot: + let snapshot = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data + SnapshotDAO.shared.save(snapshot: snapshot.toSnapshot()) + case .sticker: + let sticker = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data + StickerDAO.shared.save(sticker: sticker.toSticker()) + case .pinMessage: + let pinMessage = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data + PinMessageDAO.shared.save(pinMessage: pinMessage.toPinMessage()) + case .transcriptMessage: + let transcriptMessage = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data + TranscriptMessageDAO.shared.save(transcriptMessage: transcriptMessage.toTranscriptMessage()) + case .message: + let message = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data + if MessageCategory.isLegal(category: message.category) { + MessageDAO.shared.save(message: message.toMessage()) + } else { + Logger.general.warn(category: "DeviceTransferDataWriter", message: "Message is illegal: \(message)") + } + case .messageMention: + let messageMention = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data + if let mention = messageMention.toMessageMention() { + MessageMentionDAO.shared.save(messageMention: mention) + } else { + Logger.general.warn(category: "DeviceTransferDataWriter", message: "Message Mention does not exist: \(messageMention)") + } + case .expiredMessage: + let expiredMessage = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data + ExpiredMessageDAO.shared.save(expiredMessage: expiredMessage.toExpiredMessage()) + } + } catch { + let content = String(data: data, encoding: .utf8) ?? "Data(\(data.count))" + Logger.general.error(category: "DeviceTransferDataWriter", message: "Error: \(error) Content: \(content)") + } + } + + private func processFiles() { + assert(queue.isCurrent) + guard + let fileEnumerator = FileManager.default.enumerator(at: DeviceTransferData.file.url(name: nil), + includingPropertiesForKeys: [.isRegularFileKey], + options: [.skipsHiddenFiles, .skipsPackageDescendants]) + else { + Logger.general.error(category: "DeviceTransferDataWriter", message: "Can't create file enumerator") + return + } + Logger.general.info(category: "DeviceTransferDataWriter", message: "Start processing files") + let fileManager = FileManager.default + for case let fileURL as URL in fileEnumerator { + let components = fileURL.lastPathComponent.components(separatedBy: ".") + guard components.count == 2, let id = components.first else { + Logger.general.info(category: "DeviceTransferDataWriter", message: "Invalid file url: \(fileURL.path)") + continue + } + var destinationURLs: [URL] + if let message = MessageDAO.shared.getMessage(messageId: id), let mediaURL = message.mediaUrl { + guard let category = AttachmentContainer.Category(messageCategory: message.category) else { + Logger.general.error(category: "DeviceTransferDataWriter", message: "Invalid category: \(message.category)") + continue + } + let url = AttachmentContainer.url(for: category, filename: mediaURL) + destinationURLs = [url] + if let transcriptMessage = TranscriptMessageDAO.shared.transcriptMessage(messageId: id), let mediaURL = transcriptMessage.mediaUrl { + let url = AttachmentContainer.url(transcriptId: transcriptMessage.transcriptId, filename: mediaURL) + destinationURLs.append(url) + } + } else if let transcriptMessage = TranscriptMessageDAO.shared.transcriptMessage(messageId: id), let mediaURL = transcriptMessage.mediaUrl { + let url = AttachmentContainer.url(transcriptId: transcriptMessage.transcriptId, filename: mediaURL) + destinationURLs = [url] + } else { + Logger.general.warn(category: "DeviceTransferDataWriter", message: "No message found for: \(id)") + continue + } + for destinationURL in destinationURLs { + do { + if fileManager.fileExists(atPath: destinationURL.path) { + try fileManager.removeItem(at: destinationURL) + } + try fileManager.copyItem(at: fileURL, to: destinationURL) + } catch { + Logger.general.error(category: "DeviceTransferDataWriter", message: "\(id) move failed: \(error)") + } + } + } + cleanUpIfNeeded() + } + +} diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferError.swift b/Mixin/Service/DeviceTransfer/DeviceTransferError.swift index 8f127f85f0..d66e4f6ad9 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferError.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferError.swift @@ -7,4 +7,5 @@ enum DeviceTransferError: Error { case mismatchedHMAC(local: Data, remote: Data) case failed(Error) case receiveFile(Error) + case unableSaveData } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferFileStream.swift b/Mixin/Service/DeviceTransfer/DeviceTransferFileStream.swift index dc4c742a73..3c481a13cf 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferFileStream.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferFileStream.swift @@ -3,17 +3,20 @@ import MixinServices class DeviceTransferFileStream: InstanceInitializable { + weak var client: DeviceTransferClient? + let id: UUID - fileprivate init(id: UUID) { + fileprivate init(id: UUID, client: DeviceTransferClient) { self.id = id + self.client = client } - convenience init(context: DeviceTransferProtocol.FileContext, key: DeviceTransferKey) { - if let impl = DeviceTransferFileStreamImpl(context, key: key) { + convenience init(context: DeviceTransferProtocol.FileContext, key: DeviceTransferKey, client: DeviceTransferClient) { + if let impl = DeviceTransferFileStreamImpl(context, key: key, client: client) { self.init(instance: impl as! Self) } else { - self.init(id: context.fileHeader.id) + self.init(id: context.fileHeader.id, client: client) } } @@ -29,9 +32,7 @@ class DeviceTransferFileStream: InstanceInitializable { fileprivate final class DeviceTransferFileStreamImpl: DeviceTransferFileStream { - private let tempURL: URL private let handle: FileHandle - private let destinationURLs: [URL] private let fileManager: FileManager = .default private var decryptor: AESCryptor @@ -39,7 +40,7 @@ fileprivate final class DeviceTransferFileStreamImpl: DeviceTransferFileStream { private var localHMAC: HMACSHA256 private var remoteHMAC = Data(capacity: DeviceTransferProtocol.hmacDataCount) - init?(_ context: DeviceTransferProtocol.FileContext, key: DeviceTransferKey) { + init?(_ context: DeviceTransferProtocol.FileContext, key: DeviceTransferKey, client: DeviceTransferClient) { let decryptor: AESCryptor do { decryptor = try AESCryptor(operation: .decrypt, iv: context.fileHeader.iv, key: key.aes) @@ -51,40 +52,18 @@ fileprivate final class DeviceTransferFileStreamImpl: DeviceTransferFileStream { let id = context.fileHeader.id.uuidString.lowercased() let idData = context.fileHeader.id.data - var destinationURLs: [URL] - if let message = MessageDAO.shared.getMessage(messageId: id), let mediaURL = message.mediaUrl { - guard let category = AttachmentContainer.Category(messageCategory: message.category) else { - Logger.general.error(category: "DeviceTransferFileStream", message: "Invalid category: \(message.category)") - return nil - } - let url = AttachmentContainer.url(for: category, filename: mediaURL) - destinationURLs = [url] - if let transcriptMessage = TranscriptMessageDAO.shared.transcriptMessage(messageId: id), let mediaURL = transcriptMessage.mediaUrl { - let url = AttachmentContainer.url(transcriptId: transcriptMessage.transcriptId, filename: mediaURL) - destinationURLs.append(url) - } - } else if let transcriptMessage = TranscriptMessageDAO.shared.transcriptMessage(messageId: id), let mediaURL = transcriptMessage.mediaUrl { - let url = AttachmentContainer.url(transcriptId: transcriptMessage.transcriptId, filename: mediaURL) - destinationURLs = [url] - } else { - Logger.general.warn(category: "DeviceTransferFileStream", message: "No message found for: \(id)") - return nil - } - do { - let tempURL = fileManager.temporaryDirectory.appendingPathComponent("devicetransfer.tmp") - if fileManager.fileExists(atPath: tempURL.path) { - try fileManager.removeItem(at: tempURL) + let fileURL = DeviceTransferData.file.url(name: id) + if fileManager.fileExists(atPath: fileURL.path) { + try fileManager.removeItem(at: fileURL) } - fileManager.createFile(atPath: tempURL.path, contents: nil) + fileManager.createFile(atPath: fileURL.path, contents: nil) - self.tempURL = tempURL - self.handle = try FileHandle(forWritingTo: tempURL) - self.destinationURLs = destinationURLs + self.handle = try FileHandle(forWritingTo: fileURL) self.decryptor = decryptor self.remainingDataCount = Int(context.header.length) - idData.count - DeviceTransferProtocol.ivDataCount self.localHMAC = HMACSHA256(key: key.hmac) - super.init(id: context.fileHeader.id) + super.init(id: context.fileHeader.id, client: client) localHMAC.update(data: idData) localHMAC.update(data: context.fileHeader.iv) @@ -123,10 +102,6 @@ fileprivate final class DeviceTransferFileStreamImpl: DeviceTransferFileStream { } override func close() { - defer { - try? fileManager.removeItem(at: tempURL) - } - do { let finalData = try decryptor.finalize() handle.write(finalData) @@ -135,33 +110,17 @@ fileprivate final class DeviceTransferFileStreamImpl: DeviceTransferFileStream { Logger.general.error(category: "DeviceTransferFileStream", message: "\(id) Close: \(error)") } - guard remoteHMAC.count == DeviceTransferProtocol.hmacDataCount else { - Logger.general.error(category: "DeviceTransferFileStream", message: "\(id) Invalid HMAC: \(remoteHMAC.count)") - return - } let localHMAC = localHMAC.finalize() guard localHMAC == remoteHMAC else { let local = localHMAC.base64EncodedString() let remote = remoteHMAC.base64EncodedString() Logger.general.error(category: "DeviceTransferFileStream", message: "\(id) Local HMAC: \(local), Remote HMAC: \(remote)") + client?.stop(reason: .exception(.mismatchedHMAC(local: localHMAC, remote: remoteHMAC))) return } - - for destinationURL in destinationURLs { - let path = destinationURL.path - if fileManager.fileExists(atPath: path) { - if fileManager.fileSize(path) == 0 { - try? fileManager.removeItem(atPath: path) - } else { - continue - } - } - do { - try fileManager.copyItem(at: tempURL, to: destinationURL) - } catch { - Logger.general.error(category: "DeviceTransferFileStream", message: "\(id) Not copied: \(error)") - } - } + #if DEBUG + Logger.general.info(category: "DeviceTransferFileStream", message: "\(id) Closed") + #endif } } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift b/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift index 8415f60e5e..4e9250c37a 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift @@ -168,10 +168,12 @@ extension DeviceTransferServer { connection = nil DispatchQueue.main.sync(execute: stopSpeedInspecting) switch reason { - case .finished: - state = .closed(.finished) + case .transferFinished: + state = .closed(.transferFinished) case .exception(let error): state = .closed(.exception(error)) + case .importFinished: + break } } @@ -297,7 +299,7 @@ extension DeviceTransferServer { } } case .finish: - self.stop(reason: .finished) + self.stop(reason: .transferFinished) case let .progress(progress): if case let .transfer(_, speed) = state { state = .transfer(progress: progress, speed: speed) diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift index 28b1785220..901f02cd1f 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift @@ -176,6 +176,8 @@ extension DeviceTransferProgressViewController { updateTitleLabel(with: progress, speed: speed) case let .closed(reason): handleConnectionClosing(reason: reason) + case let .importing(progress): + updateTitleLabel(with: progress) } } @@ -195,20 +197,28 @@ extension DeviceTransferProgressViewController { private func handleConnectionClosing(reason: DeviceTransferClosedReason) { switch reason { - case .finished: - let hint: String + case .transferFinished: switch connection { case .server: - hint = R.string.localizable.transfer_completed() + let hint = R.string.localizable.transfer_completed() + titleLabel.text = hint + progressView.progress = 1 + transferSucceeded(hint: hint) + speedLabel.isHidden = true + stateObserver?.cancel() + Logger.general.info(category: "DeviceTransferProgress", message: "Transfer succeeded") case .client: - hint = R.string.localizable.restore_completed() + speedLabel.isHidden = true + titleLabel.text = R.string.localizable.importing_chat_progress("") + tipLabel.text = R.string.localizable.keep_running_foreground() case .cloud: return } + case .importFinished: + let hint = R.string.localizable.restore_completed() titleLabel.text = hint progressView.progress = 1 transferSucceeded(hint: hint) - speedLabel.isHidden = true stateObserver?.cancel() Logger.general.info(category: "DeviceTransferProgress", message: "Transfer succeeded") case .exception(let error): @@ -221,6 +231,11 @@ extension DeviceTransferProgressViewController { } } + private func updateTitleLabel(with importProgress: Double) { + titleLabel.text = R.string.localizable.importing_chat_progress(String(format: "%.2f", importProgress * 100)) + progressView.progress = Float(importProgress) + } + } // MARK: - Cloud Worker diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift index 84c85ded6d..7354bcacd9 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift @@ -140,7 +140,7 @@ extension RestoreFromDesktopViewController { private func stateDidChange(client: DeviceTransferClient, state: DeviceTransferClient.State) { switch state { - case .idle: + case .idle, .importing: break case .transfer: stateObserver?.cancel() diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToDesktopViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToDesktopViewController.swift index ac2901649e..302f42e596 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToDesktopViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToDesktopViewController.swift @@ -139,7 +139,7 @@ extension TransferToDesktopViewController { navigationController?.pushViewController(progress, animated: true) case let .closed(reason): switch reason { - case .finished: + case .transferFinished, .importFinished: break case .exception(let error): alert(R.string.localizable.connection_establishment_failed(), message: error.localizedDescription) { _ in diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToPhoneQRCodeViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToPhoneQRCodeViewController.swift index 7e4123019c..f8d054ad60 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToPhoneQRCodeViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToPhoneQRCodeViewController.swift @@ -129,7 +129,7 @@ extension TransferToPhoneQRCodeViewController { case let .closed(reason): isListening = false switch reason { - case .finished: + case .transferFinished, .importFinished: break case .exception(let error): presentRestartServerAlert(message: error.localizedDescription) From 8dfe62a56174ca1ac5e28eddc475349708f19070 Mon Sep 17 00:00:00 2001 From: fanyu Date: Thu, 1 Jun 2023 20:53:23 +0800 Subject: [PATCH 02/23] Apply code style --- .../DeviceTransfer/DeviceTransferClient.swift | 31 +++++++++++++------ .../DeviceTransferClosedReason.swift | 3 +- .../DeviceTransferDataWriter.swift | 12 ++----- .../DeviceTransferFileStream.swift | 22 ++++++------- .../DeviceTransfer/DeviceTransferServer.swift | 8 ++--- ...DeviceTransferProgressViewController.swift | 20 +++++++----- .../RestoreFromDesktopViewController.swift | 2 +- .../TransferToDesktopViewController.swift | 2 +- .../TransferToPhoneQRCodeViewController.swift | 2 +- 9 files changed, 51 insertions(+), 51 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index 446401041f..a32fd1b21c 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -9,6 +9,7 @@ final class DeviceTransferClient { case transfer(progress: Double, speed: String) case closed(DeviceTransferClosedReason) case importing(progress: Double) + case finished } @Published private(set) var state: State = .idle @@ -104,10 +105,8 @@ final class DeviceTransferClient { } connection.cancel() switch reason { - case .transferFinished: - state = .closed(.transferFinished) - case .importFinished: - break + case .finished: + state = .closed(.finished) case .exception(let error): DispatchQueue.main.async { self.dataWriter.delegate = nil @@ -236,7 +235,7 @@ extension DeviceTransferClient { self.dataWriter.delegate = self } dataWriter.transferFinished() - self.stop(reason: .transferFinished) + self.stop(reason: .finished) default: break } @@ -280,12 +279,18 @@ extension DeviceTransferClient { isReceivingNewFile = false } else { assertionFailure("Should be closed by the end of previous call") - currentStream.close() - stream = DeviceTransferFileStream(context: context, key: key, client: self) + do { + try currentStream.close() + } catch let DeviceTransferError.mismatchedHMAC(local, remote) { + stop(reason: .exception(.mismatchedHMAC(local: local, remote: remote))) + } catch { + stop(reason: .exception(.failed(error))) + } + stream = DeviceTransferFileStream(context: context, key: key) isReceivingNewFile = true } } else { - stream = DeviceTransferFileStream(context: context, key: key, client: self) + stream = DeviceTransferFileStream(context: context, key: key) isReceivingNewFile = true } if isReceivingNewFile { @@ -306,7 +311,13 @@ extension DeviceTransferClient { stop(reason: .exception(.receiveFile(error))) } if context.remainingLength == 0 { - stream.close() + do { + try stream.close() + } catch let DeviceTransferError.mismatchedHMAC(local, remote) { + stop(reason: .exception(.mismatchedHMAC(local: local, remote: remote))) + } catch { + stop(reason: .exception(.failed(error))) + } self.fileStream = nil } } @@ -319,7 +330,7 @@ extension DeviceTransferClient: DeviceTransferDataWriterDelegate { if progress >= 1 { Logger.general.info(category: "DeviceTransferClient", message: "Import finished") ConversationDAO.shared.updateLastMessageIdAndCreatedAt() - state = .closed(.importFinished) + state = .finished } else { state = .importing(progress: progress) } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClosedReason.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClosedReason.swift index 291fcf0925..21540c9f4a 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClosedReason.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClosedReason.swift @@ -1,7 +1,6 @@ import Foundation enum DeviceTransferClosedReason { - case transferFinished - case importFinished + case finished case exception(DeviceTransferError) } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift b/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift index 9453381a77..2351acffc4 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift @@ -17,7 +17,6 @@ final class DeviceTransferDataWriter { parsedRecordCount = 0 totalRecordCount = 0 fileHandle = nil - cleanUpIfNeeded() } } @@ -79,14 +78,6 @@ final class DeviceTransferDataWriter { extension DeviceTransferDataWriter { - private func cleanUpIfNeeded() { - let path = DeviceTransferData.url().path - if FileManager.default.fileExists(atPath: path) { - try? FileManager.default.removeItem(atPath: path) - Logger.general.info(category: "DeviceTransferDataWriter", message: "Clean folder: \(path)") - } - } - private func openNextFile() { let filePath = DeviceTransferData.record.url(index: fileIndex).path if FileManager.default.fileExists(atPath: filePath) { @@ -146,6 +137,7 @@ extension DeviceTransferDataWriter { } } fileHandle.closeFile() + try? FileManager.default.removeItem(at: filePath) queue.async { self.readAndParseRecordData() } @@ -264,7 +256,7 @@ extension DeviceTransferDataWriter { } } } - cleanUpIfNeeded() + try? FileManager.default.removeItem(atPath: DeviceTransferData.url().path) } } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferFileStream.swift b/Mixin/Service/DeviceTransfer/DeviceTransferFileStream.swift index 3c481a13cf..97c6afb4eb 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferFileStream.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferFileStream.swift @@ -3,20 +3,17 @@ import MixinServices class DeviceTransferFileStream: InstanceInitializable { - weak var client: DeviceTransferClient? - let id: UUID - fileprivate init(id: UUID, client: DeviceTransferClient) { + fileprivate init(id: UUID) { self.id = id - self.client = client } - convenience init(context: DeviceTransferProtocol.FileContext, key: DeviceTransferKey, client: DeviceTransferClient) { - if let impl = DeviceTransferFileStreamImpl(context, key: key, client: client) { + convenience init(context: DeviceTransferProtocol.FileContext, key: DeviceTransferKey) { + if let impl = DeviceTransferFileStreamImpl(context, key: key) { self.init(instance: impl as! Self) } else { - self.init(id: context.fileHeader.id, client: client) + self.init(id: context.fileHeader.id) } } @@ -24,7 +21,7 @@ class DeviceTransferFileStream: InstanceInitializable { } - func close() { + func close() throws { } @@ -40,7 +37,7 @@ fileprivate final class DeviceTransferFileStreamImpl: DeviceTransferFileStream { private var localHMAC: HMACSHA256 private var remoteHMAC = Data(capacity: DeviceTransferProtocol.hmacDataCount) - init?(_ context: DeviceTransferProtocol.FileContext, key: DeviceTransferKey, client: DeviceTransferClient) { + init?(_ context: DeviceTransferProtocol.FileContext, key: DeviceTransferKey) { let decryptor: AESCryptor do { decryptor = try AESCryptor(operation: .decrypt, iv: context.fileHeader.iv, key: key.aes) @@ -63,7 +60,7 @@ fileprivate final class DeviceTransferFileStreamImpl: DeviceTransferFileStream { self.decryptor = decryptor self.remainingDataCount = Int(context.header.length) - idData.count - DeviceTransferProtocol.ivDataCount self.localHMAC = HMACSHA256(key: key.hmac) - super.init(id: context.fileHeader.id, client: client) + super.init(id: context.fileHeader.id) localHMAC.update(data: idData) localHMAC.update(data: context.fileHeader.iv) @@ -101,7 +98,7 @@ fileprivate final class DeviceTransferFileStreamImpl: DeviceTransferFileStream { } } - override func close() { + override func close() throws { do { let finalData = try decryptor.finalize() handle.write(finalData) @@ -115,8 +112,7 @@ fileprivate final class DeviceTransferFileStreamImpl: DeviceTransferFileStream { let local = localHMAC.base64EncodedString() let remote = remoteHMAC.base64EncodedString() Logger.general.error(category: "DeviceTransferFileStream", message: "\(id) Local HMAC: \(local), Remote HMAC: \(remote)") - client?.stop(reason: .exception(.mismatchedHMAC(local: localHMAC, remote: remoteHMAC))) - return + throw DeviceTransferError.mismatchedHMAC(local: localHMAC, remote: remoteHMAC) } #if DEBUG Logger.general.info(category: "DeviceTransferFileStream", message: "\(id) Closed") diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift b/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift index 4e9250c37a..8415f60e5e 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift @@ -168,12 +168,10 @@ extension DeviceTransferServer { connection = nil DispatchQueue.main.sync(execute: stopSpeedInspecting) switch reason { - case .transferFinished: - state = .closed(.transferFinished) + case .finished: + state = .closed(.finished) case .exception(let error): state = .closed(.exception(error)) - case .importFinished: - break } } @@ -299,7 +297,7 @@ extension DeviceTransferServer { } } case .finish: - self.stop(reason: .transferFinished) + self.stop(reason: .finished) case let .progress(progress): if case let .transfer(_, speed) = state { state = .transfer(progress: progress, speed: speed) diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift index 901f02cd1f..aaabf62e2f 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift @@ -178,6 +178,8 @@ extension DeviceTransferProgressViewController { handleConnectionClosing(reason: reason) case let .importing(progress): updateTitleLabel(with: progress) + case .finished: + importFinished() } } @@ -197,7 +199,7 @@ extension DeviceTransferProgressViewController { private func handleConnectionClosing(reason: DeviceTransferClosedReason) { switch reason { - case .transferFinished: + case .finished: switch connection { case .server: let hint = R.string.localizable.transfer_completed() @@ -214,13 +216,6 @@ extension DeviceTransferProgressViewController { case .cloud: return } - case .importFinished: - let hint = R.string.localizable.restore_completed() - titleLabel.text = hint - progressView.progress = 1 - transferSucceeded(hint: hint) - stateObserver?.cancel() - Logger.general.info(category: "DeviceTransferProgress", message: "Transfer succeeded") case .exception(let error): let hint = R.string.localizable.transfer_failed() titleLabel.text = hint @@ -236,6 +231,15 @@ extension DeviceTransferProgressViewController { progressView.progress = Float(importProgress) } + private func importFinished() { + let hint = R.string.localizable.restore_completed() + titleLabel.text = hint + progressView.progress = 1 + transferSucceeded(hint: hint) + stateObserver?.cancel() + Logger.general.info(category: "DeviceTransferProgress", message: "Transfer succeeded") + } + } // MARK: - Cloud Worker diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift index 7354bcacd9..ea453bf1ea 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift @@ -140,7 +140,7 @@ extension RestoreFromDesktopViewController { private func stateDidChange(client: DeviceTransferClient, state: DeviceTransferClient.State) { switch state { - case .idle, .importing: + case .idle, .importing, .finished: break case .transfer: stateObserver?.cancel() diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToDesktopViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToDesktopViewController.swift index 302f42e596..ac2901649e 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToDesktopViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToDesktopViewController.swift @@ -139,7 +139,7 @@ extension TransferToDesktopViewController { navigationController?.pushViewController(progress, animated: true) case let .closed(reason): switch reason { - case .transferFinished, .importFinished: + case .finished: break case .exception(let error): alert(R.string.localizable.connection_establishment_failed(), message: error.localizedDescription) { _ in diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToPhoneQRCodeViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToPhoneQRCodeViewController.swift index f8d054ad60..7e4123019c 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToPhoneQRCodeViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/TransferToPhoneQRCodeViewController.swift @@ -129,7 +129,7 @@ extension TransferToPhoneQRCodeViewController { case let .closed(reason): isListening = false switch reason { - case .transferFinished, .importFinished: + case .finished: break case .exception(let error): presentRestartServerAlert(message: error.localizedDescription) From 86d2f8f8fa63d67fe91a23bd34dcb63d24bb82b6 Mon Sep 17 00:00:00 2001 From: fanyu Date: Fri, 2 Jun 2023 09:17:24 +0800 Subject: [PATCH 03/23] Apply code style --- Mixin/Service/DeviceTransfer/DeviceTransferClient.swift | 4 ++-- Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift | 4 ++-- .../DeviceTransfer/DeviceTransferProgressViewController.swift | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index a32fd1b21c..db4a7167a9 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -8,7 +8,7 @@ final class DeviceTransferClient { case idle case transfer(progress: Double, speed: String) case closed(DeviceTransferClosedReason) - case importing(progress: Double) + case importing(progress: Float) case finished } @@ -326,7 +326,7 @@ extension DeviceTransferClient { extension DeviceTransferClient: DeviceTransferDataWriterDelegate { - func deviceTransferDataWriter(_ writer: DeviceTransferDataWriter, update progress: Double) { + func deviceTransferDataWriter(_ writer: DeviceTransferDataWriter, update progress: Float) { if progress >= 1 { Logger.general.info(category: "DeviceTransferClient", message: "Import finished") ConversationDAO.shared.updateLastMessageIdAndCreatedAt() diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift b/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift index 2351acffc4..0220742ee1 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift @@ -3,7 +3,7 @@ import MixinServices protocol DeviceTransferDataWriterDelegate: AnyObject { - func deviceTransferDataWriter(_ writer: DeviceTransferDataWriter, update progress: Double) + func deviceTransferDataWriter(_ writer: DeviceTransferDataWriter, update progress: Float) } @@ -130,7 +130,7 @@ extension DeviceTransferDataWriter { } } } else { - let progress = Double(self.parsedRecordCount) / Double(self.totalRecordCount + 1) + let progress = Float(self.parsedRecordCount) / Float(self.totalRecordCount + 1) delegate.deviceTransferDataWriter(self, update: progress) } } diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift index aaabf62e2f..1234b7de78 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift @@ -226,9 +226,9 @@ extension DeviceTransferProgressViewController { } } - private func updateTitleLabel(with importProgress: Double) { + private func updateTitleLabel(with importProgress: Float) { titleLabel.text = R.string.localizable.importing_chat_progress(String(format: "%.2f", importProgress * 100)) - progressView.progress = Float(importProgress) + progressView.progress = importProgress } private func importFinished() { From 271a3d8307fa2771b736706fb0771167a81f1f9b Mon Sep 17 00:00:00 2001 From: fanyu Date: Mon, 5 Jun 2023 17:53:43 +0800 Subject: [PATCH 04/23] Refactor error handling --- Mixin.xcodeproj/project.pbxproj | 4 -- .../DeviceTransfer/DeviceTransferClient.swift | 43 +++++++-------- .../DeviceTransferClosedReason.swift | 6 --- .../DeviceTransferDataWriter.swift | 2 +- .../DeviceTransfer/DeviceTransferError.swift | 1 + .../DeviceTransfer/DeviceTransferServer.swift | 15 ++++-- ...DeviceTransferProgressViewController.swift | 53 ++++++++----------- .../RestoreFromDesktopViewController.swift | 8 ++- 8 files changed, 58 insertions(+), 74 deletions(-) delete mode 100644 Mixin/Service/DeviceTransfer/DeviceTransferClosedReason.swift diff --git a/Mixin.xcodeproj/project.pbxproj b/Mixin.xcodeproj/project.pbxproj index 5439afbf31..bdd77c2683 100644 --- a/Mixin.xcodeproj/project.pbxproj +++ b/Mixin.xcodeproj/project.pbxproj @@ -699,7 +699,6 @@ 940BAE222629741C00FFF753 /* AuthorizationsContentViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 940BAE212629741C00FFF753 /* AuthorizationsContentViewController.swift */; }; 94149B432A17B4D5003E9E1A /* NetworkSpeedConditioner.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94149B422A17B4D5003E9E1A /* NetworkSpeedConditioner.swift */; }; 94149B482A190889003E9E1A /* DeviceTransferRecord.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94149B472A190889003E9E1A /* DeviceTransferRecord.swift */; }; - 94149B582A1A6A6D003E9E1A /* DeviceTransferClosedReason.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94149B572A1A6A6D003E9E1A /* DeviceTransferClosedReason.swift */; }; 9416548325CD2D9A007E76D0 /* OggOpusRecorder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9416548225CD2D9A007E76D0 /* OggOpusRecorder.swift */; }; 9416548B25CD7190007E76D0 /* AudioMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9416548A25CD7190007E76D0 /* AudioMetadata.swift */; }; 941655C425CD7DA7007E76D0 /* AudioSessionClientPriority.swift in Sources */ = {isa = PBXBuildFile; fileRef = 941655C325CD7DA7007E76D0 /* AudioSessionClientPriority.swift */; }; @@ -1777,7 +1776,6 @@ 940BAE212629741C00FFF753 /* AuthorizationsContentViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AuthorizationsContentViewController.swift; sourceTree = ""; }; 94149B422A17B4D5003E9E1A /* NetworkSpeedConditioner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NetworkSpeedConditioner.swift; sourceTree = ""; }; 94149B472A190889003E9E1A /* DeviceTransferRecord.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeviceTransferRecord.swift; sourceTree = ""; }; - 94149B572A1A6A6D003E9E1A /* DeviceTransferClosedReason.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeviceTransferClosedReason.swift; sourceTree = ""; }; 9416548225CD2D9A007E76D0 /* OggOpusRecorder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OggOpusRecorder.swift; sourceTree = ""; }; 9416548A25CD7190007E76D0 /* AudioMetadata.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioMetadata.swift; sourceTree = ""; }; 941655C325CD7DA7007E76D0 /* AudioSessionClientPriority.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioSessionClientPriority.swift; sourceTree = ""; }; @@ -2766,7 +2764,6 @@ 7CEB735A29DBB6F3006FB5B2 /* TransferMessage */, 7C140F5829E19C9D00F05506 /* Metadata */, 94C199722A1CA9430098EDB3 /* DeviceTransferError.swift */, - 94149B572A1A6A6D003E9E1A /* DeviceTransferClosedReason.swift */, 94E1D44D29E9393C00511267 /* DeviceTransferServer.swift */, 940B304A2A160BAB00B45D26 /* DeviceTransferServerDataSource.swift */, 94396F2929EBE52400A57833 /* DeviceTransferClient.swift */, @@ -4706,7 +4703,6 @@ 7BD38B6020BEA03200D06E5C /* WaveformView.swift in Sources */, DFB19006233220290021CAF3 /* PINLogCell.swift in Sources */, 94DF7D5928DC0F17006E415B /* Acknowledgement.swift in Sources */, - 94149B582A1A6A6D003E9E1A /* DeviceTransferClosedReason.swift in Sources */, 7C427BC428373F8000FFDE12 /* Wallpaper.swift in Sources */, E090B90F23B0B27F0012C7E9 /* ConversationDAO+Search.swift in Sources */, 7BF42E3122DC85E9005066E6 /* GalleryImageItemViewController.swift in Sources */, diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index db4a7167a9..9252f1fc41 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -7,7 +7,7 @@ final class DeviceTransferClient { enum State { case idle case transfer(progress: Double, speed: String) - case closed(DeviceTransferClosedReason) + case failed(DeviceTransferError) case importing(progress: Float) case finished } @@ -86,7 +86,7 @@ final class DeviceTransferClient { case .failed(let error): Logger.general.warn(category: "DeviceTransferClient", message: "Failed: \(error)") if let self { - self.stop(reason: .exception(.failed(error))) + self.stop(error: .connectionFailed(error)) } case .cancelled: Logger.general.info(category: "DeviceTransferClient", message: "Connection cancelled") @@ -97,23 +97,18 @@ final class DeviceTransferClient { connection.start(queue: queue.dispatchQueue) } - func stop(reason: DeviceTransferClosedReason) { + func stop(error: DeviceTransferError) { assert(queue.isCurrent) - Logger.general.info(category: "DeviceTransferClient", message: "Stop: \(reason) Processed: \(processedCount) Total: \(totalCount)") + Logger.general.info(category: "DeviceTransferClient", message: "Stop: \(error) Processed: \(processedCount) Total: \(totalCount)") DispatchQueue.main.sync { self.statisticsTimer?.invalidate() } connection.cancel() - switch reason { - case .finished: - state = .closed(.finished) - case .exception(let error): - DispatchQueue.main.async { - self.dataWriter.delegate = nil - self.dataWriter.canProcessData = false - } - state = .closed(.exception(error)) + DispatchQueue.main.async { + self.dataWriter.delegate = nil + self.dataWriter.canProcessData = false } + state = .failed(error) } private func startUpdatingProgressAndSpeed() { @@ -162,7 +157,9 @@ extension DeviceTransferClient { let message = contentContext?.protocolMetadata(definition: DeviceTransferProtocol.definition) as? NWProtocolFramer.Message else { if isComplete { - self.stop(reason: .exception(.remoteComplete)) + if case .transfer = self.state { + self.stop(error: .remoteComplete) + } Logger.general.warn(category: "DeviceTransferClient", message: "Remote closed") } return @@ -199,7 +196,7 @@ extension DeviceTransferClient { let localHMAC = HMACSHA256.mac(for: encryptedData, using: key.hmac) let remoteHMAC = content[firstHMACIndex...] guard localHMAC == remoteHMAC else { - stop(reason: .exception(.mismatchedHMAC(local: localHMAC, remote: remoteHMAC))) + stop(error: .mismatchedHMAC(local: localHMAC, remote: remoteHMAC)) return } @@ -234,8 +231,8 @@ extension DeviceTransferClient { DispatchQueue.main.async { self.dataWriter.delegate = self } + state = .importing(progress: 0) dataWriter.transferFinished() - self.stop(reason: .finished) default: break } @@ -253,13 +250,13 @@ extension DeviceTransferClient { let localHMAC = HMACSHA256.mac(for: encryptedData, using: key.hmac) let remoteHMAC = content[firstHMACIndex...] guard localHMAC == remoteHMAC else { - stop(reason: .exception(.mismatchedHMAC(local: localHMAC, remote: remoteHMAC))) + stop(error: .mismatchedHMAC(local: localHMAC, remote: remoteHMAC)) return } do { let decryptedData = try AESCryptor.decrypt(encryptedData, with: key.aes) if !dataWriter.write(data: decryptedData) { - stop(reason: .exception(.unableSaveData)) + stop(error: .unableSaveData) } } catch { Logger.general.error(category: "DeviceTransferClient", message: "Unable to decrypt: \(error)") @@ -282,9 +279,9 @@ extension DeviceTransferClient { do { try currentStream.close() } catch let DeviceTransferError.mismatchedHMAC(local, remote) { - stop(reason: .exception(.mismatchedHMAC(local: local, remote: remote))) + stop(error: .mismatchedHMAC(local: local, remote: remote)) } catch { - stop(reason: .exception(.failed(error))) + stop(error: .failed(error)) } stream = DeviceTransferFileStream(context: context, key: key) isReceivingNewFile = true @@ -308,15 +305,15 @@ extension DeviceTransferClient { try stream.write(data: content) } catch { Logger.general.error(category: "DeviceTransferClient", message: "Failed to write: \(error)") - stop(reason: .exception(.receiveFile(error))) + stop(error: .receiveFile(error)) } if context.remainingLength == 0 { do { try stream.close() } catch let DeviceTransferError.mismatchedHMAC(local, remote) { - stop(reason: .exception(.mismatchedHMAC(local: local, remote: remote))) + stop(error: .mismatchedHMAC(local: local, remote: remote)) } catch { - stop(reason: .exception(.failed(error))) + stop(error: .failed(error)) } self.fileStream = nil } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClosedReason.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClosedReason.swift deleted file mode 100644 index 21540c9f4a..0000000000 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClosedReason.swift +++ /dev/null @@ -1,6 +0,0 @@ -import Foundation - -enum DeviceTransferClosedReason { - case finished - case exception(DeviceTransferError) -} diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift b/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift index 0220742ee1..6910e0e25b 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift @@ -58,7 +58,7 @@ final class DeviceTransferDataWriter { let lenghtData = UInt32(data.count).data(endianness: .big) fileHandle.write(lenghtData) fileHandle.write(data) - DispatchQueue.main.sync { + DispatchQueue.main.async { self.totalRecordCount += 1 } return true diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferError.swift b/Mixin/Service/DeviceTransfer/DeviceTransferError.swift index d66e4f6ad9..198c3f196a 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferError.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferError.swift @@ -5,6 +5,7 @@ enum DeviceTransferError: Error { case mismatchedConnection case encrypt(Error) case mismatchedHMAC(local: Data, remote: Data) + case connectionFailed(Error) case failed(Error) case receiveFile(Error) case unableSaveData diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift b/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift index 8415f60e5e..8a884d3683 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift @@ -13,7 +13,12 @@ final class DeviceTransferServer { case idle case listening(hostname: String, port: UInt16) case transfer(progress: Double, speed: String) - case closed(DeviceTransferClosedReason) + case closed(ClosedReason) + } + + enum ClosedReason { + case finished + case exception(DeviceTransferError) } let code: UInt16 = .random(in: 0...999) @@ -85,7 +90,7 @@ extension DeviceTransferServer { } } catch { Logger.general.warn(category: "DeviceTransferServer", message: "Listener ready without a hostname") - self?.state = .closed(.exception(.failed(error))) + self?.state = .closed(.exception(.connectionFailed(error))) } case let .failed(error), let .waiting(error): Logger.general.warn(category: "DeviceTransferServer", message: "Not listening: \(error)") @@ -159,7 +164,7 @@ extension DeviceTransferServer { lastConnectionRejectedReason = nil } - private func stop(reason: DeviceTransferClosedReason) { + private func stop(reason: ClosedReason) { assert(queue.isCurrent) Logger.general.info(category: "DeviceTransferServer", message: "Stop with reason: \(reason)") listener?.cancel() @@ -186,7 +191,7 @@ extension DeviceTransferServer { } startListening { error in Logger.general.error(category: "DeviceTransferServer", message: "Failed to start listening after connection rejected") - self.state = .closed(.exception(.failed(error))) + self.state = .closed(.exception(.connectionFailed(error))) } } @@ -214,7 +219,7 @@ extension DeviceTransferServer { } case .failed(let error): Logger.general.warn(category: "DeviceTransferServer", message: "Failed: \(error)") - self?.stop(reason: .exception(.failed(error))) + self?.stop(reason: .exception(.connectionFailed(error))) case .cancelled: Logger.general.info(category: "DeviceTransferServer", message: "Connection cancelled") @unknown default: diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift index 1234b7de78..564d210594 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift @@ -164,7 +164,18 @@ extension DeviceTransferProgressViewController { case let .transfer(progress, speed): updateTitleLabel(with: progress, speed: speed) case let .closed(reason): - handleConnectionClosing(reason: reason) + switch reason { + case .finished: + let hint = R.string.localizable.transfer_completed() + titleLabel.text = hint + progressView.progress = 1 + transferSucceeded(hint: hint) + speedLabel.isHidden = true + stateObserver?.cancel() + Logger.general.info(category: "DeviceTransferProgress", message: "Transfer succeeded") + case .exception(let error): + handleConnectionClosing(error: error) + } } } @@ -174,8 +185,8 @@ extension DeviceTransferProgressViewController { Logger.general.warn(category: "DeviceTransferProgress", message: "Invalid state: \(state)") case let .transfer(progress, speed): updateTitleLabel(with: progress, speed: speed) - case let .closed(reason): - handleConnectionClosing(reason: reason) + case let .failed(error): + handleConnectionClosing(error: error) case let .importing(progress): updateTitleLabel(with: progress) case .finished: @@ -197,36 +208,18 @@ extension DeviceTransferProgressViewController { speedLabel.text = speed } - private func handleConnectionClosing(reason: DeviceTransferClosedReason) { - switch reason { - case .finished: - switch connection { - case .server: - let hint = R.string.localizable.transfer_completed() - titleLabel.text = hint - progressView.progress = 1 - transferSucceeded(hint: hint) - speedLabel.isHidden = true - stateObserver?.cancel() - Logger.general.info(category: "DeviceTransferProgress", message: "Transfer succeeded") - case .client: - speedLabel.isHidden = true - titleLabel.text = R.string.localizable.importing_chat_progress("") - tipLabel.text = R.string.localizable.keep_running_foreground() - case .cloud: - return - } - case .exception(let error): - let hint = R.string.localizable.transfer_failed() - titleLabel.text = hint - transferFailed(hint: hint) - speedLabel.isHidden = true - stateObserver?.cancel() - Logger.general.error(category: "DeviceTransferProgress", message: "Transfer failed: \(error)") - } + private func handleConnectionClosing(error: DeviceTransferError) { + let hint = R.string.localizable.transfer_failed() + titleLabel.text = hint + transferFailed(hint: hint) + speedLabel.isHidden = true + stateObserver?.cancel() + Logger.general.error(category: "DeviceTransferProgress", message: "Transfer failed: \(error)") } private func updateTitleLabel(with importProgress: Float) { + speedLabel.isHidden = true + tipLabel.text = R.string.localizable.keep_running_foreground() titleLabel.text = R.string.localizable.importing_chat_progress(String(format: "%.2f", importProgress * 100)) progressView.progress = importProgress } diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift index ea453bf1ea..a7957e9790 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift @@ -148,14 +148,12 @@ extension RestoreFromDesktopViewController { tableView.isUserInteractionEnabled = true let progress = DeviceTransferProgressViewController(connection: .client(client, .desktop)) navigationController?.pushViewController(progress, animated: true) - case let .closed(reason): + case let .failed(error): dataSource.replaceSection(at: 0, with: section, animation: .automatic) tableView.isUserInteractionEnabled = true stateObserver?.cancel() - if case let .exception(error) = reason { - alert(R.string.localizable.connection_establishment_failed(), message: error.localizedDescription) { _ in - self.navigationController?.popViewController(animated: true) - } + alert(R.string.localizable.connection_establishment_failed(), message: error.localizedDescription) { _ in + self.navigationController?.popViewController(animated: true) } } } From 2ee65531bff5a27619215da6470a14f700f79f5b Mon Sep 17 00:00:00 2001 From: fanyu Date: Tue, 6 Jun 2023 15:02:13 +0800 Subject: [PATCH 05/23] Batch save messages --- .../DeviceTransferDataWriter.swift | 16 +++++++- .../Database/User/DAO/MessageDAO.swift | 38 ++++++++++--------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift b/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift index 6910e0e25b..dc90b281b5 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift @@ -28,6 +28,7 @@ final class DeviceTransferDataWriter { private var totalRecordCount = 0 private var parsedRecordCount = 0 private var pendingParsedRecordPath = [URL]() + private var pendingSaveMessages = [Message]() init(remotePlatform: DeviceTransferPlatform) { self.remotePlatform = remotePlatform @@ -187,11 +188,16 @@ extension DeviceTransferDataWriter { case .message: let message = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data if MessageCategory.isLegal(category: message.category) { - MessageDAO.shared.save(message: message.toMessage()) + pendingSaveMessages.append(message.toMessage()) + if pendingSaveMessages.count > 1000 { + MessageDAO.shared.save(messages: pendingSaveMessages) + pendingSaveMessages.removeAll() + } } else { Logger.general.warn(category: "DeviceTransferDataWriter", message: "Message is illegal: \(message)") } case .messageMention: + saveMessagesIfNeeded() let messageMention = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data if let mention = messageMention.toMessageMention() { MessageMentionDAO.shared.save(messageMention: mention) @@ -210,6 +216,7 @@ extension DeviceTransferDataWriter { private func processFiles() { assert(queue.isCurrent) + saveMessagesIfNeeded() guard let fileEnumerator = FileManager.default.enumerator(at: DeviceTransferData.file.url(name: nil), includingPropertiesForKeys: [.isRegularFileKey], @@ -259,4 +266,11 @@ extension DeviceTransferDataWriter { try? FileManager.default.removeItem(atPath: DeviceTransferData.url().path) } + private func saveMessagesIfNeeded() { + if !pendingSaveMessages.isEmpty { + MessageDAO.shared.save(messages: pendingSaveMessages) + pendingSaveMessages.removeAll() + } + } + } diff --git a/MixinServices/MixinServices/Database/User/DAO/MessageDAO.swift b/MixinServices/MixinServices/Database/User/DAO/MessageDAO.swift index 9611c9d115..82c1ba4d81 100644 --- a/MixinServices/MixinServices/Database/User/DAO/MessageDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/MessageDAO.swift @@ -1000,26 +1000,28 @@ extension MessageDAO { order: [Message.column(of: .createdAt).desc]) } - public func save(message: Message) { + public func save(messages: [Message]) { db.write { db in - let exists = try message.exists(db) - guard !exists else { - return - } - try message.save(db) - let shouldInsertIntoFTSTable = AppGroupUserDefaults.Database.isFTSInitialized - && message.status != MessageStatus.FAILED.rawValue - && MessageCategory.ftsAvailableCategoryStrings.contains(message.category) - if shouldInsertIntoFTSTable { - let children: [TranscriptMessage]? - if message.category.hasSuffix("_TRANSCRIPT") { - children = try TranscriptMessage - .filter(TranscriptMessage.column(of: .transcriptId) == message.messageId) - .fetchAll(db) - } else { - children = nil + for message in messages { + let exists = try message.exists(db) + guard !exists else { + continue + } + try message.save(db) + let shouldInsertIntoFTSTable = AppGroupUserDefaults.Database.isFTSInitialized + && message.status != MessageStatus.FAILED.rawValue + && MessageCategory.ftsAvailableCategoryStrings.contains(message.category) + if shouldInsertIntoFTSTable { + let children: [TranscriptMessage]? + if message.category.hasSuffix("_TRANSCRIPT") { + children = try TranscriptMessage + .filter(TranscriptMessage.column(of: .transcriptId) == message.messageId) + .fetchAll(db) + } else { + children = nil + } + try insertFTSContent(db, message: message, children: children) } - try insertFTSContent(db, message: message, children: children) } } } From af280196b9f04ea4d9fa01416e9a637d145e5151 Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Tue, 6 Jun 2023 01:35:07 +0800 Subject: [PATCH 06/23] Apply code style --- .../DeviceTransfer/DeviceTransferClient.swift | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index 9252f1fc41..9e874a4737 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -86,7 +86,7 @@ final class DeviceTransferClient { case .failed(let error): Logger.general.warn(category: "DeviceTransferClient", message: "Failed: \(error)") if let self { - self.stop(error: .connectionFailed(error)) + self.fail(error: .connectionFailed(error)) } case .cancelled: Logger.general.info(category: "DeviceTransferClient", message: "Connection cancelled") @@ -97,7 +97,7 @@ final class DeviceTransferClient { connection.start(queue: queue.dispatchQueue) } - func stop(error: DeviceTransferError) { + private func fail(error: DeviceTransferError) { assert(queue.isCurrent) Logger.general.info(category: "DeviceTransferClient", message: "Stop: \(error) Processed: \(processedCount) Total: \(totalCount)") DispatchQueue.main.sync { @@ -158,7 +158,7 @@ extension DeviceTransferClient { else { if isComplete { if case .transfer = self.state { - self.stop(error: .remoteComplete) + self.fail(error: .remoteComplete) } Logger.general.warn(category: "DeviceTransferClient", message: "Remote closed") } @@ -196,7 +196,7 @@ extension DeviceTransferClient { let localHMAC = HMACSHA256.mac(for: encryptedData, using: key.hmac) let remoteHMAC = content[firstHMACIndex...] guard localHMAC == remoteHMAC else { - stop(error: .mismatchedHMAC(local: localHMAC, remote: remoteHMAC)) + fail(error: .mismatchedHMAC(local: localHMAC, remote: remoteHMAC)) return } @@ -250,13 +250,13 @@ extension DeviceTransferClient { let localHMAC = HMACSHA256.mac(for: encryptedData, using: key.hmac) let remoteHMAC = content[firstHMACIndex...] guard localHMAC == remoteHMAC else { - stop(error: .mismatchedHMAC(local: localHMAC, remote: remoteHMAC)) + fail(error: .mismatchedHMAC(local: localHMAC, remote: remoteHMAC)) return } do { let decryptedData = try AESCryptor.decrypt(encryptedData, with: key.aes) if !dataWriter.write(data: decryptedData) { - stop(error: .unableSaveData) + fail(error: .unableSaveData) } } catch { Logger.general.error(category: "DeviceTransferClient", message: "Unable to decrypt: \(error)") @@ -278,10 +278,10 @@ extension DeviceTransferClient { assertionFailure("Should be closed by the end of previous call") do { try currentStream.close() - } catch let DeviceTransferError.mismatchedHMAC(local, remote) { - stop(error: .mismatchedHMAC(local: local, remote: remote)) + } catch let error as DeviceTransferError { + fail(error: error) } catch { - stop(error: .failed(error)) + fail(error: .receiveFile(error)) } stream = DeviceTransferFileStream(context: context, key: key) isReceivingNewFile = true @@ -305,15 +305,15 @@ extension DeviceTransferClient { try stream.write(data: content) } catch { Logger.general.error(category: "DeviceTransferClient", message: "Failed to write: \(error)") - stop(error: .receiveFile(error)) + fail(error: .receiveFile(error)) } if context.remainingLength == 0 { do { try stream.close() - } catch let DeviceTransferError.mismatchedHMAC(local, remote) { - stop(error: .mismatchedHMAC(local: local, remote: remote)) + } catch let error as DeviceTransferError { + fail(error: error) } catch { - stop(error: .failed(error)) + fail(error: .receiveFile(error)) } self.fileStream = nil } From 5efe12a685523f79e1a70187d1fabebf205ea7fa Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Thu, 8 Jun 2023 16:25:19 +0800 Subject: [PATCH 07/23] Refactor --- Mixin.xcodeproj/project.pbxproj | 12 +- .../DeviceTransfer/DeviceTransferClient.swift | 83 ++-- .../DeviceTransfer/DeviceTransferData.swift | 36 -- .../DeviceTransferDataWriter.swift | 276 ------------ .../DeviceTransfer/DeviceTransferError.swift | 4 +- .../DeviceTransferFileStream.swift | 10 +- .../DeviceTransferMessageProcessor.swift | 400 ++++++++++++++++++ ...DeviceTransferProgressViewController.swift | 14 +- 8 files changed, 469 insertions(+), 366 deletions(-) delete mode 100644 Mixin/Service/DeviceTransfer/DeviceTransferData.swift delete mode 100644 Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift create mode 100644 Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift diff --git a/Mixin.xcodeproj/project.pbxproj b/Mixin.xcodeproj/project.pbxproj index bdd77c2683..cbe912d99e 100644 --- a/Mixin.xcodeproj/project.pbxproj +++ b/Mixin.xcodeproj/project.pbxproj @@ -663,8 +663,7 @@ 7CE3A25C2771A8AB006BE765 /* DeleteAccountVerifyCodeViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CE3A25B2771A8AB006BE765 /* DeleteAccountVerifyCodeViewController.swift */; }; 7CE5E7A8269BDA29000B7904 /* HomeAppsPinTipsViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CE5E7A6269BDA29000B7904 /* HomeAppsPinTipsViewController.swift */; }; 7CE5E7A9269BDA29000B7904 /* HomeAppsPinTipsView.xib in Resources */ = {isa = PBXBuildFile; fileRef = 7CE5E7A7269BDA29000B7904 /* HomeAppsPinTipsView.xib */; }; - 7CE7ADD72A24983800A6259F /* DeviceTransferDataWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CE7ADD62A24983800A6259F /* DeviceTransferDataWriter.swift */; }; - 7CE7ADD92A24A83A00A6259F /* DeviceTransferData.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CE7ADD82A24A83A00A6259F /* DeviceTransferData.swift */; }; + 7CE7ADD72A24983800A6259F /* DeviceTransferMessageProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CE7ADD62A24983800A6259F /* DeviceTransferMessageProcessor.swift */; }; 7CEB735429DB24F3006FB5B2 /* RestoreChatViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CEB735229DB24F3006FB5B2 /* RestoreChatViewController.swift */; }; 7CEB735529DB24F3006FB5B2 /* RestoreChatView.xib in Resources */ = {isa = PBXBuildFile; fileRef = 7CEB735329DB24F3006FB5B2 /* RestoreChatView.xib */; }; 7CEB735829DB272F006FB5B2 /* RestoreChatTableViewCell.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CEB735629DB272F006FB5B2 /* RestoreChatTableViewCell.swift */; }; @@ -1737,8 +1736,7 @@ 7CE3A25B2771A8AB006BE765 /* DeleteAccountVerifyCodeViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeleteAccountVerifyCodeViewController.swift; sourceTree = ""; }; 7CE5E7A6269BDA29000B7904 /* HomeAppsPinTipsViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HomeAppsPinTipsViewController.swift; sourceTree = ""; }; 7CE5E7A7269BDA29000B7904 /* HomeAppsPinTipsView.xib */ = {isa = PBXFileReference; lastKnownFileType = file.xib; path = HomeAppsPinTipsView.xib; sourceTree = ""; }; - 7CE7ADD62A24983800A6259F /* DeviceTransferDataWriter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeviceTransferDataWriter.swift; sourceTree = ""; }; - 7CE7ADD82A24A83A00A6259F /* DeviceTransferData.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeviceTransferData.swift; sourceTree = ""; }; + 7CE7ADD62A24983800A6259F /* DeviceTransferMessageProcessor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeviceTransferMessageProcessor.swift; sourceTree = ""; }; 7CEB735229DB24F3006FB5B2 /* RestoreChatViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RestoreChatViewController.swift; sourceTree = ""; }; 7CEB735329DB24F3006FB5B2 /* RestoreChatView.xib */ = {isa = PBXFileReference; lastKnownFileType = file.xib; path = RestoreChatView.xib; sourceTree = ""; }; 7CEB735629DB272F006FB5B2 /* RestoreChatTableViewCell.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RestoreChatTableViewCell.swift; sourceTree = ""; }; @@ -2767,8 +2765,7 @@ 94E1D44D29E9393C00511267 /* DeviceTransferServer.swift */, 940B304A2A160BAB00B45D26 /* DeviceTransferServerDataSource.swift */, 94396F2929EBE52400A57833 /* DeviceTransferClient.swift */, - 7CE7ADD62A24983800A6259F /* DeviceTransferDataWriter.swift */, - 7CE7ADD82A24A83A00A6259F /* DeviceTransferData.swift */, + 7CE7ADD62A24983800A6259F /* DeviceTransferMessageProcessor.swift */, 94396F2F29ED005200A57833 /* DeviceTransferFileStream.swift */, 94396F2529EB11E300A57833 /* DeviceTransferProtocol.swift */, 94396F2729EB475500A57833 /* NWParameters+DeviceTransfer.swift */, @@ -4773,7 +4770,7 @@ 7C7635B826A13461006101DB /* HomeAppsConstants.swift in Sources */, DFB190002330F6A40021CAF3 /* BiographyViewController.swift in Sources */, 7B7F7E391FD43F2500A1C91F /* DetailInfoMessageCell.swift in Sources */, - 7CE7ADD72A24983800A6259F /* DeviceTransferDataWriter.swift in Sources */, + 7CE7ADD72A24983800A6259F /* DeviceTransferMessageProcessor.swift in Sources */, 7B8BFC8A1FDD77F9004E19DB /* UnknownMessageViewModel.swift in Sources */, 7CEB736C29DBD1C0006FB5B2 /* DeviceTransferMessage.swift in Sources */, 7B35AF7C228AA6CD00E8101D /* MessagesWithinConversationSearchResult.swift in Sources */, @@ -4970,7 +4967,6 @@ DFB19002233219650021CAF3 /* LogViewController.swift in Sources */, 7B054052243CCCD500C1FCB6 /* HomeTitleButton.swift in Sources */, 7BD127FC22D8892200BB5316 /* GalleryViewController.swift in Sources */, - 7CE7ADD92A24A83A00A6259F /* DeviceTransferData.swift in Sources */, DFD1FBC72302CB7A00C570D4 /* DatabaseUpgradeViewController.swift in Sources */, 944C656125D9BA0A008BDDD3 /* OggOpusWriter.swift in Sources */, 7BB8417C2230C63000FB88B9 /* ConversationAccessible.swift in Sources */, diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index 9e874a4737..d055f9d523 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -1,5 +1,6 @@ import Foundation import Network +import Combine import MixinServices final class DeviceTransferClient { @@ -21,8 +22,9 @@ final class DeviceTransferClient { private let remotePlatform: DeviceTransferPlatform private let connection: NWConnection private let queue = Queue(label: "one.mixin.messenger.DeviceTransferClient") + private let cacheContainerURL = FileManager.default.temporaryDirectory.appendingPathComponent("DeviceTransfer", isDirectory: true) private let speedInspector = NetworkSpeedInspector() - private let dataWriter: DeviceTransferDataWriter + private let messageProcessor: DeviceTransferMessageProcessor private weak var statisticsTimer: Timer? @@ -32,6 +34,8 @@ final class DeviceTransferClient { private var processedCount = 0 private var totalCount: Int? + private var messageProcessingObservers: Set = [] + private var opaquePointer: UnsafeMutableRawPointer { Unmanaged.passUnretained(self).toOpaque() } @@ -48,7 +52,9 @@ final class DeviceTransferClient { let endpoint = NWEndpoint.hostPort(host: host, port: port) return NWConnection(to: endpoint, using: .deviceTransfer) }() - self.dataWriter = DeviceTransferDataWriter(remotePlatform: remotePlatform) + self.messageProcessor = .init(remotePlatform: remotePlatform, + cacheContainerURL: cacheContainerURL, + inputQueue: queue) Logger.general.info(category: "DeviceTransferClient", message: "\(opaquePointer) init") } @@ -58,6 +64,25 @@ final class DeviceTransferClient { func start() { Logger.general.info(category: "DeviceTransferClient", message: "Will start connecting to [\(hostname)]:\(port)") + do { + let manager = FileManager.default + if manager.fileExists(atPath: cacheContainerURL.path) { + try? manager.removeItem(at: cacheContainerURL) + } + try manager.createDirectory(at: cacheContainerURL, withIntermediateDirectories: true) + } catch { + Logger.general.error(category: "DeviceTransferClient", message: "Failed to create cache container: \(error)") + fail(error: .createCacheContainer(error)) + return + } + messageProcessor.$processingError + .receive(on: queue.dispatchQueue) + .sink { error in + if let error { + self.fail(error: .importing(error)) + } + } + .store(in: &messageProcessingObservers) connection.stateUpdateHandler = { [weak self, unowned connection] state in switch state { case .setup: @@ -104,10 +129,11 @@ final class DeviceTransferClient { self.statisticsTimer?.invalidate() } connection.cancel() - DispatchQueue.main.async { - self.dataWriter.delegate = nil - self.dataWriter.canProcessData = false - } + messageProcessingObservers.forEach { $0.cancel() } + messageProcessingObservers.removeAll() + messageProcessor.cancel() + try? FileManager.default.removeItem(at: cacheContainerURL) + Logger.general.info(category: "DeviceTransferClient", message: "Cache container removed") state = .failed(error) } @@ -214,7 +240,6 @@ extension DeviceTransferClient { Logger.general.info(category: "DeviceTransferClient", message: "Total count: \(count)") self.state = .transfer(progress: 0, speed: "") DispatchQueue.main.async { - self.dataWriter.canProcessData = true self.totalCount = count self.startUpdatingProgressAndSpeed() } @@ -228,11 +253,21 @@ extension DeviceTransferClient { } catch { Logger.general.error(category: "DeviceTransferClient", message: "Failed to finish command: \(error)") } - DispatchQueue.main.async { - self.dataWriter.delegate = self - } - state = .importing(progress: 0) - dataWriter.transferFinished() + messageProcessor.$progress + .receive(on: queue.dispatchQueue) + .sink { progress in + if progress == 1 { + Logger.general.info(category: "DeviceTransferClient", message: "Import finished") + ConversationDAO.shared.updateLastMessageIdAndCreatedAt() + try? FileManager.default.removeItem(at: self.cacheContainerURL) + Logger.general.info(category: "DeviceTransferClient", message: "Cache container removed") + self.state = .finished + } else { + self.state = .importing(progress: progress) + } + } + .store(in: &messageProcessingObservers) + messageProcessor.finishProcessing() default: break } @@ -255,11 +290,9 @@ extension DeviceTransferClient { } do { let decryptedData = try AESCryptor.decrypt(encryptedData, with: key.aes) - if !dataWriter.write(data: decryptedData) { - fail(error: .unableSaveData) - } + try messageProcessor.process(message: decryptedData) } catch { - Logger.general.error(category: "DeviceTransferClient", message: "Unable to decrypt: \(error)") + Logger.general.error(category: "DeviceTransferClient", message: "Handle message: \(error)") return } } @@ -283,11 +316,11 @@ extension DeviceTransferClient { } catch { fail(error: .receiveFile(error)) } - stream = DeviceTransferFileStream(context: context, key: key) + stream = .init(context: context, key: key, containerURL: cacheContainerURL) isReceivingNewFile = true } } else { - stream = DeviceTransferFileStream(context: context, key: key) + stream = .init(context: context, key: key, containerURL: cacheContainerURL) isReceivingNewFile = true } if isReceivingNewFile { @@ -320,17 +353,3 @@ extension DeviceTransferClient { } } - -extension DeviceTransferClient: DeviceTransferDataWriterDelegate { - - func deviceTransferDataWriter(_ writer: DeviceTransferDataWriter, update progress: Float) { - if progress >= 1 { - Logger.general.info(category: "DeviceTransferClient", message: "Import finished") - ConversationDAO.shared.updateLastMessageIdAndCreatedAt() - state = .finished - } else { - state = .importing(progress: progress) - } - } - -} diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferData.swift b/Mixin/Service/DeviceTransfer/DeviceTransferData.swift deleted file mode 100644 index 9f5f144de4..0000000000 --- a/Mixin/Service/DeviceTransfer/DeviceTransferData.swift +++ /dev/null @@ -1,36 +0,0 @@ -import Foundation -import MixinServices - -enum DeviceTransferData: String { - - case record - case file - - static let payloadLength: UInt64 = 4 - static let maxSizePerFile = 10 * Int(bytesPerMegaByte) - - static func url() -> URL { - let url = FileManager.default.temporaryDirectory.appendingPathComponent("DeviceTransfer", isDirectory: true) - _ = try? FileManager.default.createDirectoryIfNotExists(at: url) - return url - } - - func url(index: Int?) -> URL { - if let index { - return url(name: "\(index)") - } else { - return url(name: nil) - } - } - - func url(name: String?) -> URL { - let url = Self.url().appendingPathComponent(self.rawValue, isDirectory: true) - _ = try? FileManager.default.createDirectoryIfNotExists(at: url) - if let name { - return url.appendingPathComponent("\(name).bin") - } else { - return url - } - } - -} diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift b/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift deleted file mode 100644 index dc90b281b5..0000000000 --- a/Mixin/Service/DeviceTransfer/DeviceTransferDataWriter.swift +++ /dev/null @@ -1,276 +0,0 @@ -import Foundation -import MixinServices - -protocol DeviceTransferDataWriterDelegate: AnyObject { - - func deviceTransferDataWriter(_ writer: DeviceTransferDataWriter, update progress: Float) - -} - -final class DeviceTransferDataWriter { - - weak var delegate: DeviceTransferDataWriterDelegate? - - var canProcessData: Bool = true { - didSet { - fileIndex = 0 - parsedRecordCount = 0 - totalRecordCount = 0 - fileHandle = nil - } - } - - private let remotePlatform: DeviceTransferPlatform - private let queue = Queue(label: "one.mixin.messenger.DeviceTransferClient.parse") - - private var fileHandle: FileHandle? - private var fileIndex = 0 - private var totalRecordCount = 0 - private var parsedRecordCount = 0 - private var pendingParsedRecordPath = [URL]() - private var pendingSaveMessages = [Message]() - - init(remotePlatform: DeviceTransferPlatform) { - self.remotePlatform = remotePlatform - } - - func write(data: Data) -> Bool { - if let fileHandle { - let fileSize = fileHandle.seekToEndOfFile() - let maxSizeExceeded = fileSize + UInt64(data.count) > DeviceTransferData.maxSizePerFile - if maxSizeExceeded { - fileHandle.closeFile() - let filePath = DeviceTransferData.record.url(index: fileIndex) - Logger.general.info(category: "DeviceTransferDataWriter", message: "Close file: \(filePath)") - queue.async { - self.pendingParsedRecordPath.append(filePath) - self.readAndParseRecordData() - } - fileIndex += 1 - openNextFile() - } - } else { - openNextFile() - } - guard let fileHandle else { - Logger.general.warn(category: "DeviceTransferDataWriter", message: "FileHandle is nil") - return false - } - let lenghtData = UInt32(data.count).data(endianness: .big) - fileHandle.write(lenghtData) - fileHandle.write(data) - DispatchQueue.main.async { - self.totalRecordCount += 1 - } - return true - } - - func transferFinished() { - fileHandle?.closeFile() - let filePath = DeviceTransferData.record.url(index: fileIndex) - Logger.general.info(category: "DeviceTransferDataWriter", message: "Close file: \(filePath)") - queue.async { - self.pendingParsedRecordPath.append(filePath) - self.readAndParseRecordData() - } - } - -} - -extension DeviceTransferDataWriter { - - private func openNextFile() { - let filePath = DeviceTransferData.record.url(index: fileIndex).path - if FileManager.default.fileExists(atPath: filePath) { - try? FileManager.default.removeItem(atPath: filePath) - } - FileManager.default.createFile(atPath: filePath, contents: nil, attributes: nil) - fileHandle = FileHandle(forUpdatingAtPath: filePath) - Logger.general.info(category: "DeviceTransferDataWriter", message: "Open file: \(filePath)") - } - - private func readAndParseRecordData() { - assert(queue.isCurrent) - guard !pendingParsedRecordPath.isEmpty else { - return - } - let filePath = pendingParsedRecordPath.removeFirst() - let fileHandle: FileHandle - do { - fileHandle = try FileHandle(forReadingFrom: filePath) - } catch { - Logger.general.error(category: "DeviceTransferDataWriter", message: "Reading from \(filePath) failed: \(error)") - return - } - Logger.general.info(category: "DeviceTransferDataWriter", message: "Parse file: \(filePath.path)") - let fileSize = fileHandle.seekToEndOfFile() - var offset: UInt64 = 0 - while offset < fileSize, canProcessData { - autoreleasepool { - fileHandle.seek(toFileOffset: offset) - let lengthData = fileHandle.readData(ofLength: Int(DeviceTransferData.payloadLength)) - let length = Int(Int32(data: lengthData, endianess: .big)) - offset += DeviceTransferData.payloadLength - fileHandle.seek(toFileOffset: offset) - let data = fileHandle.readData(ofLength: Int(length)) - offset += UInt64(length) - parseRecord(data: data) - DispatchQueue.main.async { - self.parsedRecordCount += 1 - guard let delegate = self.delegate else { - return - } - if self.parsedRecordCount >= self.totalRecordCount { - guard self.canProcessData else { - return - } - self.queue.async { - self.processFiles() - DispatchQueue.main.async { - delegate.deviceTransferDataWriter(self, update: 1) - } - } - } else { - let progress = Float(self.parsedRecordCount) / Float(self.totalRecordCount + 1) - delegate.deviceTransferDataWriter(self, update: progress) - } - } - } - } - fileHandle.closeFile() - try? FileManager.default.removeItem(at: filePath) - queue.async { - self.readAndParseRecordData() - } - } - -} - -extension DeviceTransferDataWriter { - - private func parseRecord(data: Data) { - assert(queue.isCurrent) - do { - struct TypeWrapper: Decodable { - let type: DeviceTransferRecordType - } - - let decoder = JSONDecoder.default - let wrapper = try decoder.decode(TypeWrapper.self, from: data) - switch wrapper.type { - case .conversation: - let conversation = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data - ConversationDAO.shared.save(conversation: conversation.toConversation(from: remotePlatform)) - case .participant: - let participant = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data - ParticipantDAO.shared.save(participant: participant.toParticipant()) - case .user: - let user = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data - UserDAO.shared.save(user: user.toUser()) - case .app: - let app = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data - AppDAO.shared.save(app: app.toApp()) - case .asset: - let asset = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data - AssetDAO.shared.save(asset: asset.toAsset()) - case .snapshot: - let snapshot = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data - SnapshotDAO.shared.save(snapshot: snapshot.toSnapshot()) - case .sticker: - let sticker = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data - StickerDAO.shared.save(sticker: sticker.toSticker()) - case .pinMessage: - let pinMessage = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data - PinMessageDAO.shared.save(pinMessage: pinMessage.toPinMessage()) - case .transcriptMessage: - let transcriptMessage = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data - TranscriptMessageDAO.shared.save(transcriptMessage: transcriptMessage.toTranscriptMessage()) - case .message: - let message = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data - if MessageCategory.isLegal(category: message.category) { - pendingSaveMessages.append(message.toMessage()) - if pendingSaveMessages.count > 1000 { - MessageDAO.shared.save(messages: pendingSaveMessages) - pendingSaveMessages.removeAll() - } - } else { - Logger.general.warn(category: "DeviceTransferDataWriter", message: "Message is illegal: \(message)") - } - case .messageMention: - saveMessagesIfNeeded() - let messageMention = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data - if let mention = messageMention.toMessageMention() { - MessageMentionDAO.shared.save(messageMention: mention) - } else { - Logger.general.warn(category: "DeviceTransferDataWriter", message: "Message Mention does not exist: \(messageMention)") - } - case .expiredMessage: - let expiredMessage = try decoder.decode(DeviceTransferTypedRecord.self, from: data).data - ExpiredMessageDAO.shared.save(expiredMessage: expiredMessage.toExpiredMessage()) - } - } catch { - let content = String(data: data, encoding: .utf8) ?? "Data(\(data.count))" - Logger.general.error(category: "DeviceTransferDataWriter", message: "Error: \(error) Content: \(content)") - } - } - - private func processFiles() { - assert(queue.isCurrent) - saveMessagesIfNeeded() - guard - let fileEnumerator = FileManager.default.enumerator(at: DeviceTransferData.file.url(name: nil), - includingPropertiesForKeys: [.isRegularFileKey], - options: [.skipsHiddenFiles, .skipsPackageDescendants]) - else { - Logger.general.error(category: "DeviceTransferDataWriter", message: "Can't create file enumerator") - return - } - Logger.general.info(category: "DeviceTransferDataWriter", message: "Start processing files") - let fileManager = FileManager.default - for case let fileURL as URL in fileEnumerator { - let components = fileURL.lastPathComponent.components(separatedBy: ".") - guard components.count == 2, let id = components.first else { - Logger.general.info(category: "DeviceTransferDataWriter", message: "Invalid file url: \(fileURL.path)") - continue - } - var destinationURLs: [URL] - if let message = MessageDAO.shared.getMessage(messageId: id), let mediaURL = message.mediaUrl { - guard let category = AttachmentContainer.Category(messageCategory: message.category) else { - Logger.general.error(category: "DeviceTransferDataWriter", message: "Invalid category: \(message.category)") - continue - } - let url = AttachmentContainer.url(for: category, filename: mediaURL) - destinationURLs = [url] - if let transcriptMessage = TranscriptMessageDAO.shared.transcriptMessage(messageId: id), let mediaURL = transcriptMessage.mediaUrl { - let url = AttachmentContainer.url(transcriptId: transcriptMessage.transcriptId, filename: mediaURL) - destinationURLs.append(url) - } - } else if let transcriptMessage = TranscriptMessageDAO.shared.transcriptMessage(messageId: id), let mediaURL = transcriptMessage.mediaUrl { - let url = AttachmentContainer.url(transcriptId: transcriptMessage.transcriptId, filename: mediaURL) - destinationURLs = [url] - } else { - Logger.general.warn(category: "DeviceTransferDataWriter", message: "No message found for: \(id)") - continue - } - for destinationURL in destinationURLs { - do { - if fileManager.fileExists(atPath: destinationURL.path) { - try fileManager.removeItem(at: destinationURL) - } - try fileManager.copyItem(at: fileURL, to: destinationURL) - } catch { - Logger.general.error(category: "DeviceTransferDataWriter", message: "\(id) move failed: \(error)") - } - } - } - try? FileManager.default.removeItem(atPath: DeviceTransferData.url().path) - } - - private func saveMessagesIfNeeded() { - if !pendingSaveMessages.isEmpty { - MessageDAO.shared.save(messages: pendingSaveMessages) - pendingSaveMessages.removeAll() - } - } - -} diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferError.swift b/Mixin/Service/DeviceTransfer/DeviceTransferError.swift index 198c3f196a..017e71847d 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferError.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferError.swift @@ -6,7 +6,7 @@ enum DeviceTransferError: Error { case encrypt(Error) case mismatchedHMAC(local: Data, remote: Data) case connectionFailed(Error) - case failed(Error) case receiveFile(Error) - case unableSaveData + case createCacheContainer(Error) + case importing(DeviceTransferMessageProcessor.ProcessingError) } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferFileStream.swift b/Mixin/Service/DeviceTransfer/DeviceTransferFileStream.swift index 97c6afb4eb..7498baa033 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferFileStream.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferFileStream.swift @@ -9,8 +9,8 @@ class DeviceTransferFileStream: InstanceInitializable { self.id = id } - convenience init(context: DeviceTransferProtocol.FileContext, key: DeviceTransferKey) { - if let impl = DeviceTransferFileStreamImpl(context, key: key) { + convenience init(context: DeviceTransferProtocol.FileContext, key: DeviceTransferKey, containerURL: URL) { + if let impl = DeviceTransferFileStreamImpl(context, key: key, containerURL: containerURL) { self.init(instance: impl as! Self) } else { self.init(id: context.fileHeader.id) @@ -37,7 +37,7 @@ fileprivate final class DeviceTransferFileStreamImpl: DeviceTransferFileStream { private var localHMAC: HMACSHA256 private var remoteHMAC = Data(capacity: DeviceTransferProtocol.hmacDataCount) - init?(_ context: DeviceTransferProtocol.FileContext, key: DeviceTransferKey) { + init?(_ context: DeviceTransferProtocol.FileContext, key: DeviceTransferKey, containerURL: URL) { let decryptor: AESCryptor do { decryptor = try AESCryptor(operation: .decrypt, iv: context.fileHeader.iv, key: key.aes) @@ -50,7 +50,7 @@ fileprivate final class DeviceTransferFileStreamImpl: DeviceTransferFileStream { let idData = context.fileHeader.id.data do { - let fileURL = DeviceTransferData.file.url(name: id) + let fileURL = containerURL.appendingPathComponent(id) if fileManager.fileExists(atPath: fileURL.path) { try fileManager.removeItem(at: fileURL) } @@ -115,7 +115,7 @@ fileprivate final class DeviceTransferFileStreamImpl: DeviceTransferFileStream { throw DeviceTransferError.mismatchedHMAC(local: localHMAC, remote: remoteHMAC) } #if DEBUG - Logger.general.info(category: "DeviceTransferFileStream", message: "\(id) Closed") + Logger.general.debug(category: "DeviceTransferFileStream", message: "\(id) Closed") #endif } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift new file mode 100644 index 0000000000..04b8e5496c --- /dev/null +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -0,0 +1,400 @@ +import Foundation +import MixinServices + +fileprivate let fileManager = FileManager.default + +final class DeviceTransferMessageProcessor { + + // This class is not thread safe to achieve best performance within a niche usage + // To avoid data racing, always call `process(message:)`, `finishProcessing` in strict order + + enum ProcessingError: Error { + case createInputStream + case readInputStream(Error?) + } + + private enum ReadStreamResult { + case success + case endOfStream + case operationFailed(Error?) + } + + private class Cache { + + typealias MessageLength = UInt32 + + static let maxCount = 10 * Int(bytesPerMegaByte) + static let messageLengthLayoutSize = 4 + + let index: UInt + let url: URL + let handle: FileHandle + + var wroteCount: Int = 0 + + var isOversized: Bool { + wroteCount >= Self.maxCount + } + + init(index: UInt, containerURL: URL) throws { + let url = containerURL.appendingPathComponent(String(index) + ".cache") + try Data().write(to: url) + self.index = index + self.url = url + self.handle = try FileHandle(forWritingTo: url) + } + + } + + @Published private(set) var progress: Float = 0 + @Published private(set) var processingError: ProcessingError? + + private let remotePlatform: DeviceTransferPlatform + private let cacheContainerURL: URL + private let inputQueue: Queue + private let processingQueue = Queue(label: "one.mixin.messenger.DeviceTransferMessageProcessor") + private let messageSavingBatchCount = 100 + private let progressReportingInterval = 10 // Update progress every 10 items are processed + + // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/pthread_rwlock_wrlock.3.html#//apple_ref/doc/man/3/pthread_rwlock_wrlock + // To prevent writer starvation, writers are favored over readers. + private var cancellationLock = pthread_rwlock_t() + private var _cancelled = false + private var isCancelled: Bool { + get { + pthread_rwlock_rdlock(&self.cancellationLock) + let cancelled = _cancelled + pthread_rwlock_unlock(&self.cancellationLock) + return cancelled + } + set { + pthread_rwlock_wrlock(&cancellationLock) + _cancelled = newValue + pthread_rwlock_unlock(&cancellationLock) + } + } + + private var totalCount = 0 + private var processedCount = 0 + + private var writingCache: Cache? + + private var cacheReadingBuffer = Data(count: Int(bytesPerKiloByte)) + + // Messages are saved to database in batch. See `messageSavingBatchCount` + private var pendingMessages: [Message] = [] + + init(remotePlatform: DeviceTransferPlatform, cacheContainerURL: URL, inputQueue: Queue) { + self.remotePlatform = remotePlatform + self.cacheContainerURL = cacheContainerURL + self.inputQueue = inputQueue + pthread_rwlock_init(&cancellationLock, nil) + } + + func process(message: Data) throws { + assert(inputQueue.isCurrent) + + let cache: Cache + if let writingCache { + cache = writingCache + } else { + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Create cache 0") + cache = try Cache(index: 0, containerURL: cacheContainerURL) + writingCache = cache + } + + let length = Cache.MessageLength(message.count).data(endianness: .little) + cache.handle.write(length) + cache.wroteCount += length.count + cache.handle.write(message) + cache.wroteCount += message.count + processingQueue.async { + self.totalCount += 1 + } + + if cache.isOversized { + cache.handle.closeFile() + processingQueue.async { + self.process(cache: cache) + try? fileManager.removeItem(at: cache.url) + } + let nextIndex = cache.index + 1 + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Create cache \(nextIndex)") + writingCache = try Cache(index: nextIndex, containerURL: cacheContainerURL) + } + } + + func finishProcessing() { + assert(inputQueue.isCurrent) + guard let lastCache = writingCache else { + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "All pending messages are saved") + return + } + writingCache = nil + lastCache.handle.closeFile() + processingQueue.async { + self.process(cache: lastCache) + try? fileManager.removeItem(at: lastCache.url) + if !self.pendingMessages.isEmpty { + if self.isCancelled { + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Cancelled on finish processing") + } else { + MessageDAO.shared.save(messages: self.pendingMessages) + self.pendingMessages.removeAll(keepingCapacity: false) + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "All pending messages are saved") + } + } + self.processFiles() + DispatchQueue.main.async { + self.progress = 1 + } + } + } + + func cancel() { + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Cancelled") + isCancelled = true + } + +} + +extension DeviceTransferMessageProcessor { + + private func reportProgress() { + assert(processingQueue.isCurrent) + let progress = Float(processedCount) / Float(totalCount) + DispatchQueue.main.async { + self.progress = progress + } + } + + private func read(from stream: InputStream, to buffer: inout Data, length: Int) -> ReadStreamResult { + var totalBytesRead = 0 + while totalBytesRead < length { + let bytesRead = buffer.withUnsafeMutableBytes { buffer in + let pointer = buffer.baseAddress!.advanced(by: totalBytesRead) + return stream.read(pointer, maxLength: length - totalBytesRead) + } + switch bytesRead { + case 0: + return .endOfStream + case -1: + return .operationFailed(stream.streamError) + default: + totalBytesRead += bytesRead + } + } + return .success + } + + private func process(cache: Cache) { + assert(processingQueue.isCurrent) + guard !isCancelled else { + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Not processing cache \(cache.index) by cancellation") + return + } + + guard let stream = InputStream(url: cache.url) else { + processingError = .createInputStream + return + } + stream.open() + defer { + stream.close() + } + + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Begin processing cache \(cache.index)") + var processedCountOnLastProgressReporting = self.processedCount + streamReadingLoop: + while stream.hasBytesAvailable { + guard !isCancelled else { + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Not processing cache \(cache.index) by cancellation") + return + } + + let length: Int + switch read(from: stream, to: &cacheReadingBuffer, length: Cache.messageLengthLayoutSize) { + case .endOfStream: + break streamReadingLoop + case .operationFailed(let error): + processingError = .readInputStream(error) + return + case .success: + length = Int(Cache.MessageLength(data: cacheReadingBuffer, endianess: .little)) + } + + if cacheReadingBuffer.count < length { + cacheReadingBuffer.count = length + } + switch read(from: stream, to: &cacheReadingBuffer, length: length) { + case .endOfStream: + assertionFailure("Impossible") + Logger.general.error(category: "DeviceTransferMessageProcessor", message: "EOS after length is read") + case .operationFailed(let error): + processingError = .readInputStream(error) + return + case .success: + let content = cacheReadingBuffer[cacheReadingBuffer.startIndex.. Int { + struct TypeWrapper: Decodable { + let type: DeviceTransferRecordType + } + struct DataWrapper: Decodable { + let data: Record + } + let decoder = JSONDecoder.default + do { + let type = try decoder.decode(TypeWrapper.self, from: jsonData).type + switch type { + case .conversation: + let conversation = try decoder.decode(DataWrapper.self, from: jsonData).data + ConversationDAO.shared.save(conversation: conversation.toConversation(from: remotePlatform)) + return 0 + case .participant: + let participant = try decoder.decode(DataWrapper.self, from: jsonData).data + ParticipantDAO.shared.save(participant: participant.toParticipant()) + return 0 + case .user: + let user = try decoder.decode(DataWrapper.self, from: jsonData).data + UserDAO.shared.save(user: user.toUser()) + return 0 + case .app: + let app = try decoder.decode(DataWrapper.self, from: jsonData).data + AppDAO.shared.save(app: app.toApp()) + return 0 + case .asset: + let asset = try decoder.decode(DataWrapper.self, from: jsonData).data + AssetDAO.shared.save(asset: asset.toAsset()) + return 0 + case .snapshot: + let snapshot = try decoder.decode(DataWrapper.self, from: jsonData).data + SnapshotDAO.shared.save(snapshot: snapshot.toSnapshot()) + return 0 + case .sticker: + let sticker = try decoder.decode(DataWrapper.self, from: jsonData).data + StickerDAO.shared.save(sticker: sticker.toSticker()) + return 0 + case .pinMessage: + let pinMessage = try decoder.decode(DataWrapper.self, from: jsonData).data + PinMessageDAO.shared.save(pinMessage: pinMessage.toPinMessage()) + return 0 + case .transcriptMessage: + let transcriptMessage = try decoder.decode(DataWrapper.self, from: jsonData).data + TranscriptMessageDAO.shared.save(transcriptMessage: transcriptMessage.toTranscriptMessage()) + if transcriptMessage.mediaUrl.isNilOrEmpty { + return 0 + } else { + return 1 + } + case .message: + let message = try decoder.decode(DataWrapper.self, from: jsonData).data + if MessageCategory.isLegal(category: message.category) { + pendingMessages.append(message.toMessage()) + if pendingMessages.count == messageSavingBatchCount { + MessageDAO.shared.save(messages: pendingMessages) + pendingMessages.removeAll(keepingCapacity: true) + } + if message.mediaUrl.isNilOrEmpty { + return 0 + } else { + return 1 + } + } else { + Logger.general.warn(category: "DeviceTransferMessageProcessor", message: "Message is illegal: \(message)") + return 0 + } + case .messageMention: + let messageMention = try decoder.decode(DataWrapper.self, from: jsonData).data + if let mention = messageMention.toMessageMention() { + MessageMentionDAO.shared.save(messageMention: mention) + } else { + Logger.general.warn(category: "DeviceTransferMessageProcessor", message: "Message Mention does not exist: \(messageMention)") + } + return 0 + case .expiredMessage: + let expiredMessage = try decoder.decode(DataWrapper.self, from: jsonData).data + ExpiredMessageDAO.shared.save(expiredMessage: expiredMessage.toExpiredMessage()) + return 0 + } + } catch { + let content = String(data: jsonData, encoding: .utf8) ?? "Data(\(jsonData.count))" + Logger.general.error(category: "DeviceTransferMessageProcessor", message: "Error: \(error) Content: \(content)") + return 0 + } + } + + private func processFiles() { + assert(processingQueue.isCurrent) + guard let fileEnumerator = fileManager.enumerator(at: cacheContainerURL, includingPropertiesForKeys: nil) else { + Logger.general.error(category: "DeviceTransferMessageProcessor", message: "Can't create file enumerator") + return + } + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Start processing files") + var processedCountOnLastProgressReporting = processedCount + + for case let fileURL as URL in fileEnumerator { + guard !isCancelled else { + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Stop processing files by cancellation") + return + } + let id = fileURL.lastPathComponent + + var destinationURLs: [URL] = [] + if let message = MessageDAO.shared.getMessage(messageId: id), + let mediaURL = message.mediaUrl, + let category = AttachmentContainer.Category(messageCategory: message.category) + { + let url = AttachmentContainer.url(for: category, filename: mediaURL) + destinationURLs = [url] + } else { + destinationURLs = [] + } + + if let transcriptMessage = TranscriptMessageDAO.shared.transcriptMessage(messageId: id), + let mediaURL = transcriptMessage.mediaUrl + { + let url = AttachmentContainer.url(transcriptId: transcriptMessage.transcriptId, filename: mediaURL) + destinationURLs.append(url) + } + + for destinationURL in destinationURLs { + do { + if fileManager.fileExists(atPath: destinationURL.path) { + try fileManager.removeItem(at: destinationURL) + } + try fileManager.copyItem(at: fileURL, to: destinationURL) + } catch { + Logger.general.error(category: "DeviceTransferMessageProcessor", message: "\(id) copy failed: \(error)") + } + } + + processedCount += destinationURLs.count + if processedCount - processedCountOnLastProgressReporting == progressReportingInterval { + processedCountOnLastProgressReporting = processedCount + reportProgress() + } + + try? fileManager.removeItem(at: fileURL) + } + } + +} diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift index 564d210594..c18cdf5a74 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift @@ -208,6 +208,13 @@ extension DeviceTransferProgressViewController { speedLabel.text = speed } + private func updateTitleLabel(with importProgress: Float) { + tipLabel.text = R.string.localizable.keep_running_foreground() + titleLabel.text = R.string.localizable.importing_chat_progress(String(format: "%.2f", importProgress * 100)) + progressView.progress = importProgress + speedLabel.isHidden = true + } + private func handleConnectionClosing(error: DeviceTransferError) { let hint = R.string.localizable.transfer_failed() titleLabel.text = hint @@ -217,13 +224,6 @@ extension DeviceTransferProgressViewController { Logger.general.error(category: "DeviceTransferProgress", message: "Transfer failed: \(error)") } - private func updateTitleLabel(with importProgress: Float) { - speedLabel.isHidden = true - tipLabel.text = R.string.localizable.keep_running_foreground() - titleLabel.text = R.string.localizable.importing_chat_progress(String(format: "%.2f", importProgress * 100)) - progressView.progress = importProgress - } - private func importFinished() { let hint = R.string.localizable.restore_completed() titleLabel.text = hint From 13679a683090d86d13304db91c16ce594f525c7a Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Fri, 9 Jun 2023 13:24:07 +0800 Subject: [PATCH 08/23] Fix unable to resolve MessageMention with absent Messages --- .../DeviceTransferMessageProcessor.swift | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift index 04b8e5496c..dcdfca0a8e 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -135,15 +135,6 @@ final class DeviceTransferMessageProcessor { processingQueue.async { self.process(cache: lastCache) try? fileManager.removeItem(at: lastCache.url) - if !self.pendingMessages.isEmpty { - if self.isCancelled { - Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Cancelled on finish processing") - } else { - MessageDAO.shared.save(messages: self.pendingMessages) - self.pendingMessages.removeAll(keepingCapacity: false) - Logger.general.info(category: "DeviceTransferMessageProcessor", message: "All pending messages are saved") - } - } self.processFiles() DispatchQueue.main.async { self.progress = 1 @@ -323,6 +314,11 @@ extension DeviceTransferMessageProcessor { return 0 } case .messageMention: + if !pendingMessages.isEmpty { + MessageDAO.shared.save(messages: pendingMessages) + pendingMessages.removeAll(keepingCapacity: false) + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "All pending messages are saved") + } let messageMention = try decoder.decode(DataWrapper.self, from: jsonData).data if let mention = messageMention.toMessageMention() { MessageMentionDAO.shared.save(messageMention: mention) From c1cbe76c4385c628f32830baeae60bf62b637a05 Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Fri, 9 Jun 2023 13:28:13 +0800 Subject: [PATCH 09/23] Improve accuracy of progress --- .../DeviceTransfer/DeviceTransferClient.swift | 1 + .../DeviceTransferMessageProcessor.swift | 36 +++++-------------- 2 files changed, 10 insertions(+), 27 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index d055f9d523..df49bb6dcc 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -349,6 +349,7 @@ extension DeviceTransferClient { fail(error: .receiveFile(error)) } self.fileStream = nil + messageProcessor.reportFileReceived() } } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift index dcdfca0a8e..89c158ada7 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -124,6 +124,12 @@ final class DeviceTransferMessageProcessor { } } + func reportFileReceived() { + processingQueue.async { + self.totalCount += 1 + } + } + func finishProcessing() { assert(inputQueue.isCurrent) guard let lastCache = writingCache else { @@ -226,8 +232,7 @@ extension DeviceTransferMessageProcessor { return case .success: let content = cacheReadingBuffer[cacheReadingBuffer.startIndex.. Int { + private func process(jsonData: Data) { struct TypeWrapper: Decodable { let type: DeviceTransferRecordType } @@ -259,43 +263,30 @@ extension DeviceTransferMessageProcessor { case .conversation: let conversation = try decoder.decode(DataWrapper.self, from: jsonData).data ConversationDAO.shared.save(conversation: conversation.toConversation(from: remotePlatform)) - return 0 case .participant: let participant = try decoder.decode(DataWrapper.self, from: jsonData).data ParticipantDAO.shared.save(participant: participant.toParticipant()) - return 0 case .user: let user = try decoder.decode(DataWrapper.self, from: jsonData).data UserDAO.shared.save(user: user.toUser()) - return 0 case .app: let app = try decoder.decode(DataWrapper.self, from: jsonData).data AppDAO.shared.save(app: app.toApp()) - return 0 case .asset: let asset = try decoder.decode(DataWrapper.self, from: jsonData).data AssetDAO.shared.save(asset: asset.toAsset()) - return 0 case .snapshot: let snapshot = try decoder.decode(DataWrapper.self, from: jsonData).data SnapshotDAO.shared.save(snapshot: snapshot.toSnapshot()) - return 0 case .sticker: let sticker = try decoder.decode(DataWrapper.self, from: jsonData).data StickerDAO.shared.save(sticker: sticker.toSticker()) - return 0 case .pinMessage: let pinMessage = try decoder.decode(DataWrapper.self, from: jsonData).data PinMessageDAO.shared.save(pinMessage: pinMessage.toPinMessage()) - return 0 case .transcriptMessage: let transcriptMessage = try decoder.decode(DataWrapper.self, from: jsonData).data TranscriptMessageDAO.shared.save(transcriptMessage: transcriptMessage.toTranscriptMessage()) - if transcriptMessage.mediaUrl.isNilOrEmpty { - return 0 - } else { - return 1 - } case .message: let message = try decoder.decode(DataWrapper.self, from: jsonData).data if MessageCategory.isLegal(category: message.category) { @@ -304,14 +295,8 @@ extension DeviceTransferMessageProcessor { MessageDAO.shared.save(messages: pendingMessages) pendingMessages.removeAll(keepingCapacity: true) } - if message.mediaUrl.isNilOrEmpty { - return 0 - } else { - return 1 - } } else { Logger.general.warn(category: "DeviceTransferMessageProcessor", message: "Message is illegal: \(message)") - return 0 } case .messageMention: if !pendingMessages.isEmpty { @@ -325,16 +310,13 @@ extension DeviceTransferMessageProcessor { } else { Logger.general.warn(category: "DeviceTransferMessageProcessor", message: "Message Mention does not exist: \(messageMention)") } - return 0 case .expiredMessage: let expiredMessage = try decoder.decode(DataWrapper.self, from: jsonData).data ExpiredMessageDAO.shared.save(expiredMessage: expiredMessage.toExpiredMessage()) - return 0 } } catch { let content = String(data: jsonData, encoding: .utf8) ?? "Data(\(jsonData.count))" Logger.general.error(category: "DeviceTransferMessageProcessor", message: "Error: \(error) Content: \(content)") - return 0 } } @@ -383,7 +365,7 @@ extension DeviceTransferMessageProcessor { } } - processedCount += destinationURLs.count + processedCount += 1 if processedCount - processedCountOnLastProgressReporting == progressReportingInterval { processedCountOnLastProgressReporting = processedCount reportProgress() From 8858bf659c9a25418a4b332e9e267a3a2aa6a7a0 Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Fri, 9 Jun 2023 13:34:43 +0800 Subject: [PATCH 10/23] Report error if failed to create a file enumerator --- .../Service/DeviceTransfer/DeviceTransferMessageProcessor.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift index 89c158ada7..a7648d7754 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -11,6 +11,7 @@ final class DeviceTransferMessageProcessor { enum ProcessingError: Error { case createInputStream case readInputStream(Error?) + case enumerateFiles } private enum ReadStreamResult { @@ -323,6 +324,7 @@ extension DeviceTransferMessageProcessor { private func processFiles() { assert(processingQueue.isCurrent) guard let fileEnumerator = fileManager.enumerator(at: cacheContainerURL, includingPropertiesForKeys: nil) else { + processingError = .enumerateFiles Logger.general.error(category: "DeviceTransferMessageProcessor", message: "Can't create file enumerator") return } From 8dbc97436c8ddc18866dc1c9b2f42d84bcb7e132 Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Fri, 9 Jun 2023 14:04:22 +0800 Subject: [PATCH 11/23] Fix pending messages not saved if there's no MessageMention --- .../DeviceTransferMessageProcessor.swift | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift index a7648d7754..ab372ceaeb 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -142,6 +142,7 @@ final class DeviceTransferMessageProcessor { processingQueue.async { self.process(cache: lastCache) try? fileManager.removeItem(at: lastCache.url) + self.savePendingMessages() self.processFiles() DispatchQueue.main.async { self.progress = 1 @@ -166,6 +167,15 @@ extension DeviceTransferMessageProcessor { } } + private func savePendingMessages() { + guard !pendingMessages.isEmpty else { + return + } + MessageDAO.shared.save(messages: pendingMessages) + pendingMessages.removeAll(keepingCapacity: false) + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "All pending messages are saved") + } + private func read(from stream: InputStream, to buffer: inout Data, length: Int) -> ReadStreamResult { var totalBytesRead = 0 while totalBytesRead < length { @@ -300,11 +310,7 @@ extension DeviceTransferMessageProcessor { Logger.general.warn(category: "DeviceTransferMessageProcessor", message: "Message is illegal: \(message)") } case .messageMention: - if !pendingMessages.isEmpty { - MessageDAO.shared.save(messages: pendingMessages) - pendingMessages.removeAll(keepingCapacity: false) - Logger.general.info(category: "DeviceTransferMessageProcessor", message: "All pending messages are saved") - } + savePendingMessages() let messageMention = try decoder.decode(DataWrapper.self, from: jsonData).data if let mention = messageMention.toMessageMention() { MessageMentionDAO.shared.save(messageMention: mention) From 2625d392cef5d1a3ce2456093d74f33fe07b97dc Mon Sep 17 00:00:00 2001 From: fanyu Date: Mon, 12 Jun 2023 13:51:50 +0800 Subject: [PATCH 12/23] Decrypt data when processing cache --- .../DeviceTransfer/DeviceTransferClient.swift | 6 +++--- .../DeviceTransferMessageProcessor.swift | 13 ++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index df49bb6dcc..8563717fdd 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -54,7 +54,8 @@ final class DeviceTransferClient { }() self.messageProcessor = .init(remotePlatform: remotePlatform, cacheContainerURL: cacheContainerURL, - inputQueue: queue) + inputQueue: queue, + key: key.aes) Logger.general.info(category: "DeviceTransferClient", message: "\(opaquePointer) init") } @@ -289,8 +290,7 @@ extension DeviceTransferClient { return } do { - let decryptedData = try AESCryptor.decrypt(encryptedData, with: key.aes) - try messageProcessor.process(message: decryptedData) + try messageProcessor.process(message: encryptedData) } catch { Logger.general.error(category: "DeviceTransferClient", message: "Handle message: \(error)") return diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift index ab372ceaeb..369d661077 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -53,6 +53,7 @@ final class DeviceTransferMessageProcessor { private let remotePlatform: DeviceTransferPlatform private let cacheContainerURL: URL private let inputQueue: Queue + private let key: Data private let processingQueue = Queue(label: "one.mixin.messenger.DeviceTransferMessageProcessor") private let messageSavingBatchCount = 100 private let progressReportingInterval = 10 // Update progress every 10 items are processed @@ -85,10 +86,11 @@ final class DeviceTransferMessageProcessor { // Messages are saved to database in batch. See `messageSavingBatchCount` private var pendingMessages: [Message] = [] - init(remotePlatform: DeviceTransferPlatform, cacheContainerURL: URL, inputQueue: Queue) { + init(remotePlatform: DeviceTransferPlatform, cacheContainerURL: URL, inputQueue: Queue, key: Data) { self.remotePlatform = remotePlatform self.cacheContainerURL = cacheContainerURL self.inputQueue = inputQueue + self.key = key pthread_rwlock_init(&cancellationLock, nil) } @@ -242,8 +244,13 @@ extension DeviceTransferMessageProcessor { processingError = .readInputStream(error) return case .success: - let content = cacheReadingBuffer[cacheReadingBuffer.startIndex.. Date: Mon, 12 Jun 2023 14:29:37 +0800 Subject: [PATCH 13/23] Better index --- MixinServices/MixinServices/Crypto/AESCryptor.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/MixinServices/MixinServices/Crypto/AESCryptor.swift b/MixinServices/MixinServices/Crypto/AESCryptor.swift index 1c33a97dcb..277762a6ed 100644 --- a/MixinServices/MixinServices/Crypto/AESCryptor.swift +++ b/MixinServices/MixinServices/Crypto/AESCryptor.swift @@ -126,10 +126,11 @@ extension AESCryptor { guard ivPlusCipher.count > ivSize else { throw Error.badInput } - return try crypt(input: ivPlusCipher[ivSize...], + let firstCipherIndex = ivPlusCipher.startIndex.advanced(by: ivSize) + return try crypt(input: ivPlusCipher[firstCipherIndex...], operation: CCOperation(kCCDecrypt), key: key, - iv: ivPlusCipher[0.. Data { From 666423b834023d238be1a1eea0379d9f5da6a562 Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Mon, 12 Jun 2023 14:43:11 +0800 Subject: [PATCH 14/23] Check read length --- .../DeviceTransferMessageProcessor.swift | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift index 369d661077..d004352f8b 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -11,11 +11,12 @@ final class DeviceTransferMessageProcessor { enum ProcessingError: Error { case createInputStream case readInputStream(Error?) + case mismatchedLengthRead(required: Int, read: Int) case enumerateFiles } private enum ReadStreamResult { - case success + case success(Int) case endOfStream case operationFailed(Error?) } @@ -194,7 +195,7 @@ extension DeviceTransferMessageProcessor { totalBytesRead += bytesRead } } - return .success + return .success(totalBytesRead) } private func process(cache: Cache) { @@ -222,7 +223,7 @@ extension DeviceTransferMessageProcessor { return } - let length: Int + let requiredLength: Int switch read(from: stream, to: &cacheReadingBuffer, length: Cache.messageLengthLayoutSize) { case .endOfStream: break streamReadingLoop @@ -230,21 +231,26 @@ extension DeviceTransferMessageProcessor { processingError = .readInputStream(error) return case .success: - length = Int(Cache.MessageLength(data: cacheReadingBuffer, endianess: .little)) + requiredLength = Int(Cache.MessageLength(data: cacheReadingBuffer, endianess: .little)) } - if cacheReadingBuffer.count < length { - cacheReadingBuffer.count = length + if cacheReadingBuffer.count < requiredLength { + cacheReadingBuffer.count = requiredLength } - switch read(from: stream, to: &cacheReadingBuffer, length: length) { + switch read(from: stream, to: &cacheReadingBuffer, length: requiredLength) { case .endOfStream: assertionFailure("Impossible") Logger.general.error(category: "DeviceTransferMessageProcessor", message: "EOS after length is read") case .operationFailed(let error): processingError = .readInputStream(error) return - case .success: - let encryptedData = cacheReadingBuffer[cacheReadingBuffer.startIndex.. Date: Mon, 12 Jun 2023 16:04:49 +0800 Subject: [PATCH 15/23] Apply code style --- .../DeviceTransfer/DeviceTransferClient.swift | 8 ++++---- .../DeviceTransferMessageProcessor.swift | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index 8563717fdd..8e734b4e59 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -52,10 +52,10 @@ final class DeviceTransferClient { let endpoint = NWEndpoint.hostPort(host: host, port: port) return NWConnection(to: endpoint, using: .deviceTransfer) }() - self.messageProcessor = .init(remotePlatform: remotePlatform, + self.messageProcessor = .init(key: key.aes, + remotePlatform: remotePlatform, cacheContainerURL: cacheContainerURL, - inputQueue: queue, - key: key.aes) + inputQueue: queue) Logger.general.info(category: "DeviceTransferClient", message: "\(opaquePointer) init") } @@ -290,7 +290,7 @@ extension DeviceTransferClient { return } do { - try messageProcessor.process(message: encryptedData) + try messageProcessor.process(encryptedMessage: encryptedData) } catch { Logger.general.error(category: "DeviceTransferClient", message: "Handle message: \(error)") return diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift index d004352f8b..d69aedf022 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -51,10 +51,10 @@ final class DeviceTransferMessageProcessor { @Published private(set) var progress: Float = 0 @Published private(set) var processingError: ProcessingError? + private let key: Data private let remotePlatform: DeviceTransferPlatform private let cacheContainerURL: URL private let inputQueue: Queue - private let key: Data private let processingQueue = Queue(label: "one.mixin.messenger.DeviceTransferMessageProcessor") private let messageSavingBatchCount = 100 private let progressReportingInterval = 10 // Update progress every 10 items are processed @@ -87,15 +87,15 @@ final class DeviceTransferMessageProcessor { // Messages are saved to database in batch. See `messageSavingBatchCount` private var pendingMessages: [Message] = [] - init(remotePlatform: DeviceTransferPlatform, cacheContainerURL: URL, inputQueue: Queue, key: Data) { + init(key: Data, remotePlatform: DeviceTransferPlatform, cacheContainerURL: URL, inputQueue: Queue) { + self.key = key self.remotePlatform = remotePlatform self.cacheContainerURL = cacheContainerURL self.inputQueue = inputQueue - self.key = key pthread_rwlock_init(&cancellationLock, nil) } - func process(message: Data) throws { + func process(encryptedMessage: Data) throws { assert(inputQueue.isCurrent) let cache: Cache @@ -107,11 +107,11 @@ final class DeviceTransferMessageProcessor { writingCache = cache } - let length = Cache.MessageLength(message.count).data(endianness: .little) + let length = Cache.MessageLength(encryptedMessage.count).data(endianness: .little) cache.handle.write(length) cache.wroteCount += length.count - cache.handle.write(message) - cache.wroteCount += message.count + cache.handle.write(encryptedMessage) + cache.wroteCount += encryptedMessage.count processingQueue.async { self.totalCount += 1 } From faa78b6e5d2e9809a005e6d17a8c2bc6f9636721 Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Tue, 13 Jun 2023 16:42:33 +0800 Subject: [PATCH 16/23] Divide the count as integer to prevent progress from rounding when counts are large --- .../DeviceTransferMessageProcessor.swift | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift index d69aedf022..bbabc6a549 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -147,9 +147,7 @@ final class DeviceTransferMessageProcessor { try? fileManager.removeItem(at: lastCache.url) self.savePendingMessages() self.processFiles() - DispatchQueue.main.async { - self.progress = 1 - } + self.progress = 1 } } @@ -164,10 +162,9 @@ extension DeviceTransferMessageProcessor { private func reportProgress() { assert(processingQueue.isCurrent) - let progress = Float(processedCount) / Float(totalCount) - DispatchQueue.main.async { - self.progress = progress - } + // Divide the count as integer to prevent `progress` from rounding when counts are large + let progress = Float(processedCount * 100 / totalCount) / 100 + self.progress = progress } private func savePendingMessages() { From 3b03bff57c3dfb3b3627db9c1087bfc2832cb9a5 Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Tue, 13 Jun 2023 16:51:40 +0800 Subject: [PATCH 17/23] Apply code style --- MixinServices/MixinServices/Crypto/AESCryptor.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MixinServices/MixinServices/Crypto/AESCryptor.swift b/MixinServices/MixinServices/Crypto/AESCryptor.swift index 277762a6ed..1016e2be83 100644 --- a/MixinServices/MixinServices/Crypto/AESCryptor.swift +++ b/MixinServices/MixinServices/Crypto/AESCryptor.swift @@ -130,7 +130,7 @@ extension AESCryptor { return try crypt(input: ivPlusCipher[firstCipherIndex...], operation: CCOperation(kCCDecrypt), key: key, - iv: ivPlusCipher[ivPlusCipher.startIndex.. Data { From 5ac416f69e982860ac054a5c5becfbb96b17734c Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Wed, 14 Jun 2023 15:16:39 +0800 Subject: [PATCH 18/23] Apply code style --- .../DeviceTransferMessageProcessor.swift | 78 +++++++++++-------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift index bbabc6a549..450868b9d8 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -15,12 +15,6 @@ final class DeviceTransferMessageProcessor { case enumerateFiles } - private enum ReadStreamResult { - case success(Int) - case endOfStream - case operationFailed(Error?) - } - private class Cache { typealias MessageLength = UInt32 @@ -158,6 +152,7 @@ final class DeviceTransferMessageProcessor { } +// MARK: - Cache Processing extension DeviceTransferMessageProcessor { private func reportProgress() { @@ -176,25 +171,6 @@ extension DeviceTransferMessageProcessor { Logger.general.info(category: "DeviceTransferMessageProcessor", message: "All pending messages are saved") } - private func read(from stream: InputStream, to buffer: inout Data, length: Int) -> ReadStreamResult { - var totalBytesRead = 0 - while totalBytesRead < length { - let bytesRead = buffer.withUnsafeMutableBytes { buffer in - let pointer = buffer.baseAddress!.advanced(by: totalBytesRead) - return stream.read(pointer, maxLength: length - totalBytesRead) - } - switch bytesRead { - case 0: - return .endOfStream - case -1: - return .operationFailed(stream.streamError) - default: - totalBytesRead += bytesRead - } - } - return .success(totalBytesRead) - } - private func process(cache: Cache) { assert(processingQueue.isCurrent) guard !isCancelled else { @@ -213,7 +189,6 @@ extension DeviceTransferMessageProcessor { Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Begin processing cache \(cache.index)") var processedCountOnLastProgressReporting = self.processedCount - streamReadingLoop: while stream.hasBytesAvailable { guard !isCancelled else { Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Not processing cache \(cache.index) by cancellation") @@ -223,7 +198,13 @@ extension DeviceTransferMessageProcessor { let requiredLength: Int switch read(from: stream, to: &cacheReadingBuffer, length: Cache.messageLengthLayoutSize) { case .endOfStream: - break streamReadingLoop + if isCancelled { + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "End processing cache \(cache.index) with cancellation") + } else { + reportProgress() + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "End processing cache \(cache.index)") + } + return case .operationFailed(let error): processingError = .readInputStream(error) return @@ -238,6 +219,7 @@ extension DeviceTransferMessageProcessor { case .endOfStream: assertionFailure("Impossible") Logger.general.error(category: "DeviceTransferMessageProcessor", message: "EOS after length is read") + return case .operationFailed(let error): processingError = .readInputStream(error) return @@ -245,7 +227,7 @@ extension DeviceTransferMessageProcessor { guard requiredLength == readLength else { Logger.general.error(category: "DeviceTransferMessageProcessor", message: "Error reading: \(readLength), required: \(requiredLength)") processingError = .mismatchedLengthRead(required: requiredLength, read: readLength) - break streamReadingLoop + return } let encryptedData = cacheReadingBuffer[.. ReadStreamResult { + var totalBytesRead = 0 + while totalBytesRead < length { + let bytesRead = buffer.withUnsafeMutableBytes { buffer in + let pointer = buffer.baseAddress!.advanced(by: totalBytesRead) + return stream.read(pointer, maxLength: length - totalBytesRead) + } + switch bytesRead { + case 0: + return .endOfStream + case -1: + return .operationFailed(stream.streamError) + default: + totalBytesRead += bytesRead + } } - reportProgress() - Logger.general.info(category: "DeviceTransferMessageProcessor", message: "End processing cache \(cache.index)") + return .success(totalBytesRead) } +} + +// MARK: - Data Processing +extension DeviceTransferMessageProcessor { + private func process(jsonData: Data) { struct TypeWrapper: Decodable { let type: DeviceTransferRecordType From 2094bb51d11ca89132766e83a5c1229ac8784423 Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Wed, 14 Jun 2023 15:25:44 +0800 Subject: [PATCH 19/23] Fix expired messages not matching messages when mentions are empty --- .../Service/DeviceTransfer/DeviceTransferMessageProcessor.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift index 450868b9d8..b352acebd6 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -338,6 +338,7 @@ extension DeviceTransferMessageProcessor { Logger.general.warn(category: "DeviceTransferMessageProcessor", message: "Message Mention does not exist: \(messageMention)") } case .expiredMessage: + savePendingMessages() let expiredMessage = try decoder.decode(DataWrapper.self, from: jsonData).data ExpiredMessageDAO.shared.save(expiredMessage: expiredMessage.toExpiredMessage()) } From b2683dc74b30bd1a414dbc640bcd67aee70624ab Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Wed, 14 Jun 2023 17:04:49 +0800 Subject: [PATCH 20/23] Prevent system from deleting caches while app is running --- .../DeviceTransfer/DeviceTransferClient.swift | 29 ++++++++++--------- .../DeviceTransfer/DeviceTransferError.swift | 1 - .../RestoreFromDesktopViewController.swift | 27 ++++++++++------- Mixin/UserInterface/Windows/UrlWindow.swift | 27 ++++++++++------- 4 files changed, 48 insertions(+), 36 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index 8e734b4e59..1bc119ecdd 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -21,10 +21,10 @@ final class DeviceTransferClient { private let key: DeviceTransferKey private let remotePlatform: DeviceTransferPlatform private let connection: NWConnection + private let cacheContainerURL: URL + private let messageProcessor: DeviceTransferMessageProcessor private let queue = Queue(label: "one.mixin.messenger.DeviceTransferClient") - private let cacheContainerURL = FileManager.default.temporaryDirectory.appendingPathComponent("DeviceTransfer", isDirectory: true) private let speedInspector = NetworkSpeedInspector() - private let messageProcessor: DeviceTransferMessageProcessor private weak var statisticsTimer: Timer? @@ -40,7 +40,18 @@ final class DeviceTransferClient { Unmanaged.passUnretained(self).toOpaque() } - init(hostname: String, port: UInt16, code: UInt16, key: DeviceTransferKey, remotePlatform: DeviceTransferPlatform) { + init(hostname: String, port: UInt16, code: UInt16, key: DeviceTransferKey, remotePlatform: DeviceTransferPlatform) throws { + // https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW2 + // In iOS 5.0 and later, the system may delete the Caches directory on rare occasions when the system is very low on disk space. + // This will never occur while an app is running. + let manager = FileManager.default + let cacheContainerURL = try manager.url(for: .cachesDirectory, in: .userDomainMask, appropriateFor: nil, create: true) + .appendingPathComponent("DeviceTransfer") + if manager.fileExists(atPath: cacheContainerURL.path) { + try? manager.removeItem(at: cacheContainerURL) + } + try manager.createDirectory(at: cacheContainerURL, withIntermediateDirectories: true) + self.hostname = hostname self.port = port self.code = code @@ -52,6 +63,7 @@ final class DeviceTransferClient { let endpoint = NWEndpoint.hostPort(host: host, port: port) return NWConnection(to: endpoint, using: .deviceTransfer) }() + self.cacheContainerURL = cacheContainerURL self.messageProcessor = .init(key: key.aes, remotePlatform: remotePlatform, cacheContainerURL: cacheContainerURL, @@ -65,17 +77,6 @@ final class DeviceTransferClient { func start() { Logger.general.info(category: "DeviceTransferClient", message: "Will start connecting to [\(hostname)]:\(port)") - do { - let manager = FileManager.default - if manager.fileExists(atPath: cacheContainerURL.path) { - try? manager.removeItem(at: cacheContainerURL) - } - try manager.createDirectory(at: cacheContainerURL, withIntermediateDirectories: true) - } catch { - Logger.general.error(category: "DeviceTransferClient", message: "Failed to create cache container: \(error)") - fail(error: .createCacheContainer(error)) - return - } messageProcessor.$processingError .receive(on: queue.dispatchQueue) .sink { error in diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferError.swift b/Mixin/Service/DeviceTransfer/DeviceTransferError.swift index 017e71847d..5af46cc258 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferError.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferError.swift @@ -7,6 +7,5 @@ enum DeviceTransferError: Error { case mismatchedHMAC(local: Data, remote: Data) case connectionFailed(Error) case receiveFile(Error) - case createCacheContainer(Error) case importing(DeviceTransferMessageProcessor.ProcessingError) } diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift index a7957e9790..45c23b48e5 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/RestoreFromDesktopViewController.swift @@ -119,17 +119,24 @@ extension RestoreFromDesktopViewController { dataSource.replaceSection(at: 0, with: section, animation: .automatic) tableView.isUserInteractionEnabled = true case let .push(context): - let client = DeviceTransferClient(hostname: context.hostname, - port: context.port, - code: context.code, - key: context.key, - remotePlatform: command.platform) - stateObserver = client.$state - .receive(on: DispatchQueue.main) - .sink { [weak self] state in - self?.stateDidChange(client: client, state: state) + do { + let client = try DeviceTransferClient(hostname: context.hostname, + port: context.port, + code: context.code, + key: context.key, + remotePlatform: command.platform) + stateObserver = client.$state + .receive(on: DispatchQueue.main) + .sink { [weak self] state in + self?.stateDidChange(client: client, state: state) + } + client.start() + } catch { + Logger.general.error(category: "RestoreFromDesktop", message: "Unable to init client: \(error)") + alert(R.string.localizable.connection_establishment_failed(), message: nil) { _ in + self.navigationController?.popViewController(animated: true) } - client.start() + } default: Logger.general.info(category: "RestoreFromDesktop", message: "Invalid command") alert(R.string.localizable.connection_establishment_failed(), message: nil) { _ in diff --git a/Mixin/UserInterface/Windows/UrlWindow.swift b/Mixin/UserInterface/Windows/UrlWindow.swift index 2697ea8258..2de79dad94 100644 --- a/Mixin/UserInterface/Windows/UrlWindow.swift +++ b/Mixin/UserInterface/Windows/UrlWindow.swift @@ -888,17 +888,22 @@ class UrlWindow { UIApplication.currentActivity()?.alert(R.string.localizable.unable_synced_between_different_account()) return true } - let client = DeviceTransferClient(hostname: context.hostname, - port: context.port, - code: context.code, - key: context.key, - remotePlatform: command.platform) - client.start() - let progress = DeviceTransferProgressViewController(connection: .client(client, .phone)) - if let navigationController = AppDelegate.current.mainWindow.rootViewController as? UINavigationController { - navigationController.pushViewController(progress, animated: true) - } else { - AppDelegate.current.mainWindow.rootViewController?.present(progress, animated: true) + do { + let client = try DeviceTransferClient(hostname: context.hostname, + port: context.port, + code: context.code, + key: context.key, + remotePlatform: command.platform) + client.start() + let progress = DeviceTransferProgressViewController(connection: .client(client, .phone)) + if let navigationController = AppDelegate.current.mainWindow.rootViewController as? UINavigationController { + navigationController.pushViewController(progress, animated: true) + } else { + AppDelegate.current.mainWindow.rootViewController?.present(progress, animated: true) + } + } catch { + Logger.general.error(category: "UrlWindow", message: "Unable to init client: \(error)") + UIApplication.currentActivity()?.alert(R.string.localizable.connection_establishment_failed()) } return true } From b150f10fb9bcfd2c4453785bbc9b65db59c5900f Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Wed, 14 Jun 2023 22:33:49 +0800 Subject: [PATCH 21/23] Better progress reporting --- Mixin.xcodeproj/project.pbxproj | 4 +++ .../DeviceTransfer/DeviceTransferClient.swift | 31 +++++++--------- .../DeviceTransferMessageProcessor.swift | 32 ++++++++--------- .../DeviceTransfer/DeviceTransferServer.swift | 2 +- .../DeviceTransferServerDataSource.swift | 10 +++--- .../Utility/DeviceTransferProgress.swift | 36 +++++++++++++++++++ ...DeviceTransferProgressViewController.swift | 6 ++-- .../Database/User/DAO/AppDAO.swift | 4 +-- .../Database/User/DAO/AssetDAO.swift | 4 +-- .../Database/User/DAO/ConversationDAO.swift | 4 +-- .../Database/User/DAO/ExpiredMessageDAO.swift | 4 +-- .../Database/User/DAO/MessageDAO.swift | 4 +-- .../Database/User/DAO/MessageMentionDAO.swift | 4 +-- .../Database/User/DAO/ParticipantDAO.swift | 4 +-- .../Database/User/DAO/PinMessageDAO.swift | 4 +-- .../Database/User/DAO/SnapshotDAO.swift | 4 +-- .../Database/User/DAO/StickerDAO.swift | 4 +-- .../User/DAO/TranscriptMessageDAO.swift | 4 +-- .../Database/User/DAO/UserDAO.swift | 4 +-- .../Model/DeviceTransferCommand.swift | 12 +++---- 20 files changed, 105 insertions(+), 76 deletions(-) create mode 100644 Mixin/Service/DeviceTransfer/Utility/DeviceTransferProgress.swift diff --git a/Mixin.xcodeproj/project.pbxproj b/Mixin.xcodeproj/project.pbxproj index cbe912d99e..c4e4147884 100644 --- a/Mixin.xcodeproj/project.pbxproj +++ b/Mixin.xcodeproj/project.pbxproj @@ -719,6 +719,7 @@ 94341BFB2862F302009C9147 /* libopusenc.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 94341BF72862F302009C9147 /* libopusenc.xcframework */; }; 94341C002863530B009C9147 /* libogg.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 94341BFF2862F454009C9147 /* libogg.xcframework */; }; 9438252725EE697300709B7D /* CacheableAssetFileManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9438252625EE697300709B7D /* CacheableAssetFileManager.swift */; }; + 9439116A2A39EA4300CF6DC7 /* DeviceTransferProgress.swift in Sources */ = {isa = PBXBuildFile; fileRef = 943911692A39EA4300CF6DC7 /* DeviceTransferProgress.swift */; }; 94396F2629EB11E300A57833 /* DeviceTransferProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94396F2529EB11E300A57833 /* DeviceTransferProtocol.swift */; }; 94396F2829EB475500A57833 /* NWParameters+DeviceTransfer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94396F2729EB475500A57833 /* NWParameters+DeviceTransfer.swift */; }; 94396F2A29EBE52400A57833 /* DeviceTransferClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94396F2929EBE52400A57833 /* DeviceTransferClient.swift */; }; @@ -1796,6 +1797,7 @@ 94341BF72862F302009C9147 /* libopusenc.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = libopusenc.xcframework; sourceTree = ""; }; 94341BFF2862F454009C9147 /* libogg.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = libogg.xcframework; sourceTree = ""; }; 9438252625EE697300709B7D /* CacheableAssetFileManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CacheableAssetFileManager.swift; sourceTree = ""; }; + 943911692A39EA4300CF6DC7 /* DeviceTransferProgress.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeviceTransferProgress.swift; sourceTree = ""; }; 94396F2529EB11E300A57833 /* DeviceTransferProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeviceTransferProtocol.swift; sourceTree = ""; }; 94396F2729EB475500A57833 /* NWParameters+DeviceTransfer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "NWParameters+DeviceTransfer.swift"; sourceTree = ""; }; 94396F2929EBE52400A57833 /* DeviceTransferClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeviceTransferClient.swift; sourceTree = ""; }; @@ -2869,6 +2871,7 @@ 7C07A17C29D8FDBC00D4835C /* NetworkInterface.swift */, 940B30482A16050D00B45D26 /* NetworkSpeedInspector.swift */, 94149B422A17B4D5003E9E1A /* NetworkSpeedConditioner.swift */, + 943911692A39EA4300CF6DC7 /* DeviceTransferProgress.swift */, ); path = Utility; sourceTree = ""; @@ -5064,6 +5067,7 @@ 9424C64B259246B600FFDAE0 /* main.swift in Sources */, 945278982626BCD600023A6C /* HighlightableButton.swift in Sources */, 7BEB5D9F22B79F5500B8B10E /* EmergencyContactLoginVerificationCodeViewController.swift in Sources */, + 9439116A2A39EA4300CF6DC7 /* DeviceTransferProgress.swift in Sources */, 7CEB736429DBCBE5006FB5B2 /* DeviceTransferSnapshot.swift in Sources */, 7B59535122672D3500D59DB4 /* TopResultCell.swift in Sources */, 7CEB736629DBCD6A006FB5B2 /* DeviceTransferSticker.swift in Sources */, diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index 1bc119ecdd..41b32e16bb 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -7,9 +7,9 @@ final class DeviceTransferClient { enum State { case idle - case transfer(progress: Double, speed: String) + case transfer(progress: Float, speed: String) // `progress` is between 0.0 and 1.0 case failed(DeviceTransferError) - case importing(progress: Float) + case importing(progress: Float) // `progress` is between 0.0 and 1.0 case finished } @@ -29,13 +29,11 @@ final class DeviceTransferClient { private weak var statisticsTimer: Timer? private var fileStream: DeviceTransferFileStream? - - // Access counts on main queue - private var processedCount = 0 - private var totalCount: Int? - private var messageProcessingObservers: Set = [] + // Access on main queue + private var progress = DeviceTransferProgress() + private var opaquePointer: UnsafeMutableRawPointer { Unmanaged.passUnretained(self).toOpaque() } @@ -126,7 +124,7 @@ final class DeviceTransferClient { private func fail(error: DeviceTransferError) { assert(queue.isCurrent) - Logger.general.info(category: "DeviceTransferClient", message: "Stop: \(error) Processed: \(processedCount) Total: \(totalCount)") + Logger.general.info(category: "DeviceTransferClient", message: "Failed: \(error), progress: \(progress)") DispatchQueue.main.sync { self.statisticsTimer?.invalidate() } @@ -139,7 +137,7 @@ final class DeviceTransferClient { state = .failed(error) } - private func startUpdatingProgressAndSpeed() { + private func startUpdatingStatistics() { assert(Queue.main.isCurrent) statisticsTimer?.invalidate() statisticsTimer = Timer.scheduledTimer(withTimeInterval: 1, repeats: true) { [weak self] timer in @@ -149,12 +147,7 @@ final class DeviceTransferClient { return } let speed = self.speedInspector.drain() - let progress: Double - if let totalCount = self.totalCount { - progress = Double(self.processedCount) * 100 / Double(totalCount) - } else { - progress = 0 - } + let progress = self.progress.fractionCompleted self.queue.async { guard case .transfer = self.state else { DispatchQueue.main.sync(execute: timer.invalidate) @@ -242,8 +235,8 @@ extension DeviceTransferClient { Logger.general.info(category: "DeviceTransferClient", message: "Total count: \(count)") self.state = .transfer(progress: 0, speed: "") DispatchQueue.main.async { - self.totalCount = count - self.startUpdatingProgressAndSpeed() + self.progress.totalUnitCount = count + self.startUpdatingStatistics() } case .finish: Logger.general.info(category: "DeviceTransferClient", message: "Received finish command") @@ -279,7 +272,7 @@ extension DeviceTransferClient { assert(queue.isCurrent) DispatchQueue.main.sync { speedInspector.add(byteCount: content.count) - processedCount += 1 + progress.completedUnitCount += 1 } let firstHMACIndex = content.endIndex.advanced(by: -DeviceTransferProtocol.hmacDataCount) @@ -331,7 +324,7 @@ extension DeviceTransferClient { DispatchQueue.main.sync { speedInspector.add(byteCount: content.count) if isReceivingNewFile { - processedCount += 1 + progress.completedUnitCount += 1 } } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift index b352acebd6..96b9fe6799 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -71,11 +71,8 @@ final class DeviceTransferMessageProcessor { } } - private var totalCount = 0 - private var processedCount = 0 - + private var processingProgress = DeviceTransferProgress() private var writingCache: Cache? - private var cacheReadingBuffer = Data(count: Int(bytesPerKiloByte)) // Messages are saved to database in batch. See `messageSavingBatchCount` @@ -107,7 +104,7 @@ final class DeviceTransferMessageProcessor { cache.handle.write(encryptedMessage) cache.wroteCount += encryptedMessage.count processingQueue.async { - self.totalCount += 1 + self.processingProgress.totalUnitCount += 1 } if cache.isOversized { @@ -124,14 +121,14 @@ final class DeviceTransferMessageProcessor { func reportFileReceived() { processingQueue.async { - self.totalCount += 1 + self.processingProgress.totalUnitCount += 1 } } func finishProcessing() { assert(inputQueue.isCurrent) guard let lastCache = writingCache else { - Logger.general.info(category: "DeviceTransferMessageProcessor", message: "All pending messages are saved") + Logger.general.error(category: "DeviceTransferMessageProcessor", message: "No writing cache on finished") return } writingCache = nil @@ -141,6 +138,7 @@ final class DeviceTransferMessageProcessor { try? fileManager.removeItem(at: lastCache.url) self.savePendingMessages() self.processFiles() + Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Processing finished with progress: \(self.processingProgress)") self.progress = 1 } } @@ -157,9 +155,7 @@ extension DeviceTransferMessageProcessor { private func reportProgress() { assert(processingQueue.isCurrent) - // Divide the count as integer to prevent `progress` from rounding when counts are large - let progress = Float(processedCount * 100 / totalCount) / 100 - self.progress = progress + self.progress = processingProgress.fractionCompleted } private func savePendingMessages() { @@ -188,7 +184,7 @@ extension DeviceTransferMessageProcessor { } Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Begin processing cache \(cache.index)") - var processedCountOnLastProgressReporting = self.processedCount + var completedCountOnLastProgressReporting = processingProgress.completedUnitCount while stream.hasBytesAvailable { guard !isCancelled else { Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Not processing cache \(cache.index) by cancellation") @@ -236,9 +232,9 @@ extension DeviceTransferMessageProcessor { } catch { Logger.general.error(category: "DeviceTransferMessageProcessor", message: "Decrypt failed: \(error)") } - processedCount += 1 - if processedCount - processedCountOnLastProgressReporting == progressReportingInterval { - processedCountOnLastProgressReporting = processedCount + processingProgress.completedUnitCount += 1 + if processingProgress.completedUnitCount - completedCountOnLastProgressReporting == progressReportingInterval { + completedCountOnLastProgressReporting = processingProgress.completedUnitCount reportProgress() } } @@ -356,7 +352,7 @@ extension DeviceTransferMessageProcessor { return } Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Start processing files") - var processedCountOnLastProgressReporting = processedCount + var processedCountOnLastProgressReporting = processingProgress.completedUnitCount for case let fileURL as URL in fileEnumerator { guard !isCancelled else { @@ -394,9 +390,9 @@ extension DeviceTransferMessageProcessor { } } - processedCount += 1 - if processedCount - processedCountOnLastProgressReporting == progressReportingInterval { - processedCountOnLastProgressReporting = processedCount + processingProgress.completedUnitCount += 1 + if processingProgress.completedUnitCount - processedCountOnLastProgressReporting == progressReportingInterval { + processedCountOnLastProgressReporting = processingProgress.completedUnitCount reportProgress() } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift b/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift index 8a884d3683..6697c729e5 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferServer.swift @@ -12,7 +12,7 @@ final class DeviceTransferServer { enum State { case idle case listening(hostname: String, port: UInt16) - case transfer(progress: Double, speed: String) + case transfer(progress: Float, speed: String) // `progress` is between 0.0 and 1.0 case closed(ClosedReason) } diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferServerDataSource.swift b/Mixin/Service/DeviceTransfer/DeviceTransferServerDataSource.swift index da676c6142..a7a8af26fe 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferServerDataSource.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferServerDataSource.swift @@ -25,7 +25,7 @@ final class DeviceTransferServerDataSource { // MARK: - Data Count extension DeviceTransferServerDataSource { - func totalCount() -> Int { + func totalCount() -> Int64 { assert(!Queue.main.isCurrent) let messagesCount = MessageDAO.shared.messagesCount() let attachmentsCount = attachmentsCount() @@ -46,9 +46,9 @@ extension DeviceTransferServerDataSource { return total } - private func attachmentsCount() -> Int { + private func attachmentsCount() -> Int64 { let folders = AttachmentContainer.Category.allCases.map(\.pathComponent) + ["Transcript"] - let count = folders.reduce(0) { previousCount, folder in + let count: Int64 = folders.reduce(0) { previousCount, folder in let folderURL = AttachmentContainer.url.appendingPathComponent(folder) let count = validFileCount(in: folderURL) return previousCount + count @@ -56,8 +56,8 @@ extension DeviceTransferServerDataSource { return count } - private func validFileCount(in url: URL) -> Int { - var count = 0 + private func validFileCount(in url: URL) -> Int64 { + var count: Int64 = 0 guard let fileEnumerator = FileManager.default.enumerator(at: url, includingPropertiesForKeys: [.isRegularFileKey], options: [.skipsHiddenFiles, .skipsPackageDescendants]) else { return 0 } diff --git a/Mixin/Service/DeviceTransfer/Utility/DeviceTransferProgress.swift b/Mixin/Service/DeviceTransfer/Utility/DeviceTransferProgress.swift new file mode 100644 index 0000000000..ceaeaa22d8 --- /dev/null +++ b/Mixin/Service/DeviceTransfer/Utility/DeviceTransferProgress.swift @@ -0,0 +1,36 @@ +import Foundation + +struct DeviceTransferProgress { + + // NSProgress rounds `fractionCompleted` as binary, which may not be as + // expected from the view of decimal progress + // e.g. when `totalUnitCount` is `.max / 100`, and `completedUnitCount` + // is `(.max / 100) - 1`, the `fractionCompleted` will be 1.0, which may + // be considered as finished, or leads to misunderstanding + + var totalUnitCount: Int64 + var completedUnitCount: Int64 + + var fractionCompleted: Float { + if totalUnitCount == 0 { + return 0 + } else { + // Currently provides 4 digits for precision, that is 0.01% ~ 100.0% + return Float(completedUnitCount * 10000 / totalUnitCount) / 10000 + } + } + + init(totalUnitCount: Int64 = 0, completedUnitCount: Int64 = 0) { + self.totalUnitCount = 0 + self.completedUnitCount = 0 + } + +} + +extension DeviceTransferProgress: CustomStringConvertible { + + var description: String { + "" + } + +} diff --git a/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift b/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift index c18cdf5a74..2a82f0aaaa 100644 --- a/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift +++ b/Mixin/UserInterface/Controllers/DeviceTransfer/DeviceTransferProgressViewController.swift @@ -194,8 +194,8 @@ extension DeviceTransferProgressViewController { } } - private func updateTitleLabel(with transferProgress: Double, speed: String) { - let progress = String(format: "%.2f", transferProgress) + private func updateTitleLabel(with transferProgress: Float, speed: String) { + let progress = String(format: "%.2f", transferProgress * 100) switch connection { case .server: titleLabel.text = R.string.localizable.transferring_chat_progress(progress) @@ -204,7 +204,7 @@ extension DeviceTransferProgressViewController { case .cloud: break } - progressView.progress = Float(transferProgress / 100) + progressView.progress = transferProgress speedLabel.text = speed } diff --git a/MixinServices/MixinServices/Database/User/DAO/AppDAO.swift b/MixinServices/MixinServices/Database/User/DAO/AppDAO.swift index 986108d201..1e19bc678c 100644 --- a/MixinServices/MixinServices/Database/User/DAO/AppDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/AppDAO.swift @@ -44,8 +44,8 @@ public final class AppDAO: UserDatabaseDAO { return db.select(with: sql, arguments: [limit]) } - public func appsCount() -> Int { - let count: Int? = db.select(with: "SELECT COUNT(*) FROM apps") + public func appsCount() -> Int64 { + let count: Int64? = db.select(with: "SELECT COUNT(*) FROM apps") return count ?? 0 } diff --git a/MixinServices/MixinServices/Database/User/DAO/AssetDAO.swift b/MixinServices/MixinServices/Database/User/DAO/AssetDAO.swift index 87de4889f4..36d5e23c71 100644 --- a/MixinServices/MixinServices/Database/User/DAO/AssetDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/AssetDAO.swift @@ -113,8 +113,8 @@ public final class AssetDAO: UserDatabaseDAO { return db.select(with: sql, arguments: [limit]) } - public func assetsCount() -> Int { - let count: Int? = db.select(with: "SELECT COUNT(*) FROM assets") + public func assetsCount() -> Int64 { + let count: Int64? = db.select(with: "SELECT COUNT(*) FROM assets") return count ?? 0 } diff --git a/MixinServices/MixinServices/Database/User/DAO/ConversationDAO.swift b/MixinServices/MixinServices/Database/User/DAO/ConversationDAO.swift index 45026fb0e4..295b2fccf6 100644 --- a/MixinServices/MixinServices/Database/User/DAO/ConversationDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/ConversationDAO.swift @@ -668,8 +668,8 @@ public final class ConversationDAO: UserDatabaseDAO { return db.select(with: sql, arguments: [limit]) } - public func conversationsCount() -> Int { - let count: Int? = db.select(with: "SELECT COUNT(*) FROM conversations") + public func conversationsCount() -> Int64 { + let count: Int64? = db.select(with: "SELECT COUNT(*) FROM conversations") return count ?? 0 } diff --git a/MixinServices/MixinServices/Database/User/DAO/ExpiredMessageDAO.swift b/MixinServices/MixinServices/Database/User/DAO/ExpiredMessageDAO.swift index 49db63fa0a..eeb2d396de 100644 --- a/MixinServices/MixinServices/Database/User/DAO/ExpiredMessageDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/ExpiredMessageDAO.swift @@ -152,8 +152,8 @@ public final class ExpiredMessageDAO: UserDatabaseDAO { return db.select(with: sql, arguments: [limit]) } - public func expiredMessagesCount() -> Int { - let count: Int? = db.select(with: "SELECT COUNT(*) FROM expired_messages") + public func expiredMessagesCount() -> Int64 { + let count: Int64? = db.select(with: "SELECT COUNT(*) FROM expired_messages") return count ?? 0 } diff --git a/MixinServices/MixinServices/Database/User/DAO/MessageDAO.swift b/MixinServices/MixinServices/Database/User/DAO/MessageDAO.swift index 82c1ba4d81..20c08f1fa6 100644 --- a/MixinServices/MixinServices/Database/User/DAO/MessageDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/MessageDAO.swift @@ -989,8 +989,8 @@ extension MessageDAO { return db.select(with: sql, arguments: [limit]) } - public func messagesCount() -> Int { - let count: Int? = db.select(with: "SELECT COUNT(*) FROM messages") + public func messagesCount() -> Int64 { + let count: Int64? = db.select(with: "SELECT COUNT(*) FROM messages") return count ?? 0 } diff --git a/MixinServices/MixinServices/Database/User/DAO/MessageMentionDAO.swift b/MixinServices/MixinServices/Database/User/DAO/MessageMentionDAO.swift index 3b7540ab5f..c54815f56b 100644 --- a/MixinServices/MixinServices/Database/User/DAO/MessageMentionDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/MessageMentionDAO.swift @@ -24,8 +24,8 @@ public final class MessageMentionDAO: UserDatabaseDAO { return db.select(with: sql, arguments: [limit]) } - public func messageMentionsCount() -> Int { - let count: Int? = db.select(with: "SELECT COUNT(*) FROM message_mentions") + public func messageMentionsCount() -> Int64 { + let count: Int64? = db.select(with: "SELECT COUNT(*) FROM message_mentions") return count ?? 0 } diff --git a/MixinServices/MixinServices/Database/User/DAO/ParticipantDAO.swift b/MixinServices/MixinServices/Database/User/DAO/ParticipantDAO.swift index 835a16c125..13ddb6b2a0 100644 --- a/MixinServices/MixinServices/Database/User/DAO/ParticipantDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/ParticipantDAO.swift @@ -184,8 +184,8 @@ public final class ParticipantDAO: UserDatabaseDAO { return db.select(with: sql, arguments: [limit]) } - public func participantsCount() -> Int { - let count: Int? = db.select(with: "SELECT COUNT(*) FROM participants") + public func participantsCount() -> Int64 { + let count: Int64? = db.select(with: "SELECT COUNT(*) FROM participants") return count ?? 0 } diff --git a/MixinServices/MixinServices/Database/User/DAO/PinMessageDAO.swift b/MixinServices/MixinServices/Database/User/DAO/PinMessageDAO.swift index 9809fdd072..f8990652ca 100644 --- a/MixinServices/MixinServices/Database/User/DAO/PinMessageDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/PinMessageDAO.swift @@ -122,8 +122,8 @@ public final class PinMessageDAO: UserDatabaseDAO { return db.select(with: sql, arguments: [limit]) } - public func pinMessagesCount() -> Int { - let count: Int? = db.select(with: "SELECT COUNT(*) FROM pin_messages") + public func pinMessagesCount() -> Int64 { + let count: Int64? = db.select(with: "SELECT COUNT(*) FROM pin_messages") return count ?? 0 } diff --git a/MixinServices/MixinServices/Database/User/DAO/SnapshotDAO.swift b/MixinServices/MixinServices/Database/User/DAO/SnapshotDAO.swift index 3b03d67012..50cbb983c0 100644 --- a/MixinServices/MixinServices/Database/User/DAO/SnapshotDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/SnapshotDAO.swift @@ -125,8 +125,8 @@ public final class SnapshotDAO: UserDatabaseDAO { return db.select(with: sql, arguments: [limit]) } - public func snapshotsCount() -> Int { - let count: Int? = db.select(with: "SELECT COUNT(*) FROM snapshots") + public func snapshotsCount() -> Int64 { + let count: Int64? = db.select(with: "SELECT COUNT(*) FROM snapshots") return count ?? 0 } diff --git a/MixinServices/MixinServices/Database/User/DAO/StickerDAO.swift b/MixinServices/MixinServices/Database/User/DAO/StickerDAO.swift index e17f27c9dc..34a72250e7 100644 --- a/MixinServices/MixinServices/Database/User/DAO/StickerDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/StickerDAO.swift @@ -172,8 +172,8 @@ public final class StickerDAO: UserDatabaseDAO { return db.select(with: sql, arguments: [limit]) } - public func stickersCount() -> Int { - let count: Int? = db.select(with: "SELECT COUNT(*) FROM stickers") + public func stickersCount() -> Int64 { + let count: Int64? = db.select(with: "SELECT COUNT(*) FROM stickers") return count ?? 0 } diff --git a/MixinServices/MixinServices/Database/User/DAO/TranscriptMessageDAO.swift b/MixinServices/MixinServices/Database/User/DAO/TranscriptMessageDAO.swift index 8eef4a5c9a..e64330fcd9 100644 --- a/MixinServices/MixinServices/Database/User/DAO/TranscriptMessageDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/TranscriptMessageDAO.swift @@ -99,8 +99,8 @@ public final class TranscriptMessageDAO: UserDatabaseDAO { return db.select(with: sql, arguments: [limit]) } - public func transcriptMessagesCount() -> Int { - let count: Int? = db.select(with: "SELECT COUNT(*) FROM transcript_messages") + public func transcriptMessagesCount() -> Int64 { + let count: Int64? = db.select(with: "SELECT COUNT(*) FROM transcript_messages") return count ?? 0 } diff --git a/MixinServices/MixinServices/Database/User/DAO/UserDAO.swift b/MixinServices/MixinServices/Database/User/DAO/UserDAO.swift index 28fc0a20aa..f45485d069 100644 --- a/MixinServices/MixinServices/Database/User/DAO/UserDAO.swift +++ b/MixinServices/MixinServices/Database/User/DAO/UserDAO.swift @@ -230,8 +230,8 @@ public final class UserDAO: UserDatabaseDAO { return db.select(with: sql, arguments: [limit]) } - public func usersCount() -> Int { - let count: Int? = db.select(with: "SELECT COUNT(*) FROM users") + public func usersCount() -> Int64 { + let count: Int64? = db.select(with: "SELECT COUNT(*) FROM users") return count ?? 0 } diff --git a/MixinServices/MixinServices/Services/WebSocket/Model/DeviceTransferCommand.swift b/MixinServices/MixinServices/Services/WebSocket/Model/DeviceTransferCommand.swift index c8940fb076..7533756a7f 100644 --- a/MixinServices/MixinServices/Services/WebSocket/Model/DeviceTransferCommand.swift +++ b/MixinServices/MixinServices/Services/WebSocket/Model/DeviceTransferCommand.swift @@ -25,9 +25,9 @@ public struct DeviceTransferCommand { public enum Action { case pull case push(PushContext) - case start(Int) + case start(Int64) case connect(code: UInt16, userID: String) - case progress(Double) + case progress(Float) // `progress` is between 0.0 and 1.0, encoded to / decoded from between 0.0 and 100.0 case cancel case finish } @@ -115,15 +115,15 @@ extension DeviceTransferCommand: Codable { userID: userID) return .push(context) case ActionName.start: - let count = try container.decode(Int.self, forKey: .total) + let count = try container.decode(Int64.self, forKey: .total) return .start(count) case ActionName.connect: let code = try container.decode(UInt16.self, forKey: .code) let userID = try container.decode(String.self, forKey: .userID) return .connect(code: code, userID: userID) case ActionName.progress: - let progress = try container.decode(Double.self, forKey: .progress) - return .progress(progress) + let progress = try container.decode(Float.self, forKey: .progress) + return .progress(progress / 100) case ActionName.cancel: return .cancel case ActionName.finish: @@ -160,7 +160,7 @@ extension DeviceTransferCommand: Codable { try container.encode(userID, forKey: .userID) case let .progress(progress): try container.encode(ActionName.progress, forKey: .action) - try container.encode(progress, forKey: .progress) + try container.encode(progress * 100, forKey: .progress) case .cancel: try container.encode(ActionName.cancel, forKey: .action) case .finish: From 4ca8f5e870c61f6eae96ee398fa8267b8f007c50 Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Thu, 15 Jun 2023 00:11:39 +0800 Subject: [PATCH 22/23] Invalidate statistics timer after finished --- Mixin/Service/DeviceTransfer/DeviceTransferClient.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index 41b32e16bb..d216e19b8f 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -256,6 +256,9 @@ extension DeviceTransferClient { ConversationDAO.shared.updateLastMessageIdAndCreatedAt() try? FileManager.default.removeItem(at: self.cacheContainerURL) Logger.general.info(category: "DeviceTransferClient", message: "Cache container removed") + DispatchQueue.main.sync { + self.statisticsTimer?.invalidate() + } self.state = .finished } else { self.state = .importing(progress: progress) From 317d12c20d56d19511cae20ca7db8556bf56b991 Mon Sep 17 00:00:00 2001 From: wuyueyang Date: Fri, 16 Jun 2023 17:45:30 +0800 Subject: [PATCH 23/23] Reduce potential risk of processing may finish multiple times --- .../DeviceTransfer/DeviceTransferClient.swift | 27 +++++++++++-------- .../DeviceTransferMessageProcessor.swift | 5 +++- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift index d216e19b8f..bf608d75f5 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferClient.swift @@ -251,18 +251,23 @@ extension DeviceTransferClient { messageProcessor.$progress .receive(on: queue.dispatchQueue) .sink { progress in - if progress == 1 { - Logger.general.info(category: "DeviceTransferClient", message: "Import finished") - ConversationDAO.shared.updateLastMessageIdAndCreatedAt() - try? FileManager.default.removeItem(at: self.cacheContainerURL) - Logger.general.info(category: "DeviceTransferClient", message: "Cache container removed") - DispatchQueue.main.sync { - self.statisticsTimer?.invalidate() - } - self.state = .finished - } else { - self.state = .importing(progress: progress) + self.state = .importing(progress: progress) + } + .store(in: &messageProcessingObservers) + messageProcessor.$isFinished + .receive(on: queue.dispatchQueue) + .sink { isFinished in + guard isFinished else { + return + } + Logger.general.info(category: "DeviceTransferClient", message: "Import finished") + ConversationDAO.shared.updateLastMessageIdAndCreatedAt() + try? FileManager.default.removeItem(at: self.cacheContainerURL) + Logger.general.info(category: "DeviceTransferClient", message: "Cache container removed") + DispatchQueue.main.sync { + self.statisticsTimer?.invalidate() } + self.state = .finished } .store(in: &messageProcessingObservers) messageProcessor.finishProcessing() diff --git a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift index 96b9fe6799..f2212cefc5 100644 --- a/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift +++ b/Mixin/Service/DeviceTransfer/DeviceTransferMessageProcessor.swift @@ -44,6 +44,7 @@ final class DeviceTransferMessageProcessor { @Published private(set) var progress: Float = 0 @Published private(set) var processingError: ProcessingError? + @Published private(set) var isFinished = false private let key: Data private let remotePlatform: DeviceTransferPlatform @@ -139,7 +140,9 @@ final class DeviceTransferMessageProcessor { self.savePendingMessages() self.processFiles() Logger.general.info(category: "DeviceTransferMessageProcessor", message: "Processing finished with progress: \(self.processingProgress)") - self.progress = 1 + if !self.isCancelled && self.processingError == nil { + self.isFinished = true + } } }