From c7af9bcb0410c803b02df4ff2dcdcfc88b522fe5 Mon Sep 17 00:00:00 2001 From: kchro3 Date: Mon, 20 Nov 2023 01:07:55 -0800 Subject: [PATCH] aaaaaaaahhhhhhhhh --- TypeaheadAI.xcodeproj/project.pbxproj | 12 ++- TypeaheadAI/Actors/SpecialSaveActor.swift | 7 +- TypeaheadAI/ClientManager.swift | 4 - TypeaheadAI/CrudManagers/IntentManager.swift | 52 ------------- TypeaheadAI/Functions/FunctionManager.swift | 73 +++++++++++++++++++ TypeaheadAI/Functions/Functions.swift | 51 ------------- TypeaheadAI/Traits/CanSimulateSelectAll.swift | 30 ++++++++ .../Views/Modal/Message/MessageView.swift | 8 ++ TypeaheadAI/Views/Modal/Message/WebView.swift | 9 ++- TypeaheadAI/WindowManagers/ModalManager.swift | 19 ++++- 10 files changed, 148 insertions(+), 117 deletions(-) create mode 100644 TypeaheadAI/Functions/FunctionManager.swift delete mode 100644 TypeaheadAI/Functions/Functions.swift create mode 100644 TypeaheadAI/Traits/CanSimulateSelectAll.swift diff --git a/TypeaheadAI.xcodeproj/project.pbxproj b/TypeaheadAI.xcodeproj/project.pbxproj index 50ac320..06a6aa7 100644 --- a/TypeaheadAI.xcodeproj/project.pbxproj +++ b/TypeaheadAI.xcodeproj/project.pbxproj @@ -20,6 +20,7 @@ 2B0B30102ACF8C8000338B76 /* SpecialOpenActor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B0B300F2ACF8C8000338B76 /* SpecialOpenActor.swift */; }; 2B0B30192ADCBF1600338B76 /* QuickActionsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B0B30182ADCBF1600338B76 /* QuickActionsView.swift */; }; 2B0B301B2ADE8E2A00338B76 /* SettingsManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B0B301A2ADE8E2A00338B76 /* SettingsManager.swift */; }; + 2B11FCDF2B0B548100325F38 /* CanSimulateSelectAll.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B11FCDE2B0B548100325F38 /* CanSimulateSelectAll.swift */; }; 2B1F3B7E2AEC5F8500F1BB60 /* Supabase in Frameworks */ = {isa = PBXBuildFile; productRef = 2B1F3B7D2AEC5F8500F1BB60 /* Supabase */; }; 2B1F3B812AEDBDFF00F1BB60 /* LaunchAtLogin in Frameworks */ = {isa = PBXBuildFile; productRef = 2B1F3B802AEDBDFF00F1BB60 /* LaunchAtLogin */; }; 2B27450A2AB01CF400F37D3E /* SpecialSaveActor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B2745092AB01CF400F37D3E /* SpecialSaveActor.swift */; }; @@ -54,7 +55,7 @@ 2B8B952B2A9C528B00FB9EA9 /* ScriptManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B8B952A2A9C528B00FB9EA9 /* ScriptManager.swift */; }; 2B8CD4942B05D278003E0589 /* CanPerformOCR.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B8CD4932B05D278003E0589 /* CanPerformOCR.swift */; }; 2B8CD4962B05FF59003E0589 /* ModalFooterView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B8CD4952B05FF59003E0589 /* ModalFooterView.swift */; }; - 2B8CD4992B06DD36003E0589 /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B8CD4982B06DD36003E0589 /* Functions.swift */; }; + 2B8CD4992B06DD36003E0589 /* FunctionManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B8CD4982B06DD36003E0589 /* FunctionManager.swift */; }; 2B8CD49B2B076AE6003E0589 /* ActivateOnboardingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B8CD49A2B076AE6003E0589 /* ActivateOnboardingView.swift */; }; 2B8CD4A22B09C3A8003E0589 /* CanScreenshot.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B8CD4A12B09C3A8003E0589 /* CanScreenshot.swift */; }; 2B8CD4B42B0A9A11003E0589 /* Theme+Custom.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B8CD4B32B0A9A11003E0589 /* Theme+Custom.swift */; }; @@ -133,6 +134,7 @@ 2B0B300F2ACF8C8000338B76 /* SpecialOpenActor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpecialOpenActor.swift; sourceTree = ""; }; 2B0B30182ADCBF1600338B76 /* QuickActionsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QuickActionsView.swift; sourceTree = ""; }; 2B0B301A2ADE8E2A00338B76 /* SettingsManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsManager.swift; sourceTree = ""; }; + 2B11FCDE2B0B548100325F38 /* CanSimulateSelectAll.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CanSimulateSelectAll.swift; sourceTree = ""; }; 2B2745092AB01CF400F37D3E /* SpecialSaveActor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpecialSaveActor.swift; sourceTree = ""; }; 2B27450D2AB0380C00F37D3E /* AppContextManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppContextManager.swift; sourceTree = ""; }; 2B27450F2AB03A3D00F37D3E /* CanSimulateCopy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CanSimulateCopy.swift; sourceTree = ""; }; @@ -165,7 +167,7 @@ 2B8B952A2A9C528B00FB9EA9 /* ScriptManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScriptManager.swift; sourceTree = ""; }; 2B8CD4932B05D278003E0589 /* CanPerformOCR.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CanPerformOCR.swift; sourceTree = ""; }; 2B8CD4952B05FF59003E0589 /* ModalFooterView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModalFooterView.swift; sourceTree = ""; }; - 2B8CD4982B06DD36003E0589 /* Functions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Functions.swift; sourceTree = ""; }; + 2B8CD4982B06DD36003E0589 /* FunctionManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FunctionManager.swift; sourceTree = ""; }; 2B8CD49A2B076AE6003E0589 /* ActivateOnboardingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ActivateOnboardingView.swift; sourceTree = ""; }; 2B8CD4A12B09C3A8003E0589 /* CanScreenshot.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CanScreenshot.swift; sourceTree = ""; }; 2B8CD4B32B0A9A11003E0589 /* Theme+Custom.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Theme+Custom.swift"; sourceTree = ""; }; @@ -329,7 +331,7 @@ 2B8CD4972B06DD1D003E0589 /* Functions */ = { isa = PBXGroup; children = ( - 2B8CD4982B06DD36003E0589 /* Functions.swift */, + 2B8CD4982B06DD36003E0589 /* FunctionManager.swift */, ); path = Functions; sourceTree = ""; @@ -337,6 +339,7 @@ 2B8CD4A02B09C357003E0589 /* Traits */ = { isa = PBXGroup; children = ( + 2B11FCDE2B0B548100325F38 /* CanSimulateSelectAll.swift */, 2B27450F2AB03A3D00F37D3E /* CanSimulateCopy.swift */, 2B3792302AB83739008D812F /* CanSimulatePaste.swift */, 2B8CD4932B05D278003E0589 /* CanPerformOCR.swift */, @@ -723,6 +726,7 @@ 2B3FAC212AAAF22500B2D405 /* LlamaWrapper.cpp in Sources */, 2BA3C2372AADAD9A00537F95 /* SpecialCopyActor.swift in Sources */, 2B520D852AC82EA100310426 /* AccountView.swift in Sources */, + 2B11FCDF2B0B548100325F38 /* CanSimulateSelectAll.swift in Sources */, 2B27450E2AB0380C00F37D3E /* AppContextManager.swift in Sources */, 2B0B301B2ADE8E2A00338B76 /* SettingsManager.swift in Sources */, 2B7D357E2B00C1D100E85AEF /* SafariView.swift in Sources */, @@ -740,7 +744,7 @@ 2B2EF1522AC40CB500EF2BD4 /* MessagePendingView.swift in Sources */, 2BA7F0A92A9ABBE2003D38BA /* ClientManager.swift in Sources */, 2B44FA982ABA783F00C6B542 /* OnboardingView.swift in Sources */, - 2B8CD4992B06DD36003E0589 /* Functions.swift in Sources */, + 2B8CD4992B06DD36003E0589 /* FunctionManager.swift in Sources */, 2BE0EC272AA17F9100E47C52 /* MouseClickMonitor.swift in Sources */, 2BF929792AB04D2600FC105B /* MemoManager.swift in Sources */, 2BE0EC222AA0956C00E47C52 /* ModalView.swift in Sources */, diff --git a/TypeaheadAI/Actors/SpecialSaveActor.swift b/TypeaheadAI/Actors/SpecialSaveActor.swift index ee49cf8..d9469bd 100644 --- a/TypeaheadAI/Actors/SpecialSaveActor.swift +++ b/TypeaheadAI/Actors/SpecialSaveActor.swift @@ -12,7 +12,7 @@ import os.log actor SpecialSaveActor: CanSimulateCopy { private let modalManager: ModalManager private let clientManager: ClientManager - private let memoManager: MemoManager +// private let memoManager: MemoManager private let logger = Logger( subsystem: "ai.typeahead.TypeaheadAI", @@ -26,7 +26,7 @@ actor SpecialSaveActor: CanSimulateCopy { ) { self.modalManager = modalManager self.clientManager = clientManager - self.memoManager = memoManager +// self.memoManager = memoManager } func specialSave() { @@ -53,7 +53,8 @@ actor SpecialSaveActor: CanSimulateCopy { switch result { case .success(let output): if let text = output.text { - _ = self.memoManager.createEntry(summary: text, content: copiedText) + print(text) +// _ = self.memoManager.createEntry(summary: text, content: copiedText) } case .failure(let error): DispatchQueue.main.async { diff --git a/TypeaheadAI/ClientManager.swift b/TypeaheadAI/ClientManager.swift index 362a5cb..91edca1 100644 --- a/TypeaheadAI/ClientManager.swift +++ b/TypeaheadAI/ClientManager.swift @@ -203,10 +203,6 @@ class ClientManager: ObservableObject { history = self.historyManager?.fetchHistoryEntriesAsMessages(limit: 10, appContext: payload.appContext, quickActionID: quickActionID) } else { quickAction = await self.promptManager?.addPrompt(userIntent) - history = self.intentManager?.fetchIntentsAsMessages( - limit: 10, - appContext: payload.appContext - ) } // NOTE: We cached the copiedText earlier diff --git a/TypeaheadAI/CrudManagers/IntentManager.swift b/TypeaheadAI/CrudManagers/IntentManager.swift index 655c60f..c6aa943 100644 --- a/TypeaheadAI/CrudManagers/IntentManager.swift +++ b/TypeaheadAI/CrudManagers/IntentManager.swift @@ -62,58 +62,6 @@ class IntentManager { return originalScore * exp(-decayConstant * deltaTime) } - func fetchIntentsAsMessages( - limit: Int, - appContext: AppContext? - ) -> [Message] { - guard let appContext = appContext else { - return [] - } - - let fetchRequest: NSFetchRequest = IntentEntry.fetchRequest() - var predicates = [NSPredicate]() - - if let url = appContext.url?.host { - predicates.append(NSPredicate(format: "url == %@", url)) - } - - if let appName = appContext.appName { - predicates.append(NSPredicate(format: "appName == %@", appName)) - } - - if let bundleIdentifier = appContext.bundleIdentifier { - predicates.append(NSPredicate(format: "bundleIdentifier == %@", bundleIdentifier)) - } - - fetchRequest.predicate = NSCompoundPredicate(andPredicateWithSubpredicates: predicates) - fetchRequest.sortDescriptors = [NSSortDescriptor(key: "updatedAt", ascending: false)] - fetchRequest.fetchLimit = 50 - - do { - return try backgroundContext.performAndWait { - let entries = try backgroundContext.fetch(fetchRequest) - - // NOTE: Count the most common recent prompts and construct a user message - var promptCounts = [String: Int]() - for entry in entries { - if let prompt = entry.prompt { - promptCounts[prompt] = (promptCounts[prompt] ?? 0) + 1 - } - } - - var topPromptsString = "Most common user intents for this context:\n" - for (prompt, count) in promptCounts.sorted(by: { $0.value > $1.value }).prefix(limit) { - topPromptsString += "- \(prompt) (used \(count)x)\n" - } - - return [Message(id: UUID(), text: topPromptsString, isCurrentUser: true)] - } - } catch { - logger.error("Failed to fetch history entries: \(error.localizedDescription)") - return [] - } - } - func fetchContextualIntents( limit: Int, appContext: AppContext? diff --git a/TypeaheadAI/Functions/FunctionManager.swift b/TypeaheadAI/Functions/FunctionManager.swift new file mode 100644 index 0000000..578ef91 --- /dev/null +++ b/TypeaheadAI/Functions/FunctionManager.swift @@ -0,0 +1,73 @@ +// +// FunctionManager.swift +// TypeaheadAI +// +// Created by Jeff Hara on 11/16/23. +// + +import AppKit +import Foundation +import WebKit + +enum FunctionError: LocalizedError { + case openURL(_ message: String) + case notFound(_ message: String) + + var errorDescription: String { + switch self { + case .openURL(let message): return message + case .notFound(let message): return message + } + } +} + +struct FunctionCall: Codable { + let name: String + let args: [String: String] +} + +class FunctionManager: CanSimulateCopy, CanSimulateSelectAll { + func openURL(_ url: String) async throws { + guard let url = URL(string: url) else { + throw FunctionError.openURL("URL not found") + } + + NSWorkspace.shared.open(url) + } + + func parseAndCallFunction(jsonString: String, modalManager: ModalManager) async throws { + guard let jsonData = jsonString.data(using: .utf8), + let functionCall = try? JSONDecoder().decode(FunctionCall.self, from: jsonData) else { + return + } + + switch functionCall.name { + case "open_url": + print(functionCall.args) + let url = functionCall.args["url"]! + await modalManager.closeModal() + try await openURL(url) + try await Task.sleep(for: .seconds(3)) + try await simulateSelectAll() + try await simulateCopy() + await modalManager.showModal() + + guard let copiedText = NSPasteboard.general.string(forType: .string) else { + await modalManager.appendText("Couldn't fetch data from \(url)") + return + } + + try await modalManager.clientManager?.predict( + id: UUID(), + copiedText: copiedText, + incognitoMode: !modalManager.online, + userObjective: functionCall.args["prompt"], + stream: true, + streamHandler: modalManager.defaultHandler, + completion: modalManager.defaultCompletionHandler + ) + default: + throw FunctionError.notFound("Function \(functionCall.name) not found.") + } + } +} diff --git a/TypeaheadAI/Functions/Functions.swift b/TypeaheadAI/Functions/Functions.swift deleted file mode 100644 index 761a044..0000000 --- a/TypeaheadAI/Functions/Functions.swift +++ /dev/null @@ -1,51 +0,0 @@ -// -// Functions.swift -// TypeaheadAI -// -// Created by Jeff Hara on 11/16/23. -// - -import AppKit -import Foundation - -enum FunctionError: LocalizedError { - case openURL(_ message: String) - case notFound(_ message: String) - - var errorDescription: String { - switch self { - case .openURL(let message): return message - case .notFound(let message): return message - } - } -} - -struct FunctionCall: Codable { - let name: String - let args: [String: String] -} - -class Functions { - static func openURL(_ url: String) async throws { - guard let url = URL(string: url) else { - throw FunctionError.openURL("URL not found") - } - - NSWorkspace.shared.open(url) - } - - static func parseAndCallFunction(jsonString: String, modalManager: ModalManager) async throws { - guard let jsonData = jsonString.data(using: .utf8), - let functionCall = try? JSONDecoder().decode(FunctionCall.self, from: jsonData) else { - return - } - - switch functionCall.name { - case "open_url": - try await openURL(functionCall.args["url"]!) - await modalManager.appendText("Opening \(functionCall.args["url"] ?? "url")...") - default: - throw FunctionError.notFound("Function \(functionCall.name) not found.") - } - } -} diff --git a/TypeaheadAI/Traits/CanSimulateSelectAll.swift b/TypeaheadAI/Traits/CanSimulateSelectAll.swift new file mode 100644 index 0000000..37dbd45 --- /dev/null +++ b/TypeaheadAI/Traits/CanSimulateSelectAll.swift @@ -0,0 +1,30 @@ +// +// CanSimulateSelectAll.swift +// TypeaheadAI +// +// Created by Jeff Hara on 11/20/23. +// + +import Foundation +import Carbon.HIToolbox + +protocol CanSimulateSelectAll { + func simulateSelectAll() async throws +} + +extension CanSimulateSelectAll { + func simulateSelectAll() async throws { + // Post a Command-A keystroke + let source = CGEventSource(stateID: .hidSystemState)! + let cmdADown = CGEvent(keyboardEventSource: source, virtualKey: 0x00, keyDown: true)! // a key + let cmdAUp = CGEvent(keyboardEventSource: source, virtualKey: 0x00, keyDown: false)! // a key + + cmdADown.flags = [.maskCommand] + cmdAUp.flags = [.maskCommand] + + cmdADown.post(tap: .cghidEventTap) + try await Task.sleep(for: .milliseconds(20)) + cmdAUp.post(tap: .cghidEventTap) + try await Task.sleep(for: .milliseconds(200)) + } +} diff --git a/TypeaheadAI/Views/Modal/Message/MessageView.swift b/TypeaheadAI/Views/Modal/Message/MessageView.swift index 507ec9d..75792c4 100644 --- a/TypeaheadAI/Views/Modal/Message/MessageView.swift +++ b/TypeaheadAI/Views/Modal/Message/MessageView.swift @@ -141,6 +141,10 @@ struct MessageView: View { WebView(html: data, dynamicHeight: $webViewHeight) .frame(width: 400, height: webViewHeight) .background(Color.accentColor.opacity(0.8)) + case .url(let data): + WebView(url: data, dynamicHeight: $webViewHeight) + .frame(width: 400, height: webViewHeight) + .background(Color.accentColor.opacity(0.8)) case .image(let data): if let imageData = try? self.decodeBase64Image(data.image) { Image(nsImage: imageData) @@ -213,6 +217,10 @@ struct MessageView: View { WebView(html: data, dynamicHeight: $webViewHeight) .frame(width: 400, height: webViewHeight) .background(Color.accentColor.opacity(0.8)) + case .url(let data): + WebView(url: data, dynamicHeight: $webViewHeight) + .frame(width: 400, height: webViewHeight) + .background(Color.accentColor.opacity(0.8)) case .image(let data): if let imageData = try? self.decodeBase64Image(data.image) { Image(nsImage: imageData) diff --git a/TypeaheadAI/Views/Modal/Message/WebView.swift b/TypeaheadAI/Views/Modal/Message/WebView.swift index a87a3a1..c50b24d 100644 --- a/TypeaheadAI/Views/Modal/Message/WebView.swift +++ b/TypeaheadAI/Views/Modal/Message/WebView.swift @@ -9,7 +9,8 @@ import SwiftUI import WebKit struct WebView: NSViewRepresentable { - var html: String + var html: String? + var url: URL? @Binding var dynamicHeight: CGFloat func makeNSView(context: Context) -> WKWebView { @@ -19,7 +20,11 @@ struct WebView: NSViewRepresentable { } func updateNSView(_ nsView: WKWebView, context: Context) { - nsView.loadHTMLString(html, baseURL: nil) + if let url = url { + nsView.load(URLRequest(url: url)) + } else if let html = html { + nsView.loadHTMLString(html, baseURL: nil) + } } func makeCoordinator() -> Coordinator { diff --git a/TypeaheadAI/WindowManagers/ModalManager.swift b/TypeaheadAI/WindowManagers/ModalManager.swift index e970fe6..8c1afa4 100644 --- a/TypeaheadAI/WindowManagers/ModalManager.swift +++ b/TypeaheadAI/WindowManagers/ModalManager.swift @@ -10,12 +10,14 @@ import SwiftUI import Foundation import MarkdownUI import os.log +import WebKit enum MessageType: Codable, Equatable { case string case html(data: String) case image(data: ImageData) case data(data: Data) + case url(data: URL) } // TODO: Add to persistence @@ -69,6 +71,8 @@ class ModalManager: ObservableObject { var intentManager: IntentManager? = nil var settingsManager: SettingsManager? = nil + private let functionManager = FunctionManager() + var toastWindow: CustomModalWindow? func hasText() -> Bool { @@ -152,6 +156,19 @@ class ModalManager: ObservableObject { messages[idx].text += text } + @MainActor + func appendURL(_ url: URL) async { + isPending = false + userIntents = nil + + messages.append(Message( + id: UUID(), + text: "", + isCurrentUser: false, + messageType: .url(data: url) + )) + } + @MainActor func appendImage(_ image: ImageData, prompt: String, caption: String?) { isPending = false @@ -445,7 +462,7 @@ class ModalManager: ObservableObject { case .function: Task { do { - try await Functions.parseAndCallFunction(jsonString: text, modalManager: self) + try await functionManager.parseAndCallFunction(jsonString: text, modalManager: self) } catch { await self.setError(error.localizedDescription) }