Skip to content

work on improving the streaming. #291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kchro3 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

work on improving the streaming. #291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion TypeaheadAI/Accessibility/Speaker.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class Speaker: NSObject, AVSpeechSynthesizerDelegate, CanSimulateControl {
func speak(_ text: String, withCallback: Bool = false) {
if isNarrateEnabled {
Task {
let utterance = AVSpeechUtterance(string: NSLocalizedString(text, comment: ""))
let utterance = AVSpeechUtterance(string: text)

if let selectedVoice = selectedVoice {
utterance.voice = AVSpeechSynthesisVoice(identifier: selectedVoice)
Expand Down
45 changes: 33 additions & 12 deletions TypeaheadAI/WindowManagers/ModalManager.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ class ModalManager: ObservableObject, CanSimulateDictation {
private let maxIntents = 9

private let speaker = Speaker()
private var speakerCursor = 0

init(context: NSManagedObjectContext) {
self.context = context
Expand Down Expand Up @@ -114,6 +115,7 @@ class ModalManager: ObservableObject, CanSimulateDictation {
currentTask?.cancel()
currentTask = nil
isPending = false
speakerCursor = 0

// NOTE: Worry about this later... Deal with cancellations of tools
// var toolCallId: String? = nil
Expand Down Expand Up @@ -146,6 +148,7 @@ class ModalManager: ObservableObject, CanSimulateDictation {
messages = []
isPending = false
userIntents = nil
speakerCursor = 0
}

func getQuickAction() -> QuickAction? {
Expand All @@ -163,7 +166,7 @@ class ModalManager: ObservableObject, CanSimulateDictation {
messageContext: MessageContext? = nil
) {
if !isHidden {
speaker.speak(text, withCallback: true)
speaker.speak(NSLocalizedString(text, comment: ""), withCallback: true)
}

if !isHidden,
Expand Down Expand Up @@ -253,7 +256,7 @@ class ModalManager: ObservableObject, CanSimulateDictation {
@MainActor
func appendToolError(_ responseError: String, functionCall: FunctionCall, appContext: AppContext?) {
isPending = false
speaker.speak(responseError)
speaker.speak(NSLocalizedString(responseError, comment: ""))

if let lastMessage = messages.last {
messages.append(
Expand Down Expand Up @@ -296,7 +299,7 @@ class ModalManager: ObservableObject, CanSimulateDictation {
@MainActor
func setError(_ responseError: String, isHidden: Bool = false, appContext: AppContext?) {
isPending = false
speaker.speak(responseError)
speaker.speak(NSLocalizedString(responseError, comment: ""))

if let idx = messages.indices.last, !messages[idx].isCurrentUser, !messages[idx].isHidden {
messages[idx].responseError = responseError
Expand Down Expand Up @@ -342,6 +345,7 @@ class ModalManager: ObservableObject, CanSimulateDictation {
func appendText(_ text: String, appContext: AppContext?) async {
guard let idx = messages.indices.last, !messages[idx].isCurrentUser, !messages[idx].isHidden else {
// If the AI response doesn't exist yet, create one.
speakerCursor = 0
if let lastMessage = messages.last {
messages.append(
Message(
Expand Down Expand Up @@ -379,19 +383,36 @@ class ModalManager: ObservableObject, CanSimulateDictation {
}

messages[idx].text += text
if text.contains("\n") {
let chunks = messages[idx].text.split(separator: "\n", omittingEmptySubsequences: true)

if text.trimmingCharacters(in: .whitespaces).hasSuffix("\n") {
// Case: If text looks like "bcd\n ", then chunks will look like ["abc", "bcd"]
speaker.speak(String(chunks[chunks.count - 1]))
} else if chunks.count >= 2 {
// Case: If chunks looks like ["abc", "bcd", "def"]

let containsWhitespace = text.rangeOfCharacter(from: .whitespacesAndNewlines) != nil
if containsWhitespace {
if let lastIndex = messages[idx].text.lastIndex(where: { $0.isWhitespace || $0.isNewline }) {
print(lastIndex)
speaker.speak(String()
}


let chunks = messages[idx].text.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
let numComponents = text.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }.count

if chunks.count >= 2 {
speaker.speak(String(chunks[chunks.count - 2]))
} else {
speaker.speak(String(chunks[0]))
}

print(chunks, numComponents, text)
}

// if text.trimmingCharacters(in: .whitespaces).hasSuffix("\n") {
// // Case: If text looks like "bcd\n ", then chunks will look like ["abc", "bcd"]
// speaker.speak(String(chunks[chunks.count - 1]))
// } else if chunks.count >= 2 {
// // Case: If chunks looks like ["abc", "bcd", "def"]
// speaker.speak(String(chunks[chunks.count - 2]))
// } else {
// speaker.speak(String(chunks[0]))
// }
}

/// Append plan to the AI response. Creates a new message if there is nothing to append to.
Expand Down Expand Up @@ -752,7 +773,7 @@ class ModalManager: ObservableObject, CanSimulateDictation {

if bufferedPayload.mode == .text, let text = bufferedPayload.text {

if let lastChunk = text.split(separator: "\n", omittingEmptySubsequences: true).last {
if let lastChunk = text.split(separator: " ", omittingEmptySubsequences: true).last {
speaker.speak(String(lastChunk), withCallback: true)
}

Expand Down