From 067b4210dbb25c9e59d0b76102b3187f7ce80bda Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Wed, 25 Dec 2024 13:20:36 +0500 Subject: [PATCH 01/13] receive reaction events from bot --- .../react-agents/lib/discord/discord-client.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js b/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js index 4089113fc..bf5168439 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js @@ -536,6 +536,20 @@ export class DiscordBotClient extends EventTarget { this.input.handleVoiceIdle(args); break; } + case 'messagereactionadd': { + console.log('messagereactionadd', args); + this.dispatchEvent(new MessageEvent('messagereactionadd', { + data: args, + })); + break; + } + case 'messagereactionremove': { + console.log('messagereactionremove', args); + this.dispatchEvent(new MessageEvent('messagereactionremove', { + data: args, + })); + break; + } default: { console.warn('unhandled json method', method); break; From c0f6e5d1465c213dfa3a56a69ca6b6fde526bf45 Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Wed, 25 Dec 2024 13:32:46 +0500 Subject: [PATCH 02/13] delegate reaction handling to DiscordOutput, bind incoming message reactions to discord manager --- .../react-agents/classes/discord-manager.ts | 30 +++++++++++++++++++ .../lib/discord/discord-client.js | 24 ++++++++++----- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts index 52e5d37eb..f41132b1b 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts @@ -411,11 +411,41 @@ export class DiscordBot extends EventTarget { }); }; + // message reactions + const _bindIncomingMessageReactions = () => { + discordBotClient.output.addEventListener('messagereactionadd', (e: MessageEvent) => { + const { + userId, + messageId, + emoji, + } = e.data; + console.log('messagereactionadd', { + userId, + messageId, + emoji, + }); + }); + + discordBotClient.output.addEventListener('messagereactionremove', (e: MessageEvent) => { + const { + userId, + messageId, + emoji, + } = e.data; + console.log('messagereactionremove', { + userId, + messageId, + emoji, + }); + }); + }; + (async () => { _bindChannels(); _bindGuildMemberAdd(); _bindGuildMemberRemove(); _bindIncoming(); + _bindIncomingMessageReactions(); await _connect(); })().catch(err => { console.warn('discord bot error', err); diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js b/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js index bf5168439..f0468f55b 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js @@ -340,6 +340,20 @@ export class DiscordOutput extends EventTarget { } } + handleMessageReactionAdd(args) { + console.log('handleMessageReactionAdd', args); + this.dispatchEvent(new MessageEvent('messagereactionadd', { + data: args, + })); + } + + handleMessageReactionRemove(args) { + console.log('handleMessageReactionRemove', args); + this.dispatchEvent(new MessageEvent('messagereactionremove', { + data: args, + })); + } + destroy() { for (const stream of this.streams.values()) { stream.destroy(); @@ -537,17 +551,11 @@ export class DiscordBotClient extends EventTarget { break; } case 'messagereactionadd': { - console.log('messagereactionadd', args); - this.dispatchEvent(new MessageEvent('messagereactionadd', { - data: args, - })); + this.output.handleMessageReactionAdd(args); break; } case 'messagereactionremove': { - console.log('messagereactionremove', args); - this.dispatchEvent(new MessageEvent('messagereactionremove', { - data: args, - })); + this.output.handleMessageReactionRemove(args); break; } default: { From 6fe58fe65eb241b9856a746a137d8c0d8370ec4f Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Wed, 25 Dec 2024 14:18:57 +0500 Subject: [PATCH 03/13] init process message reactions in runtime --- .../classes/conversation-object.ts | 24 +++++++++++++ .../react-agents/classes/discord-manager.ts | 36 +++++++++++++++++++ .../packages/react-agents/runtime.ts | 9 +++++ .../react-agents/types/react-agents.d.ts | 7 ++++ 4 files changed, 76 insertions(+) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts index 240749580..8ff948f45 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts @@ -8,6 +8,7 @@ import { PlayableAudioStream, GetHashFn, MessageCache, + MessageReactionEventData, } from '../types' import { SceneObject } from '../classes/scene-object'; import { Player } from 'react-agents-client/util/player.mjs'; @@ -358,4 +359,27 @@ export class ConversationObject extends EventTarget { }), ); } + + + async processMessageReaction( + reaction, + messageId, + userId, + ) { + console.log('process reaction: ',{ + reaction, + messageId, + userId, + }); + + const e = new ExtendableMessageEvent('localMessageReaction', { + data: { + reaction, + messageId, + userId, + }, + }); + this.dispatchEvent(e); + } + } diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts index f41132b1b..8c30ba8f9 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts @@ -370,6 +370,7 @@ export class DiscordBot extends EventTarget { text, channelId, // if there is no channelId, it's a DM // XXX discord channel/dm distinction can be made more explicit with a type: string field... + messageId, } = e.data; // look up conversation @@ -390,6 +391,7 @@ export class DiscordBot extends EventTarget { method: 'say', args: { text: formattedMessage, + messageId, }, }; const id = getIdFromUserId(userId); @@ -418,12 +420,32 @@ export class DiscordBot extends EventTarget { userId, messageId, emoji, + channelId, } = e.data; console.log('messagereactionadd', { userId, messageId, emoji, + channelId, }); + + console.log('this.dmConversations: ', this.dmConversations); + console.log('this.channelConversations: ', this.channelConversations); + console.log('userId: ', userId); + console.log('channelId: ', channelId); + + // look up conversation + let conversation: ConversationObject | null = null; + if (this.dmConversations.has(userId)) { + conversation = this.dmConversations.get(userId) ?? null; + } else { + conversation = this.channelConversations.get(channelId) ?? null; + } + + if (conversation) { + conversation.processMessageReaction(emoji, messageId, userId); + } + }); discordBotClient.output.addEventListener('messagereactionremove', (e: MessageEvent) => { @@ -431,12 +453,26 @@ export class DiscordBot extends EventTarget { userId, messageId, emoji, + channelId, } = e.data; console.log('messagereactionremove', { userId, messageId, emoji, + channelId, }); + + let conversation: ConversationObject | null = null; + if (this.dmConversations.has(userId)) { + conversation = this.dmConversations.get(userId) ?? null; + } else { + conversation = this.channelConversations.get(channelId) ?? null; + } + + if (conversation) { + conversation.processMessageReaction(emoji, messageId, userId); + } + }); }; diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts index 070065b02..dd362eb58 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts @@ -20,6 +20,7 @@ import { AbortableMessageEvent, PendingActionEventData, PerceptionPropsAux, + MessageReactionEvent, } from './types'; import { PendingActionEvent, @@ -540,6 +541,14 @@ export const bindConversationToAgent = ({ })(); } }); + conversation.addEventListener('localMessageReaction', (e: MessageReactionEvent) => { + const { reaction, messageId, userId } = e.data; + console.log('runtime localMessageReaction: ', { + reaction, + messageId, + userId, + }); + }); }; // XXX can move this to the agent renderer diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts index f81439d4e..72420c841 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts @@ -533,6 +533,13 @@ export type ActionMessageEventData = { }; export type ActionMessageEvent = ExtendableMessageEvent; +export type MessageReactionEventData = { + reaction: string; + messageId: string; + userId: string; +}; +export type MessageReactionEvent = ExtendableMessageEvent; + export type ConversationChangeEventData = { conversation: ConversationObject; }; From 5db96a7ff7b2484a38ac382915f410ee7cfdaa36 Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Wed, 25 Dec 2024 15:03:26 +0500 Subject: [PATCH 04/13] add "messageReaction" perception, add reaction as localmessage --- .../react-agents/classes/discord-manager.ts | 27 ++++++++++++++++--- .../packages/react-agents/loops/chat-loop.tsx | 18 +++++++++++++ .../packages/react-agents/runtime.ts | 12 ++++++++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts index 8c30ba8f9..0962a450f 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts @@ -421,6 +421,7 @@ export class DiscordBot extends EventTarget { messageId, emoji, channelId, + userDisplayName, } = e.data; console.log('messagereactionadd', { userId, @@ -442,8 +443,25 @@ export class DiscordBot extends EventTarget { conversation = this.channelConversations.get(channelId) ?? null; } + const rawMessageReaction = { + userId, + name: userDisplayName, + method: 'messageReaction', + args: { + reaction: emoji, + messageId, + userId, + }, + }; + + const newMessageReaction = formatConversationMessage(rawMessageReaction, { + agent, + }); + + console.log('newMessageReaction: ', newMessageReaction); + if (conversation) { - conversation.processMessageReaction(emoji, messageId, userId); + conversation.addLocalMessage(newMessageReaction); } }); @@ -469,10 +487,13 @@ export class DiscordBot extends EventTarget { conversation = this.channelConversations.get(channelId) ?? null; } + const newMessageReaction = formatConversationMessage(rawMessageReaction, { + agent, + }); + if (conversation) { - conversation.processMessageReaction(emoji, messageId, userId); + conversation.addLocalMessage(newMessageReaction); } - }); }; diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/loops/chat-loop.tsx b/packages/usdk/packages/upstreet-agent/packages/react-agents/loops/chat-loop.tsx index c2621f38c..a2a7c432d 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/loops/chat-loop.tsx +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/loops/chat-loop.tsx @@ -39,6 +39,24 @@ export const ChatLoop = (props: LoopProps) => { }} priority={-1} /> + { + const { targetAgent } = e.data; + + + console.log('messageReaction: ', e.data); + (async () => { + const abortController = new AbortController(); + const { signal } = abortController; + + await targetAgent.evaluate(evaluator, { + signal, + }); + })(); + }} + priority={-1} + /> { diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts index dd362eb58..f2a35c468 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts @@ -541,13 +541,23 @@ export const bindConversationToAgent = ({ })(); } }); - conversation.addEventListener('localMessageReaction', (e: MessageReactionEvent) => { + conversation.addEventListener('localMessageReaction', async (e: MessageReactionEvent) => { const { reaction, messageId, userId } = e.data; console.log('runtime localMessageReaction: ', { reaction, messageId, userId, }); + try { + const { + aborted, + } = await handleChatPerception(e.data, { + agent, + conversation, + }); + } catch (error) { + console.warn('caught new message reaction error', error); + } }); }; From 41b3a4e3edbffa90911481498caec5eae7d231e6 Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Wed, 25 Dec 2024 15:17:51 +0500 Subject: [PATCH 05/13] update "sendTyping" as optional for perceptions --- .../react-agents/classes/discord-manager.ts | 19 ++++++++++++++- .../classes/generative-agent-object.ts | 23 +++++++++++++++---- .../packages/react-agents/loops/chat-loop.tsx | 9 +++++--- .../react-agents/types/react-agents.d.ts | 1 + 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts index 0962a450f..f103477bd 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts @@ -425,6 +425,7 @@ export class DiscordBot extends EventTarget { } = e.data; console.log('messagereactionadd', { userId, + userDisplayName, messageId, emoji, channelId, @@ -470,12 +471,14 @@ export class DiscordBot extends EventTarget { const { userId, messageId, + userDisplayName, emoji, channelId, } = e.data; console.log('messagereactionremove', { userId, messageId, + userDisplayName, emoji, channelId, }); @@ -487,13 +490,27 @@ export class DiscordBot extends EventTarget { conversation = this.channelConversations.get(channelId) ?? null; } + const rawMessageReaction = { + userId, + name: userDisplayName, + method: 'messageReaction', + args: { + reaction: emoji, + messageId, + userId, + }, + }; + const newMessageReaction = formatConversationMessage(rawMessageReaction, { agent, }); - + + console.log('newMessageReaction: ', newMessageReaction); + if (conversation) { conversation.addLocalMessage(newMessageReaction); } + }); }; diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/generative-agent-object.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/generative-agent-object.ts index a3841cc45..85c3135da 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/generative-agent-object.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/generative-agent-object.ts @@ -11,6 +11,7 @@ import type { ActionStep, Evaluator, DebugOptions, + EvaluateOpts, } from '../types'; import { ConversationObject, @@ -110,14 +111,28 @@ export class GenerativeAgentObject { }); // }); } - async evaluate(evaluator: Evaluator) { - return await this.conversation.typing(async () => { + async evaluate(evaluator: Evaluator, opts?: EvaluateOpts) { + const { + sendTyping, + } = opts ?? {}; + + const evaluateAndExecuteStep = async () => { const step = await evaluator.evaluate({ - generativeAgent: this, + generativeAgent: this as GenerativeAgentObject, }); await executeAgentActionStep(this, step); return step; - }); + }; + + if (sendTyping) { + let result: ActionStep; + await this.conversation.typing(async () => { + result = await evaluateAndExecuteStep(); + }); + return result; + } else { + return await evaluateAndExecuteStep(); + } } /* async generate(hint: string, schema?: ZodTypeAny) { // console.log('agent renderer generate 1'); diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/loops/chat-loop.tsx b/packages/usdk/packages/upstreet-agent/packages/react-agents/loops/chat-loop.tsx index a2a7c432d..fc2e8567f 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/loops/chat-loop.tsx +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/loops/chat-loop.tsx @@ -1,7 +1,7 @@ import React, { useState } from 'react'; import { useAgent } from '../hooks'; import { Perception } from '../components/core/perception'; -import { LoopProps } from '../types'; +import { EvaluateOpts, LoopProps } from '../types'; import { BasicEvaluator } from '../evaluators/basic-evaluator'; export const ChatLoop = (props: LoopProps) => { @@ -50,9 +50,12 @@ export const ChatLoop = (props: LoopProps) => { const abortController = new AbortController(); const { signal } = abortController; - await targetAgent.evaluate(evaluator, { + const opts: EvaluateOpts = { signal, - }); + sendTyping: false, + generativeAgent: targetAgent, + }; + await targetAgent.evaluate(evaluator, opts); })(); }} priority={-1} diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts index 72420c841..e4b44f2c2 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts @@ -194,6 +194,7 @@ export type EvaluatorOpts = { export type EvaluateOpts = { generativeAgent: GenerativeAgentObject, signal?: AbortSignal, + sendTyping?: boolean, }; export type Evaluator = { evaluate: (opts: EvaluateOpts) => Promise; From 63ab1a633616bc11a64b8c8b5c0a221961a22c00 Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Wed, 25 Dec 2024 15:28:08 +0500 Subject: [PATCH 06/13] fix agent object id and name being formatted --- .../react-agents/classes/discord-manager.ts | 83 +++++-------------- 1 file changed, 19 insertions(+), 64 deletions(-) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts index f103477bd..17b0a6c3c 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts @@ -415,7 +415,7 @@ export class DiscordBot extends EventTarget { // message reactions const _bindIncomingMessageReactions = () => { - discordBotClient.output.addEventListener('messagereactionadd', (e: MessageEvent) => { + const handleReaction = (e: MessageEvent, eventType: string) => { const { userId, messageId, @@ -423,7 +423,8 @@ export class DiscordBot extends EventTarget { channelId, userDisplayName, } = e.data; - console.log('messagereactionadd', { + + console.log(eventType, { userId, userDisplayName, messageId, @@ -431,18 +432,12 @@ export class DiscordBot extends EventTarget { channelId, }); - console.log('this.dmConversations: ', this.dmConversations); - console.log('this.channelConversations: ', this.channelConversations); - console.log('userId: ', userId); - console.log('channelId: ', channelId); + // look up conversation + const conversation = this.dmConversations.has(userId) + ? this.dmConversations.get(userId) ?? null + : this.channelConversations.get(channelId) ?? null; - // look up conversation - let conversation: ConversationObject | null = null; - if (this.dmConversations.has(userId)) { - conversation = this.dmConversations.get(userId) ?? null; - } else { - conversation = this.channelConversations.get(channelId) ?? null; - } + if (!conversation) return; const rawMessageReaction = { userId, @@ -456,62 +451,22 @@ export class DiscordBot extends EventTarget { }; const newMessageReaction = formatConversationMessage(rawMessageReaction, { - agent, - }); - - console.log('newMessageReaction: ', newMessageReaction); - - if (conversation) { - conversation.addLocalMessage(newMessageReaction); - } - - }); - - discordBotClient.output.addEventListener('messagereactionremove', (e: MessageEvent) => { - const { - userId, - messageId, - userDisplayName, - emoji, - channelId, - } = e.data; - console.log('messagereactionremove', { - userId, - messageId, - userDisplayName, - emoji, - channelId, - }); - - let conversation: ConversationObject | null = null; - if (this.dmConversations.has(userId)) { - conversation = this.dmConversations.get(userId) ?? null; - } else { - conversation = this.channelConversations.get(channelId) ?? null; - } - - const rawMessageReaction = { - userId, - name: userDisplayName, - method: 'messageReaction', - args: { - reaction: emoji, - messageId, - userId, + agent: { + id: getIdFromUserId(userId), + name: userDisplayName, }, - }; - - const newMessageReaction = formatConversationMessage(rawMessageReaction, { - agent, }); - console.log('newMessageReaction: ', newMessageReaction); + conversation.addLocalMessage(newMessageReaction); + }; - if (conversation) { - conversation.addLocalMessage(newMessageReaction); - } + discordBotClient.output.addEventListener('messagereactionadd', + (e) => handleReaction(e, 'messagereactionadd') + ); - }); + discordBotClient.output.addEventListener('messagereactionremove', + (e) => handleReaction(e, 'messagereactionremove') + ); }; (async () => { From 6296aa0470cdfa12883d2cb181f43489e88259da Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Wed, 25 Dec 2024 15:38:14 +0500 Subject: [PATCH 07/13] add reaction context to message args --- .../packages/react-agents/classes/discord-manager.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts index 17b0a6c3c..b9b5d4b07 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts @@ -447,6 +447,9 @@ export class DiscordBot extends EventTarget { reaction: emoji, messageId, userId, + context: { + action: eventType === 'messagereactionadd' ? 'Reaction added' : 'Reaction removed', + }, }, }; From 039c23a41c4ebff38a54ddddb5f287a85e01b808 Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Wed, 25 Dec 2024 15:48:27 +0500 Subject: [PATCH 08/13] default sendtyping to true --- .../packages/react-agents/classes/generative-agent-object.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/generative-agent-object.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/generative-agent-object.ts index 85c3135da..159f7c09f 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/generative-agent-object.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/generative-agent-object.ts @@ -113,7 +113,7 @@ export class GenerativeAgentObject { } async evaluate(evaluator: Evaluator, opts?: EvaluateOpts) { const { - sendTyping, + sendTyping = true, } = opts ?? {}; const evaluateAndExecuteStep = async () => { From cb91177b77a17ba58efdc8d4c3fb90c9f78d6c8c Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Wed, 25 Dec 2024 15:50:36 +0500 Subject: [PATCH 09/13] comment out conversation processMessageReactions for now --- .../classes/conversation-object.ts | 40 +++++++++---------- .../packages/react-agents/runtime.ts | 36 ++++++++--------- 2 files changed, 37 insertions(+), 39 deletions(-) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts index 8ff948f45..16bc2848d 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts @@ -360,26 +360,24 @@ export class ConversationObject extends EventTarget { ); } + // async processMessageReaction( + // reaction, + // messageId, + // userId, + // ) { + // console.log('process reaction: ',{ + // reaction, + // messageId, + // userId, + // }); - async processMessageReaction( - reaction, - messageId, - userId, - ) { - console.log('process reaction: ',{ - reaction, - messageId, - userId, - }); - - const e = new ExtendableMessageEvent('localMessageReaction', { - data: { - reaction, - messageId, - userId, - }, - }); - this.dispatchEvent(e); - } - + // const e = new ExtendableMessageEvent('localMessageReaction', { + // data: { + // reaction, + // messageId, + // userId, + // }, + // }); + // this.dispatchEvent(e); + // } } diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts index f2a35c468..f80a72c94 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts @@ -541,24 +541,24 @@ export const bindConversationToAgent = ({ })(); } }); - conversation.addEventListener('localMessageReaction', async (e: MessageReactionEvent) => { - const { reaction, messageId, userId } = e.data; - console.log('runtime localMessageReaction: ', { - reaction, - messageId, - userId, - }); - try { - const { - aborted, - } = await handleChatPerception(e.data, { - agent, - conversation, - }); - } catch (error) { - console.warn('caught new message reaction error', error); - } - }); + // conversation.addEventListener('localMessageReaction', async (e: MessageReactionEvent) => { + // const { reaction, messageId, userId } = e.data; + // console.log('runtime localMessageReaction: ', { + // reaction, + // messageId, + // userId, + // }); + // try { + // const { + // aborted, + // } = await handleChatPerception(e.data, { + // agent, + // conversation, + // }); + // } catch (error) { + // console.warn('caught new message reaction error', error); + // } + // }); }; // XXX can move this to the agent renderer From b28901345351f03d71ad1975a3307f1895c7eae6 Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Wed, 25 Dec 2024 16:10:27 +0500 Subject: [PATCH 10/13] init agent message reaction support --- .../react-agents/classes/discord-manager.ts | 18 +++++++++++++ .../react-agents/components/core/chat.tsx | 27 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts index b9b5d4b07..a6fd9bcb6 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts @@ -97,6 +97,24 @@ const bindOutgoing = ({ channelId, userId, }); + } else if (method === 'messageReaction') { + const { + reaction, + messageId, + userId, + } = args as { + reaction: string, + messageId: string, + userId: string, + }; + + console.log('discord manager message reaction', { + reaction, + messageId, + userId, + }); + + discordBotClient.input.reactToMessage(reaction, messageId, userId); } else { // ignore } diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/components/core/chat.tsx b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/core/chat.tsx index 605c40cf4..cb90752e4 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/components/core/chat.tsx +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/core/chat.tsx @@ -2,6 +2,7 @@ import React from 'react'; import dedent from 'dedent'; import { z } from 'zod'; import { Action } from './action'; +import { PendingActionEvent } from '../../types/react-agents'; export const ChatActions = () => { return ( @@ -27,6 +28,32 @@ export const ChatActions = () => { // await e.commit(); // }} /> + { + await e.commit(); + }} + /> ); }; \ No newline at end of file From dbbd1048cffebb3e4807b87e0e10feb62402f96f Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Wed, 25 Dec 2024 16:56:15 +0500 Subject: [PATCH 11/13] add agent -> discord bot react back support --- .../react-agents/classes/discord-manager.ts | 22 +++++++++++++++++-- .../lib/discord/discord-client.js | 16 ++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts index a6fd9bcb6..db64ae619 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts @@ -108,13 +108,31 @@ const bindOutgoing = ({ userId: string, }; + + // TODO: current agent mentionId needs to be set + const getDiscordIdForUserId = (userId: string) => { + const agents = conversation.getAgents(); + const currentAgent = conversation.agent; + const agent = agents.find( + agent => agent.playerId === userId + ) || (currentAgent.id === userId ? currentAgent : undefined); + + const discordId = agent?.playerSpec?.mentionId; + return discordId; + }; + + const discordId = getDiscordIdForUserId(userId); console.log('discord manager message reaction', { reaction, messageId, userId, + channelId, + discordId, + }); + discordBotClient.input.reactToMessage(reaction, messageId, { + channelId, + userId: discordId, }); - - discordBotClient.input.reactToMessage(reaction, messageId, userId); } else { // ignore } diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js b/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js index f0468f55b..592df8783 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js @@ -193,6 +193,22 @@ export class DiscordInput { this.queueManager.completeStream(streamId); } + reactToMessage(reaction, messageId, { + channelId, + userId, + } = {}) { + const m = { + method: 'reactToMessage', + args: { + channelId, + reaction, + messageId, + userId, + }, + }; + this.ws.send(JSON.stringify(m)); + } + destroy() { // Clean up any remaining buffers for (const streamId of this.bufferManager.buffers.keys()) { From 6148e0486ff30946820bae177c9ecf009c456d82 Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Thu, 26 Dec 2024 10:42:34 +0500 Subject: [PATCH 12/13] remove commented code --- .../classes/conversation-object.ts | 21 ------------------- .../packages/react-agents/runtime.ts | 18 ---------------- .../react-agents/types/react-agents.d.ts | 7 ------- 3 files changed, 46 deletions(-) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts index b32014d37..52212a4aa 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts @@ -366,25 +366,4 @@ export class ConversationObject extends EventTarget { }), ); } - - // async processMessageReaction( - // reaction, - // messageId, - // userId, - // ) { - // console.log('process reaction: ',{ - // reaction, - // messageId, - // userId, - // }); - - // const e = new ExtendableMessageEvent('localMessageReaction', { - // data: { - // reaction, - // messageId, - // userId, - // }, - // }); - // this.dispatchEvent(e); - // } } diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts index f80a72c94..407536f90 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/runtime.ts @@ -541,24 +541,6 @@ export const bindConversationToAgent = ({ })(); } }); - // conversation.addEventListener('localMessageReaction', async (e: MessageReactionEvent) => { - // const { reaction, messageId, userId } = e.data; - // console.log('runtime localMessageReaction: ', { - // reaction, - // messageId, - // userId, - // }); - // try { - // const { - // aborted, - // } = await handleChatPerception(e.data, { - // agent, - // conversation, - // }); - // } catch (error) { - // console.warn('caught new message reaction error', error); - // } - // }); }; // XXX can move this to the agent renderer diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts index e4b44f2c2..f905cfe79 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts @@ -534,13 +534,6 @@ export type ActionMessageEventData = { }; export type ActionMessageEvent = ExtendableMessageEvent; -export type MessageReactionEventData = { - reaction: string; - messageId: string; - userId: string; -}; -export type MessageReactionEvent = ExtendableMessageEvent; - export type ConversationChangeEventData = { conversation: ConversationObject; }; From b8e93fc76c44081dfab990121112697dada66929 Mon Sep 17 00:00:00 2001 From: Abdurrehman Subhani Date: Thu, 26 Dec 2024 22:05:00 +0500 Subject: [PATCH 13/13] improve the prompt --- .../packages/react-agents/components/core/chat.tsx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/components/core/chat.tsx b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/core/chat.tsx index 5f27ab63b..a3d396c87 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/components/core/chat.tsx +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/core/chat.tsx @@ -30,7 +30,15 @@ export const ChatActions = () => { />