diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/active-agent-object.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/active-agent-object.ts index e6eb86753..902dbed58 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/active-agent-object.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/active-agent-object.ts @@ -34,6 +34,7 @@ import { } from './conversation-manager'; import { PingManager } from './ping-manager'; import { AgentRegistry } from './render-registry'; +import { DataSourceManager } from './data-source-manager'; // @@ -51,7 +52,7 @@ export class ActiveAgentObject extends AgentObject { telnyxManager: TelnyxManager; pingManager: PingManager; generativeAgentsMap = new WeakMap(); - + dataSourceManager: DataSourceManager; // constructor( @@ -91,6 +92,9 @@ export class ActiveAgentObject extends AgentObject { codecs: appContextValue.useCodecs(), }); this.telnyxManager = new TelnyxManager(); + + this.dataSourceManager = new DataSourceManager(); + this.pingManager = new PingManager({ userId: this.id, supabase: this.useSupabase(), diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/agent-object.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/agent-object.ts index ce5347142..3cc91b12c 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/agent-object.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/agent-object.ts @@ -1,4 +1,7 @@ -import { AgentObjectData } from '../types'; +import { + AgentObjectData, + DataSourceConfig, +} from '../types'; export class AgentObject extends EventTarget { id: string; @@ -13,6 +16,8 @@ export class AgentObject extends EventTarget { features: string[]; address: string; stripeConnectAccountId: string; + dataSources?: Record; + constructor(config: AgentObjectData) { super(); @@ -31,6 +36,7 @@ export class AgentObject extends EventTarget { features, address, stripeConnectAccountId, + dataSources, }: AgentObjectData) { this.id = id; this.ownerId = ownerId; @@ -44,5 +50,6 @@ export class AgentObject extends EventTarget { this.features = features; this.address = address; this.stripeConnectAccountId = stripeConnectAccountId; + this.dataSources = dataSources; } -} \ No newline at end of file +} diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/api-data-source-manager.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/api-data-source-manager.ts new file mode 100644 index 000000000..85c8b32e9 --- /dev/null +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/api-data-source-manager.ts @@ -0,0 +1,59 @@ +import { z } from "zod"; +import { APIDataSourceProps } from "../types/react-agents"; +import { BaseDataSource } from '../types/react-agents'; + +export class APIDataSourceManager implements BaseDataSource { + id: string; + type: 'api'; + name: string; + description: string; + endpoint: string; + headers?: Record; + params?: Record; + requiredArgs?: string[]; + examples?: string[]; + schema: z.ZodSchema; + + constructor(config: APIDataSourceProps) { + this.id = config.id; + this.type = 'api'; + this.name = config.name || config.id; + this.description = config.description || ''; + this.endpoint = config.endpoint; + this.headers = config.headers; + this.params = config.params; + this.requiredArgs = config.requiredArgs; + this.examples = config.examples; + this.schema = config.schema; + } + + async pull(args: object = {}): Promise { + try { + // Validate args against schema + const validatedArgs = this.schema.parse(args); + + const url = new URL(this.endpoint); + const params = { ...this.params, ...validatedArgs }; + + Object.entries(params || {}).forEach(([key, value]) => { + url.searchParams.append(key, String(value)); + }); + + const response = await fetch(url.toString(), { + headers: this.headers, + }); + + if (!response.ok) { + throw new Error(`API request failed: ${response.statusText}`); + } + + return response.json(); + } catch (error) { + if (error instanceof z.ZodError) { + throw new Error(`Invalid arguments: ${error.message}`); + } + console.error(`Error fetching from API ${this.id}:`, error); + throw error; + } + } +} \ No newline at end of file diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/data-source-manager.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/data-source-manager.ts new file mode 100644 index 000000000..c97e93b54 --- /dev/null +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/data-source-manager.ts @@ -0,0 +1,34 @@ +import type { BaseDataSource } from '../types/react-agents'; + +export class DataSourceManager { + private dataSources: Map; + + constructor() { + this.dataSources = new Map(); + } + + addDataSource(dataSource: BaseDataSource): void { + this.dataSources.set(dataSource.id, dataSource); + } + + removeDataSource(id: string): boolean { + return this.dataSources.delete(id); + } + + getDataSource(id: string): BaseDataSource | undefined { + return this.dataSources.get(id); + } + + getAllDataSources(): BaseDataSource[] { + return Array.from(this.dataSources.values()); + } + + pullFromDataSource(id: string, args: object): Promise { + const source = this.getDataSource(id); + if (!source) { + throw new Error(`Data source ${id} not found`); + } + return source.pull(args); + } +} + diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/components/plugins/api-data-source.tsx b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/plugins/api-data-source.tsx new file mode 100644 index 000000000..9ae98d537 --- /dev/null +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/plugins/api-data-source.tsx @@ -0,0 +1,23 @@ +import { + BaseDataSource, + DataSourceType, + APIDataSourceProps, +} from '../../types/react-agents'; +import React, { useEffect } from 'react'; +import { useAgent } from '../../hooks'; +import { APIDataSourceManager } from '../../classes/api-data-source-manager'; + + +export const APIDataSource: React.FC = (props) => { + const agent = useAgent(); + + useEffect(() => { + const dataSource = new APIDataSourceManager(props); + agent.dataSourceManager.addDataSource(dataSource); + return () => { + agent.dataSourceManager.removeDataSource(dataSource.id); + }; + }, [props.endpoint, JSON.stringify(props.headers), JSON.stringify(props.params)]); + + return null; +}; \ No newline at end of file diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/components/plugins/data-source-learner.tsx b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/plugins/data-source-learner.tsx new file mode 100644 index 000000000..448d19163 --- /dev/null +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/plugins/data-source-learner.tsx @@ -0,0 +1,48 @@ +import React from 'react'; +import { Action } from '../core/action'; +import { Prompt } from '../core/prompt'; +import { useAgent } from '../../hooks'; +import { z } from 'zod'; +import dedent from 'dedent'; +import type { PendingActionEvent } from '../../types'; +import { AutoTask } from './auto-task'; +import { ConversationProvider } from '../core/conversation'; +import { RAGMemory } from './rag-memory'; + +export const DataSourceLearner: React.FC = () => { + const agent = useAgent(); + + return ( + <> + + {dedent`\ + # Data Source Learning System + + You can learn from available data sources and store the knowledge in your memory. + Use the queryAndLearn action when you need to: + - Gather new information about a topic + - Verify or update your existing knowledge + - Get real-time data for user queries + + Available data sources: + ${agent.dataSourceManager.getAllDataSources().map(source => dedent` + - ${source.name} (ID: ${source.id}) + ${source.description} + REQUIRED: You must include these arguments in your query: + ${source.requiredArgs?.map(arg => ` - '${arg}': (string) REQUIRED`) + .join('\n') || ' - No required arguments'}`).join('\n')} + + NOTE: Queries will fail if required arguments are not provided! + `} + + + {/* add the RAG Memory Component */} + + + + + {/* Add AutoTask for Self-learning from the data sources */} + + + ); +}; \ No newline at end of file diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/components/util/config-components.tsx b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/util/config-components.tsx index c3fd108d7..fd2cbf51f 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/components/util/config-components.tsx +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/util/config-components.tsx @@ -5,6 +5,9 @@ import type { import { featureRenderers, } from '../../util/agent-features-renderer'; +import { + dataSourceRenderers, +} from '../../util/data-source-renderer'; // defaults @@ -18,6 +21,8 @@ type ConfigAgentComponentProps = { */ export const ConfigAgentComponents = (props: ConfigAgentComponentProps) => { const features = props.config?.features ?? {}; + const dataSources = props.config?.dataSources ?? {}; + return ( <> {Object.keys(features).map((key) => { @@ -27,6 +32,15 @@ export const ConfigAgentComponents = (props: ConfigAgentComponentProps) => { ); })} + + {Object.entries(dataSources).map(([key, config]) => { + if (!config.type) { + throw new Error(`Data source ${key} is missing required 'type' field`); + } + const DataSourceRenderer = dataSourceRenderers[config.type]; + if (!DataSourceRenderer) return null; + return ; + })} ); }; \ No newline at end of file diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/components/util/default-components.tsx b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/util/default-components.tsx index 364df23f2..6f9a5b3f4 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/components/util/default-components.tsx +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/util/default-components.tsx @@ -74,6 +74,7 @@ const DefaultPrompts = () => { + @@ -91,6 +92,29 @@ const DefaultHeaderPrompt = () => { ); }; +const DataSourcesPrompt = () => { + const agent = useAgent(); + const dataSources = agent.dataSourceManager.getAllDataSources(); + + if (dataSources.length === 0) return null; + + return ( + + {dedent` + # Data Sources + You have access to the following data sources that you can query: + ${dataSources.map(source => dedent` + - ${source.name} (ID: ${source.id}) + Description: ${source.description} + Type: ${source.type} + ${source.type === 'api' ? `Required args: ${(source as any).requiredArgs}` : ''} + ${source.type === 'api' ? `Examples: ${(source as any).examples}` : ''} + `).join('\n')} + `} + + ); +}; + const ConversationEnvironmentPrompt = () => { return ( <> @@ -320,14 +344,6 @@ const InstructionsPrompt = () => { # Instructions Respond with the next action taken by your character: ${agent.name} The method/args of your response must match one of the allowed actions. - - Before choosing an action, decide if you should respond at all: - - Return null (no action) if: - * Message is clearly meant for others (unless you have crucial information) - * Your input wouldn't add value to the conversation - * The conversation is naturally concluding - * You've already responded frequently in the last few messages (2-3 messages max) - * Multiple other agents are already actively participating `} ); diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/index.tsx b/packages/usdk/packages/upstreet-agent/packages/react-agents/index.tsx index eb2d915e7..b7ede25ab 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/index.tsx +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/index.tsx @@ -25,5 +25,6 @@ export * from './components/plugins/video-perception'; export * from './loops/chat-loop'; export * from './loops/action-loop'; export * from './components/plugins/auto-task'; +export * from './components/plugins/data-source-learner'; export * from './hooks'; diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts index a1851d0a1..8e8a425f7 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts @@ -1,5 +1,5 @@ import type { ReactNode, FC, Ref } from 'react'; -import type { ZodTypeAny } from 'zod'; +import type { z, ZodTypeAny } from 'zod'; // intrinsics @@ -39,6 +39,7 @@ export type AgentObjectData = { features?: string[]; address?: string; stripeConnectAccountId?: string; + dataSources?: Record; }; export type AgentObject = EventTarget & AgentObjectData & { setConfig(config: AgentObjectData): void; @@ -90,6 +91,52 @@ export type ActionStep = { thought?: string; }; +// data sources + +export type DataSourceType = 'api' | 'text' | 'pdf'; + +export interface APIDataSource extends BaseDataSource { + type: 'api'; + endpoint: string; + headers?: Record; + params?: Record; +} +export interface BaseDataSource { + id: string; + type: DataSourceType; + name: string; + description: string; + pull(args: object): Promise; + requiredArgs?: string[]; +} + +export type DataSourceConfig = { + id: string; + type: DataSourceType; + name: string; + description: string; +}; + +export type DataSourceManager = EventTarget & { + addDataSource: (source: BaseDataSource) => void; + removeDataSource: (id: string) => boolean; + getDataSource: (id: string) => BaseDataSource | undefined; + getAllDataSources: () => BaseDataSource[]; + pullFromDataSource: (id: string, args: object) => Promise; +}; + +export interface APIDataSourceProps { + id: string; + name?: string; + description?: string; + endpoint: string; + examples: string[]; + schema: z.ZodSchema; // Add schema property + headers?: Record; + params?: Record; + requiredArgs?: string[]; +} + // messages export type ChatMessage = { @@ -449,6 +496,7 @@ export type ActiveAgentObject = AgentObject & { telnyxManager: TelnyxManager; pingManager: PingManager; generativeAgentsMap: WeakMap; + dataSourceManager: DataSourceManager; // diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/util/data-source-renderer.tsx b/packages/usdk/packages/upstreet-agent/packages/react-agents/util/data-source-renderer.tsx new file mode 100644 index 000000000..909290968 --- /dev/null +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/util/data-source-renderer.tsx @@ -0,0 +1,180 @@ +import React from 'react'; +import { APIDataSource } from '../components/plugins/api-data-source'; +import { z } from 'zod'; +import { Action } from '../components/core/action'; +import { PendingActionEvent } from '../classes/pending-action-event'; +import dedent from 'dedent'; + + +/* + example for adding a API data source in the agent.json: + "dataSources": [ + { + "id": "weather-api", + "name": "Weather API", + "type": "api", + "description": "Use this data source to get the current weather for a location by providing the required arguments", + "params": { + "key": "xxx" + }, + "endpoint": "https://api.weatherapi.com/v1/current.json", + "schema": { + "type": "object", + "properties": { + "q": { + "type": "string", + "description": "Location name or coordinates" + } + }, + "required": ["q"] + }, + "examples": [ + { + "q": "London" + } + ] + }, + { + "id": "pokedex-ability-api", + "name": "Pokédex Ability API", + "type": "api", + "description": "Use this data source to get information about Pokémon abilities by providing their name or ID number", + "endpoint": "https://pokeapi.co/api/v2/ability", + "schema": { + "type": "object", + "properties": { + "pokemon": { + "type": "string", + "description": "Pokemon name or ID number" + } + }, + "required": ["pokemon"] + }, + "examples": [ + { + "pokemon": "pikachu" + }, + { + "pokemon": "charizard" + } + ] + } + ], + +*/ +const zodFromJsonSchema = (schema: any, sourceId: string): z.ZodSchema => { + // Convert JSON schema to Zod schema + if (schema.type === 'object') { + const shape: Record = { + sourceId: z.literal(sourceId), + ...Object.fromEntries( + Object.entries(schema.properties || {}).map(([key, value]: [string, any]) => [ + key, + value.type === 'string' ? z.string() : z.any() + ]) + ) + }; + const baseSchema = z.object(shape); + return schema.required ? + baseSchema.required(['sourceId' as const, ...(schema.required as const[])]) : + baseSchema.required(['sourceId' as const]); + } + return z.any(); +}; + +export const dataSourceRenderers = { + api: ({ id, name, description, endpoint, headers, params, schema, examples }) => { + const requiredParams = { + id, + name, + description, + endpoint, + schema, + examples + }; + + const missing = Object.entries(requiredParams) + .filter(([_, value]) => !value) + .map(([key]) => key); + + if (missing.length > 0) { + throw new Error(`Data source ${id || 'unknown'} is missing required parameters: ${missing.join(', ')}`); + } + + const zodSchema = zodFromJsonSchema(schema, id); + + return ( + <> + + {/* APIDataSourceManager Action, maybe i can merge this action within the APIDataSource */} + { + const { message, agent } = e.data; + const { args } = message; + + try { + // Query the data source + const data = await agent.agent.dataSourceManager.pullFromDataSource( + args.sourceId, + args + ); + + console.log('data: ', data); + + // Analyze and summarize the data + const dataSummary = await agent.complete([ + { + role: 'user', + content: dedent`\ + Analyze and summarize the following data from ${args.sourceId}: + \`\`\` + ${JSON.stringify(data, null, 2)} + \`\`\` + + Context for why this data was queried: ${args.reason} + + Provide a concise summary focusing on the most relevant and interesting information. + `, + }, + ], { + model: agent.agent.smallModel, + }); + + console.log('data summary: ', dataSummary); + // Store the learned information in memory + await agent.agent.addMemory( + `Learned from ${args.sourceId}: ${dataSummary.content}`, + { + sourceId: args.sourceId, + query: args.args, + rawData: data, + summary: dataSummary.content, + timestamp: Date.now(), + reason: args.reason + } + ); + + await e.commit(); + } catch (error) { + console.error('Error in queryAndLearn:', error); + throw error; + } + }} + /> + + ); + }, +}; \ No newline at end of file diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/util/fetch.mjs b/packages/usdk/packages/upstreet-agent/packages/react-agents/util/fetch.mjs index ec934dee2..c5850726e 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/util/fetch.mjs +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/util/fetch.mjs @@ -415,6 +415,8 @@ export const fetchJsonCompletion = async ({ throw new Error('no jwt'); } + console.log('fetchJsonCompletion', messages); + const match = model.match(/^([^:]+?):/); if (match) { const modelType = match[1]; @@ -430,6 +432,7 @@ export const fetchJsonCompletion = async ({ }, { jwt, }); + console.log('result', result); return result; } else { throw new Error('invalid model type: ' + JSON.stringify(modelType));