From 1129c75c9e0d7a388ec2a95a238acd8719dcd128 Mon Sep 17 00:00:00 2001 From: "m.fukushima" Date: Tue, 30 Dec 2025 10:08:15 +0900 Subject: [PATCH 1/4] fix embed file encoding --- packages/common/Def.ts | 2 +- packages/main/src/AvatarState.ts | 7 ++++--- packages/main/src/generators/LmStudioGenerator.ts | 1 + packages/main/src/index.ts | 3 ++- packages/renderer/src/components/InputPanel.vue | 12 +++++++----- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/common/Def.ts b/packages/common/Def.ts index 8c45ca0a..09a73251 100644 --- a/packages/common/Def.ts +++ b/packages/common/Def.ts @@ -127,7 +127,7 @@ export const AsMessageContentSchema = Schema.partial( modelName: Schema.optional(Schema.String), nextGeneratorId: Schema.String, mediaUrl: Schema.String, - mediaBin: Schema.Any, // ArrayBuffer + mediaBin: Schema.String, // ArrayBuffer mimeType: Schema.String, toolName: Schema.String, toolReq: Schema.Any, diff --git a/packages/main/src/AvatarState.ts b/packages/main/src/AvatarState.ts index a835c5bf..251f19ae 100644 --- a/packages/main/src/AvatarState.ts +++ b/packages/main/src/AvatarState.ts @@ -799,8 +799,8 @@ export class AvatarState { return undefined; } if (mes.content.mediaBin && mes.content.mimeType?.startsWith('image/')) { - const img = Buffer.from(mes.content.mediaBin).toString('base64'); - const mediaUrl = yield* DocService.saveDocMedia(mes.id, mes.content.mimeType, img, it.templateId); + // const img = Buffer.from(mes.content.mediaBin).toString('base64'); + const mediaUrl = yield* DocService.saveDocMedia(mes.id, mes.content.mimeType, mes.content.mediaBin, it.templateId); return { ...mes, content: { @@ -816,7 +816,8 @@ export class AvatarState { content: { ...mes.content, mediaBin: undefined, - text: Buffer.from(mes.content.mediaBin).toString('utf-8'), + text: Buffer.from(mes.content.mediaBin,"base64").toString('utf-8'), + // text: Buffer.from(mes.content.mediaBin).toString('utf-8'), isExternal, }, } as AsMessage; diff --git a/packages/main/src/generators/LmStudioGenerator.ts b/packages/main/src/generators/LmStudioGenerator.ts index 03bce2f9..17665185 100644 --- a/packages/main/src/generators/LmStudioGenerator.ts +++ b/packages/main/src/generators/LmStudioGenerator.ts @@ -243,6 +243,7 @@ export class LmStudioTextGenerator extends LmStudioBaseGenerator { noTool?: boolean }): Effect.Effect { const it = this; + console.log('current:', JSON.stringify(current.input)); return Effect.gen(function* () { // モデルの最大コンテキスト長をまだ未取得だったら取得する make内では依存関係で呼ぶと形がずれるので。。 if (it.maxModelContextSize === -1) { diff --git a/packages/main/src/index.ts b/packages/main/src/index.ts index 976d83a0..1b5e38b9 100644 --- a/packages/main/src/index.ts +++ b/packages/main/src/index.ts @@ -112,13 +112,14 @@ ipcMain.handle('readMcpResource', async (_,avatarId: string,userName:string,name return [AsMessage.makeMessage({ from: userName, // @ts-ignore - mediaBin: Buffer.from(b.text as string).buffer, + mediaBin: Buffer.from(b.text as string).toString('base64'), mimeType: 'text/plain', }, 'talk', 'human', 'outer')] // 前提条件テキストにしたいが今はLLM直に送る形にする } // TODO 画像等 return [] }) + console.log('readMcpResource:',JSON.stringify(mesList)); return AvatarService.askAvatar(avatarId, mesList) } return Effect.succeed([]); diff --git a/packages/renderer/src/components/InputPanel.vue b/packages/renderer/src/components/InputPanel.vue index 567b39d9..321acfbe 100644 --- a/packages/renderer/src/components/InputPanel.vue +++ b/packages/renderer/src/components/InputPanel.vue @@ -29,19 +29,21 @@ async function sendMessage() { } if (fileUpload.value) { const arrayBuffer = await fileUpload.value.arrayBuffer(); - datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: arrayBuffer, mimeType:fileUpload.value.type,},'talk','human','outer')); // TODO 将来 outerからsurfaceにピックする処理が必要 + datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: Buffer.from(arrayBuffer).toString('base64'), mimeType:fileUpload.value.type,},'talk','human','outer')); // TODO 将来 outerからsurfaceにピックする処理が必要 // datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: arrayBuffer, mimeType:fileUpload.value.type,},'talk','human','outer')); } if(mcpResource.value) { mcpResource.value.contents.forEach(v => { if (v.mimeType === 'text/plain' && v.text) { - const encoder = new TextEncoder(); - const uint8Array = encoder.encode(v.text); - datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: uint8Array.buffer, mimeType:'text/plain',},'talk','human','outer')) + // const encoder = new TextEncoder(); + // const uint8Array = encoder.encode(v.text); + datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: Buffer.from(v.text).toString('base64'), mimeType:'text/plain',},'talk','human','outer')) + // datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: uint8Array.buffer, mimeType:'text/plain',},'talk','human','outer')) } else if (v.mimeType.startsWith('image')) { const encoder = new TextEncoder(); const uint8Array = encoder.encode(v.blob); - datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: uint8Array.buffer, mimeType:v.mimeType,},'talk','human','outer')) + datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: Buffer.from(uint8Array.buffer).toString('base64'), mimeType:v.mimeType,},'talk','human','outer')) + // datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: uint8Array.buffer, mimeType:v.mimeType,},'talk','human','outer')) } }) } From 298a35804d65276cf69ca5ee8d0b947bbeb71a4c Mon Sep 17 00:00:00 2001 From: "m.fukushima" Date: Tue, 30 Dec 2025 11:44:13 +0900 Subject: [PATCH 2/4] fix mcp tool def --- packages/main/src/McpService.ts | 46 +++++++++++++++++++ .../main/src/generators/LmStudioGenerator.ts | 1 + 2 files changed, 47 insertions(+) diff --git a/packages/main/src/McpService.ts b/packages/main/src/McpService.ts index 5cc9e006..b81375d6 100644 --- a/packages/main/src/McpService.ts +++ b/packages/main/src/McpService.ts @@ -133,7 +133,52 @@ export class McpService extends Effect.Service()('avatar-shell/McpSe */ function getToolDefs(mcpList: AvatarMcpSettingList) { + const definedServers = serverInfoList.filter(v => Object.keys(mcpList).includes(v.id)) + const undefinedServers = serverInfoList.filter(v => !Object.keys(mcpList).includes(v.id)) + const updateDefServers = Object.entries(mcpList).flatMap(a => { + const find = definedServers.find(v => v.id === a[0]); + if (find) { + if (!a[1].enable) { + return []; + } + const defTools = Object.entries(a[1].useTools).flatMap(d => { + if (d[1].enable) { + const f = find.tools.find(e => e.name === d[0]); + if (f) { + // ツールの個別関数が使えることが確定 + // プラグインの定義名を足す必要がある + return [ + { + ...f, + name: `${find.id}_${f.name}`, + }, + ]; + } + } + return []; + }); + const newTools = find.tools.filter(v => !Object.keys(a[1].useTools).includes(v.name)) + const addTools = newTools.map(v => { + return { + ...v, + name: `${find.id}_${v.name}` + } + }) + return defTools.concat(addTools) + } + return []; + }); + const undefFunctions = undefinedServers.flatMap(s => { + return s.tools.map(t => { + return { + ...t, + name: `${s.id}_${t.name}` + } + }) + }) + return updateDefServers.concat(undefFunctions) // このあたりはopenAiもanthropicも同様書式のはず +/* return Object.entries(mcpList).flatMap(a => { const find = getServerInfo(a[0]); if (find) { @@ -159,6 +204,7 @@ export class McpService extends Effect.Service()('avatar-shell/McpSe } return []; }); +*/ } diff --git a/packages/main/src/generators/LmStudioGenerator.ts b/packages/main/src/generators/LmStudioGenerator.ts index 17665185..aac9a46b 100644 --- a/packages/main/src/generators/LmStudioGenerator.ts +++ b/packages/main/src/generators/LmStudioGenerator.ts @@ -281,6 +281,7 @@ export class LmStudioTextGenerator extends LmStudioBaseGenerator { const contents = (it.systemPrompt || []).concat(prev,mes); console.log('LmStudio context:\n', contents.map(a => '##' + JSON.stringify(a).slice(0, 300)).join('\n')); console.log('LmStudio context end:'); + console.log('toolsIn',avatarState.Config.mcp,JSON.stringify(toolsIn)); const body: ResponseCreateParamsStreaming = { model: it.model, input: contents, From 42f435ead3d4b81bc57eefb36cc9bc52f99c1572 Mon Sep 17 00:00:00 2001 From: "m.fukushima" Date: Tue, 30 Dec 2025 20:11:41 +0900 Subject: [PATCH 3/4] enhance generator error handling --- .idea/codeStyles/Project.xml | 1 + packages/main/src/AvatarState.ts | 39 +++++++++++++++---- .../main/src/generators/LmStudioGenerator.ts | 2 +- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml index 84f4f3d2..693f4a0f 100644 --- a/.idea/codeStyles/Project.xml +++ b/.idea/codeStyles/Project.xml @@ -23,6 +23,7 @@