20251230

packages/common/Def.ts

@@ -127,7 +127,7 @@ export const AsMessageContentSchema = Schema.partial(
     modelName: Schema.optional(Schema.String),
     nextGeneratorId: Schema.String,
     mediaUrl: Schema.String,
-    mediaBin: Schema.Any, //  ArrayBuffer
+    mediaBin: Schema.String, //  ArrayBuffer
     mimeType: Schema.String,
     toolName: Schema.String,
     toolReq: Schema.Any,

packages/main/src/AvatarState.ts

@@ -799,8 +799,8 @@ export class AvatarState {
           return undefined;
         }
         if (mes.content.mediaBin && mes.content.mimeType?.startsWith('image/')) {
-          const img = Buffer.from(mes.content.mediaBin).toString('base64');
-          const mediaUrl = yield* DocService.saveDocMedia(mes.id, mes.content.mimeType, img, it.templateId);
+          // const img = Buffer.from(mes.content.mediaBin).toString('base64');
+          const mediaUrl = yield* DocService.saveDocMedia(mes.id, mes.content.mimeType, mes.content.mediaBin, it.templateId);
           return {
             ...mes,
             content: {
@@ -816,7 +816,8 @@ export class AvatarState {
             content: {
               ...mes.content,
               mediaBin: undefined,
-              text: Buffer.from(mes.content.mediaBin).toString('utf-8'),
+              text:  Buffer.from(mes.content.mediaBin,"base64").toString('utf-8'),
+              // text: Buffer.from(mes.content.mediaBin).toString('utf-8'),
               isExternal,
             },
           } as AsMessage;
@@ -867,6 +868,7 @@ export class AvatarState {
     // console.log('start execGeneratorLoop');
     const it = this;
     let loop = true;
+    let markId:string|undefined;
     return Effect.loop(true, {
       while: a => a,
       body: b => Effect.gen(function* () {
@@ -881,7 +883,7 @@ export class AvatarState {
           return;  //  func の無限ループを防ぐ
         }
         //  Generator処理
-        const markId = short.generate()
+        markId = short.generate()
         it.sendRunningMark(markId, true, inner.toGenerator.Name);
         const sysConfig = yield* ConfigService.getSysConfig();
         const gen = inner.toGenerator; //  settings?: ContextGeneratorSetting
@@ -902,6 +904,7 @@ export class AvatarState {
         // it.clearStreamingText();
         // console.log('genLoop gen io:', io.map(v => it.debugGenOuter(v)));
         it.sendRunningMark(markId, false);
+        markId = undefined
         if (io.length > 0) {
           yield* Queue.offerAll(it.outerQueue, io);
         } else {
@@ -910,7 +913,13 @@ export class AvatarState {
       }),
       step: b => loop,
       discard: true,
-    }).pipe(Effect.catchAll(a => Effect.logError('execGeneratorLoop error:', a.message, a.stack)));
+    }).pipe(Effect.catchAll(a => {
+      if(markId) {
+        it.sendRunningMark(markId,false)
+        markId = undefined
+      }
+      return Effect.fail(a);
+    }));
   }
 
   // enterOuter(outer: GenOuter) {
@@ -948,8 +957,8 @@ export class AvatarState {
       }),
       step: b => loop,
       discard: true,
-    }).pipe(
-      Effect.catchAll(a => Effect.logError('execExternalLoop error:', a)));
+    })//.pipe(
+      //Effect.catchAll(a => Effect.logError('execExternalLoop error:', a)));
   }
 
   solveMcp(list: GenOuter[]) {
@@ -1068,22 +1077,36 @@ export class AvatarState {
         // console.log('fiberInner status:', s);
         if (FiberStatus.isDone(s)) {
           it.fiberInner = yield* Effect.forkDaemon(it.execGeneratorLoop().pipe(
-            Effect.tapError(e => Effect.logError('fiberInner error:', e)), Effect.andThen(a => Effect.log('end gen fork'))));
+            Effect.tapError(e => {
+              it.showAlert(`generator Error:${e}`)
+              return Effect.logError('fiberInner error:', e);
+            }), Effect.andThen(a => {
+              return Effect.log('end gen fork');
+            })));
         }
       } else {
         it.fiberInner = yield* Effect.forkDaemon(it.execGeneratorLoop().pipe(
-          Effect.tapError(e => Effect.logError('fiberInner error:', e)), Effect.andThen(a => Effect.log('end gen fork'))));
+          Effect.tapError(e => {
+            it.showAlert(`generator Error:${e}`)
+            return Effect.logError('fiberInner error:', e);
+          }), Effect.andThen(a => Effect.log('end gen fork'))));
       }
       if (it.fiberOuter) {
         const s = yield* it.fiberOuter.status;
         // console.log('fiberOuter status:', s);
         if (FiberStatus.isDone(s)) {
           it.fiberOuter = yield* Effect.forkDaemon(it.execExternalLoop().pipe(
-            Effect.tapError(e => Effect.logError('fiberOuter error:', e)), Effect.andThen(a => Effect.log('end io fork'))));
+            Effect.tapError(e => {
+              it.showAlert(`generator Error:${e}`)
+              return Effect.logError('fiberOuter error:', e);
+            }), Effect.andThen(a => Effect.log('end io fork'))));
         }
       } else {
         it.fiberOuter = yield* Effect.forkDaemon(it.execExternalLoop().pipe(
-          Effect.tapError(e => Effect.logError('fiberOuter error:', e)), Effect.andThen(a => Effect.log('end io fork'))));
+          Effect.tapError(e => {
+            it.showAlert(`generator Error:${e}`)
+            return Effect.logError('fiberOuter error:', e);
+          }), Effect.andThen(a => Effect.log('end io fork'))));
       }
     });
   }

packages/main/src/McpService.ts

@@ -133,7 +133,52 @@ export class McpService extends Effect.Service<McpService>()('avatar-shell/McpSe
      */
 
     function getToolDefs(mcpList: AvatarMcpSettingList) {
+      const definedServers = serverInfoList.filter(v => Object.keys(mcpList).includes(v.id))
+      const undefinedServers = serverInfoList.filter(v => !Object.keys(mcpList).includes(v.id))
+      const updateDefServers = Object.entries(mcpList).flatMap(a => {
+        const find = definedServers.find(v => v.id === a[0]);
+        if (find) {
+          if (!a[1].enable) {
+            return [];
+          }
+          const defTools = Object.entries(a[1].useTools).flatMap(d => {
+            if (d[1].enable) {
+              const f = find.tools.find(e => e.name === d[0]);
+              if (f) {
+                //  ツールの個別関数が使えることが確定
+                //  プラグインの定義名を足す必要がある
+                return [
+                  {
+                    ...f,
+                    name: `${find.id}_${f.name}`,
+                  },
+                ];
+              }
+            }
+            return [];
+          });
+          const newTools = find.tools.filter(v => !Object.keys(a[1].useTools).includes(v.name))
+          const addTools = newTools.map(v => {
+            return {
+              ...v,
+              name: `${find.id}_${v.name}`
+            }
+          })
+          return defTools.concat(addTools)
+        }
+        return [];
+      });
+      const undefFunctions = undefinedServers.flatMap(s => {
+        return s.tools.map(t => {
+          return {
+            ...t,
+            name: `${s.id}_${t.name}`
+          }
+        })
+      })
+      return updateDefServers.concat(undefFunctions)
       //  このあたりはopenAiもanthropicも同様書式のはず
+/*
       return Object.entries(mcpList).flatMap(a => {
         const find = getServerInfo(a[0]);
         if (find) {
@@ -159,6 +204,7 @@ export class McpService extends Effect.Service<McpService>()('avatar-shell/McpSe
         }
         return [];
       });
+*/
 
     }
 

packages/main/src/generators/LmStudioGenerator.ts

@@ -243,6 +243,7 @@ export class LmStudioTextGenerator extends LmStudioBaseGenerator {
     noTool?: boolean
   }): Effect.Effect<GenOuter[], Error, ConfigService | McpService | DocService | MediaService|HttpClient.HttpClient> {
     const it = this;
+    console.log('current:', JSON.stringify(current.input));
     return Effect.gen(function* () {
       //  モデルの最大コンテキスト長をまだ未取得だったら取得する make内では依存関係で呼ぶと形がずれるので。。
       if (it.maxModelContextSize === -1) {
@@ -254,7 +255,7 @@ export class LmStudioTextGenerator extends LmStudioBaseGenerator {
         ).pipe(Effect.andThen(a => a.json),
           Effect.catchAll(e => {
             console.log('error:', e);
-            return Effect.succeed(e);
+            return Effect.fail(new Error(`lmStudio error:${e}`));
           }));
         console.log('LmStudioTextGenerator model:', response);
         it.maxModelContextSize = response.max_context_length;
@@ -280,6 +281,7 @@ export class LmStudioTextGenerator extends LmStudioBaseGenerator {
       const contents = (it.systemPrompt || []).concat(prev,mes);
       console.log('LmStudio context:\n', contents.map(a => '##' + JSON.stringify(a).slice(0, 300)).join('\n'));
       console.log('LmStudio context end:');
+      console.log('toolsIn',avatarState.Config.mcp,JSON.stringify(toolsIn));
       const body: ResponseCreateParamsStreaming = {
         model: it.model,
         input: contents,

packages/main/src/index.ts

@@ -112,13 +112,14 @@ ipcMain.handle('readMcpResource', async (_,avatarId: string,userName:string,name
             return [AsMessage.makeMessage({
               from: userName,
               //  @ts-ignore
-              mediaBin: Buffer.from(b.text as string).buffer,
+              mediaBin: Buffer.from(b.text as string).toString('base64'),
               mimeType: 'text/plain',
             }, 'talk', 'human', 'outer')] //  前提条件テキストにしたいが今はLLM直に送る形にする
           }
           //  TODO 画像等
           return []
         })
+        console.log('readMcpResource:',JSON.stringify(mesList));
         return AvatarService.askAvatar(avatarId, mesList)
       }
       return Effect.succeed([]);

packages/renderer/src/components/InputPanel.vue

@@ -29,19 +29,21 @@ async function sendMessage() {
   }
   if (fileUpload.value) {
     const arrayBuffer = await fileUpload.value.arrayBuffer();
-    datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: arrayBuffer, mimeType:fileUpload.value.type,},'talk','human','outer'));  //  TODO 将来 outerからsurfaceにピックする処理が必要
+    datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: Buffer.from(arrayBuffer).toString('base64'), mimeType:fileUpload.value.type,},'talk','human','outer'));  //  TODO 将来 outerからsurfaceにピックする処理が必要
     // datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: arrayBuffer, mimeType:fileUpload.value.type,},'talk','human','outer'));
   }
   if(mcpResource.value) {
     mcpResource.value.contents.forEach(v => {
       if (v.mimeType === 'text/plain' && v.text) {
-        const encoder = new TextEncoder();
-        const uint8Array = encoder.encode(v.text);
-        datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: uint8Array.buffer, mimeType:'text/plain',},'talk','human','outer'))
+        // const encoder = new TextEncoder();
+        // const uint8Array = encoder.encode(v.text);
+        datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: Buffer.from(v.text).toString('base64'), mimeType:'text/plain',},'talk','human','outer'))
+        // datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: uint8Array.buffer, mimeType:'text/plain',},'talk','human','outer'))
       } else if (v.mimeType.startsWith('image')) {
         const encoder = new TextEncoder();
         const uint8Array = encoder.encode(v.blob);
-        datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: uint8Array.buffer, mimeType:v.mimeType,},'talk','human','outer'))
+        datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: Buffer.from(uint8Array.buffer).toString('base64'), mimeType:v.mimeType,},'talk','human','outer'))
+        // datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: uint8Array.buffer, mimeType:v.mimeType,},'talk','human','outer'))
       }
     })
   }