diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml
index 84f4f3d2..693f4a0f 100644
--- a/.idea/codeStyles/Project.xml
+++ b/.idea/codeStyles/Project.xml
@@ -23,6 +23,7 @@
+
diff --git a/packages/common/Def.ts b/packages/common/Def.ts
index 8c45ca0a..09a73251 100644
--- a/packages/common/Def.ts
+++ b/packages/common/Def.ts
@@ -127,7 +127,7 @@ export const AsMessageContentSchema = Schema.partial(
modelName: Schema.optional(Schema.String),
nextGeneratorId: Schema.String,
mediaUrl: Schema.String,
- mediaBin: Schema.Any, // ArrayBuffer
+ mediaBin: Schema.String, // ArrayBuffer
mimeType: Schema.String,
toolName: Schema.String,
toolReq: Schema.Any,
diff --git a/packages/main/src/AvatarState.ts b/packages/main/src/AvatarState.ts
index a835c5bf..204dd488 100644
--- a/packages/main/src/AvatarState.ts
+++ b/packages/main/src/AvatarState.ts
@@ -799,8 +799,8 @@ export class AvatarState {
return undefined;
}
if (mes.content.mediaBin && mes.content.mimeType?.startsWith('image/')) {
- const img = Buffer.from(mes.content.mediaBin).toString('base64');
- const mediaUrl = yield* DocService.saveDocMedia(mes.id, mes.content.mimeType, img, it.templateId);
+ // const img = Buffer.from(mes.content.mediaBin).toString('base64');
+ const mediaUrl = yield* DocService.saveDocMedia(mes.id, mes.content.mimeType, mes.content.mediaBin, it.templateId);
return {
...mes,
content: {
@@ -816,7 +816,8 @@ export class AvatarState {
content: {
...mes.content,
mediaBin: undefined,
- text: Buffer.from(mes.content.mediaBin).toString('utf-8'),
+ text: Buffer.from(mes.content.mediaBin,"base64").toString('utf-8'),
+ // text: Buffer.from(mes.content.mediaBin).toString('utf-8'),
isExternal,
},
} as AsMessage;
@@ -867,6 +868,7 @@ export class AvatarState {
// console.log('start execGeneratorLoop');
const it = this;
let loop = true;
+ let markId:string|undefined;
return Effect.loop(true, {
while: a => a,
body: b => Effect.gen(function* () {
@@ -881,7 +883,7 @@ export class AvatarState {
return; // func の無限ループを防ぐ
}
// Generator処理
- const markId = short.generate()
+ markId = short.generate()
it.sendRunningMark(markId, true, inner.toGenerator.Name);
const sysConfig = yield* ConfigService.getSysConfig();
const gen = inner.toGenerator; // settings?: ContextGeneratorSetting
@@ -902,6 +904,7 @@ export class AvatarState {
// it.clearStreamingText();
// console.log('genLoop gen io:', io.map(v => it.debugGenOuter(v)));
it.sendRunningMark(markId, false);
+ markId = undefined
if (io.length > 0) {
yield* Queue.offerAll(it.outerQueue, io);
} else {
@@ -910,7 +913,13 @@ export class AvatarState {
}),
step: b => loop,
discard: true,
- }).pipe(Effect.catchAll(a => Effect.logError('execGeneratorLoop error:', a.message, a.stack)));
+ }).pipe(Effect.catchAll(a => {
+ if(markId) {
+ it.sendRunningMark(markId,false)
+ markId = undefined
+ }
+ return Effect.fail(a);
+ }));
}
// enterOuter(outer: GenOuter) {
@@ -948,8 +957,8 @@ export class AvatarState {
}),
step: b => loop,
discard: true,
- }).pipe(
- Effect.catchAll(a => Effect.logError('execExternalLoop error:', a)));
+ })//.pipe(
+ //Effect.catchAll(a => Effect.logError('execExternalLoop error:', a)));
}
solveMcp(list: GenOuter[]) {
@@ -1068,22 +1077,36 @@ export class AvatarState {
// console.log('fiberInner status:', s);
if (FiberStatus.isDone(s)) {
it.fiberInner = yield* Effect.forkDaemon(it.execGeneratorLoop().pipe(
- Effect.tapError(e => Effect.logError('fiberInner error:', e)), Effect.andThen(a => Effect.log('end gen fork'))));
+ Effect.tapError(e => {
+ it.showAlert(`generator Error:${e}`)
+ return Effect.logError('fiberInner error:', e);
+ }), Effect.andThen(a => {
+ return Effect.log('end gen fork');
+ })));
}
} else {
it.fiberInner = yield* Effect.forkDaemon(it.execGeneratorLoop().pipe(
- Effect.tapError(e => Effect.logError('fiberInner error:', e)), Effect.andThen(a => Effect.log('end gen fork'))));
+ Effect.tapError(e => {
+ it.showAlert(`generator Error:${e}`)
+ return Effect.logError('fiberInner error:', e);
+ }), Effect.andThen(a => Effect.log('end gen fork'))));
}
if (it.fiberOuter) {
const s = yield* it.fiberOuter.status;
// console.log('fiberOuter status:', s);
if (FiberStatus.isDone(s)) {
it.fiberOuter = yield* Effect.forkDaemon(it.execExternalLoop().pipe(
- Effect.tapError(e => Effect.logError('fiberOuter error:', e)), Effect.andThen(a => Effect.log('end io fork'))));
+ Effect.tapError(e => {
+ it.showAlert(`generator Error:${e}`)
+ return Effect.logError('fiberOuter error:', e);
+ }), Effect.andThen(a => Effect.log('end io fork'))));
}
} else {
it.fiberOuter = yield* Effect.forkDaemon(it.execExternalLoop().pipe(
- Effect.tapError(e => Effect.logError('fiberOuter error:', e)), Effect.andThen(a => Effect.log('end io fork'))));
+ Effect.tapError(e => {
+ it.showAlert(`generator Error:${e}`)
+ return Effect.logError('fiberOuter error:', e);
+ }), Effect.andThen(a => Effect.log('end io fork'))));
}
});
}
diff --git a/packages/main/src/McpService.ts b/packages/main/src/McpService.ts
index 5cc9e006..b81375d6 100644
--- a/packages/main/src/McpService.ts
+++ b/packages/main/src/McpService.ts
@@ -133,7 +133,52 @@ export class McpService extends Effect.Service()('avatar-shell/McpSe
*/
function getToolDefs(mcpList: AvatarMcpSettingList) {
+ const definedServers = serverInfoList.filter(v => Object.keys(mcpList).includes(v.id))
+ const undefinedServers = serverInfoList.filter(v => !Object.keys(mcpList).includes(v.id))
+ const updateDefServers = Object.entries(mcpList).flatMap(a => {
+ const find = definedServers.find(v => v.id === a[0]);
+ if (find) {
+ if (!a[1].enable) {
+ return [];
+ }
+ const defTools = Object.entries(a[1].useTools).flatMap(d => {
+ if (d[1].enable) {
+ const f = find.tools.find(e => e.name === d[0]);
+ if (f) {
+ // ツールの個別関数が使えることが確定
+ // プラグインの定義名を足す必要がある
+ return [
+ {
+ ...f,
+ name: `${find.id}_${f.name}`,
+ },
+ ];
+ }
+ }
+ return [];
+ });
+ const newTools = find.tools.filter(v => !Object.keys(a[1].useTools).includes(v.name))
+ const addTools = newTools.map(v => {
+ return {
+ ...v,
+ name: `${find.id}_${v.name}`
+ }
+ })
+ return defTools.concat(addTools)
+ }
+ return [];
+ });
+ const undefFunctions = undefinedServers.flatMap(s => {
+ return s.tools.map(t => {
+ return {
+ ...t,
+ name: `${s.id}_${t.name}`
+ }
+ })
+ })
+ return updateDefServers.concat(undefFunctions)
// このあたりはopenAiもanthropicも同様書式のはず
+/*
return Object.entries(mcpList).flatMap(a => {
const find = getServerInfo(a[0]);
if (find) {
@@ -159,6 +204,7 @@ export class McpService extends Effect.Service()('avatar-shell/McpSe
}
return [];
});
+*/
}
diff --git a/packages/main/src/generators/LmStudioGenerator.ts b/packages/main/src/generators/LmStudioGenerator.ts
index 03bce2f9..bd27aa64 100644
--- a/packages/main/src/generators/LmStudioGenerator.ts
+++ b/packages/main/src/generators/LmStudioGenerator.ts
@@ -243,6 +243,7 @@ export class LmStudioTextGenerator extends LmStudioBaseGenerator {
noTool?: boolean
}): Effect.Effect {
const it = this;
+ console.log('current:', JSON.stringify(current.input));
return Effect.gen(function* () {
// モデルの最大コンテキスト長をまだ未取得だったら取得する make内では依存関係で呼ぶと形がずれるので。。
if (it.maxModelContextSize === -1) {
@@ -254,7 +255,7 @@ export class LmStudioTextGenerator extends LmStudioBaseGenerator {
).pipe(Effect.andThen(a => a.json),
Effect.catchAll(e => {
console.log('error:', e);
- return Effect.succeed(e);
+ return Effect.fail(new Error(`lmStudio error:${e}`));
}));
console.log('LmStudioTextGenerator model:', response);
it.maxModelContextSize = response.max_context_length;
@@ -280,6 +281,7 @@ export class LmStudioTextGenerator extends LmStudioBaseGenerator {
const contents = (it.systemPrompt || []).concat(prev,mes);
console.log('LmStudio context:\n', contents.map(a => '##' + JSON.stringify(a).slice(0, 300)).join('\n'));
console.log('LmStudio context end:');
+ console.log('toolsIn',avatarState.Config.mcp,JSON.stringify(toolsIn));
const body: ResponseCreateParamsStreaming = {
model: it.model,
input: contents,
diff --git a/packages/main/src/index.ts b/packages/main/src/index.ts
index 976d83a0..1b5e38b9 100644
--- a/packages/main/src/index.ts
+++ b/packages/main/src/index.ts
@@ -112,13 +112,14 @@ ipcMain.handle('readMcpResource', async (_,avatarId: string,userName:string,name
return [AsMessage.makeMessage({
from: userName,
// @ts-ignore
- mediaBin: Buffer.from(b.text as string).buffer,
+ mediaBin: Buffer.from(b.text as string).toString('base64'),
mimeType: 'text/plain',
}, 'talk', 'human', 'outer')] // 前提条件テキストにしたいが今はLLM直に送る形にする
}
// TODO 画像等
return []
})
+ console.log('readMcpResource:',JSON.stringify(mesList));
return AvatarService.askAvatar(avatarId, mesList)
}
return Effect.succeed([]);
diff --git a/packages/renderer/src/components/InputPanel.vue b/packages/renderer/src/components/InputPanel.vue
index 567b39d9..321acfbe 100644
--- a/packages/renderer/src/components/InputPanel.vue
+++ b/packages/renderer/src/components/InputPanel.vue
@@ -29,19 +29,21 @@ async function sendMessage() {
}
if (fileUpload.value) {
const arrayBuffer = await fileUpload.value.arrayBuffer();
- datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: arrayBuffer, mimeType:fileUpload.value.type,},'talk','human','outer')); // TODO 将来 outerからsurfaceにピックする処理が必要
+ datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: Buffer.from(arrayBuffer).toString('base64'), mimeType:fileUpload.value.type,},'talk','human','outer')); // TODO 将来 outerからsurfaceにピックする処理が必要
// datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: arrayBuffer, mimeType:fileUpload.value.type,},'talk','human','outer'));
}
if(mcpResource.value) {
mcpResource.value.contents.forEach(v => {
if (v.mimeType === 'text/plain' && v.text) {
- const encoder = new TextEncoder();
- const uint8Array = encoder.encode(v.text);
- datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: uint8Array.buffer, mimeType:'text/plain',},'talk','human','outer'))
+ // const encoder = new TextEncoder();
+ // const uint8Array = encoder.encode(v.text);
+ datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: Buffer.from(v.text).toString('base64'), mimeType:'text/plain',},'talk','human','outer'))
+ // datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: uint8Array.buffer, mimeType:'text/plain',},'talk','human','outer'))
} else if (v.mimeType.startsWith('image')) {
const encoder = new TextEncoder();
const uint8Array = encoder.encode(v.blob);
- datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: uint8Array.buffer, mimeType:v.mimeType,},'talk','human','outer'))
+ datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: Buffer.from(uint8Array.buffer).toString('base64'), mimeType:v.mimeType,},'talk','human','outer'))
+ // datas.push(AsMessage.makeMessage({from: getUserName(), mediaBin: uint8Array.buffer, mimeType:v.mimeType,},'talk','human','outer'))
}
})
}