From fe938fa4c49f9dca8ecc9d0f4076099600e88a21 Mon Sep 17 00:00:00 2001 From: akitaSummer Date: Sun, 25 Jan 2026 22:14:12 +0800 Subject: [PATCH] feat: add mysql file system --- core/langchain-decorator/index.ts | 1 + core/langchain-decorator/package.json | 1 + .../src/qualifier/FileSystemQualifier.ts | 10 + plugin/langchain/index.ts | 2 +- .../lib/filesystem/MysqlFilesystem.ts | 713 +++++++++ .../lib/filesystem/agentfs/agentfs.ts | 1328 ++++++++++++++++ .../lib/filesystem/agentfs/errors.ts | 52 + .../lib/filesystem/agentfs/guards.ts | 216 +++ .../lib/filesystem/agentfs/interface.ts | 243 +++ plugin/langchain/lib/util.ts | 13 +- plugin/langchain/package.json | 5 + .../modules/bar/controller/AppController.ts | 7 +- .../apps/langchain/app/modules/bar/module.yml | 19 +- .../app/modules/bar/service/Graph.ts | 2 + .../fixtures/apps/langchain/config/plugin.js | 5 + .../test/fixtures/sse-mcp-server/http.ts | 2 + plugin/langchain/test/llm.test.ts | 1338 ++++++++++++++++- plugin/langchain/typings/index.d.ts | 20 + 18 files changed, 3959 insertions(+), 18 deletions(-) create mode 100644 core/langchain-decorator/src/qualifier/FileSystemQualifier.ts create mode 100644 plugin/langchain/lib/filesystem/MysqlFilesystem.ts create mode 100644 plugin/langchain/lib/filesystem/agentfs/agentfs.ts create mode 100644 plugin/langchain/lib/filesystem/agentfs/errors.ts create mode 100644 plugin/langchain/lib/filesystem/agentfs/guards.ts create mode 100644 plugin/langchain/lib/filesystem/agentfs/interface.ts diff --git a/core/langchain-decorator/index.ts b/core/langchain-decorator/index.ts index 4b57db45..a587a22b 100644 --- a/core/langchain-decorator/index.ts +++ b/core/langchain-decorator/index.ts @@ -24,4 +24,5 @@ export * from './src/util/BoundModelInfoUtil'; export * from './src/qualifier/ChatModelQualifier'; export * from './src/qualifier/ChatCheckpointSaverQualifier'; +export * from './src/qualifier/FileSystemQualifier'; export * from './src/type/metadataKey'; diff --git a/core/langchain-decorator/package.json b/core/langchain-decorator/package.json index 0d59867d..bde66a8c 100644 --- a/core/langchain-decorator/package.json +++ b/core/langchain-decorator/package.json @@ -44,6 +44,7 @@ "@langchain/core": "^1.1.1", "@langchain/langgraph": "^1.0.2", "@langchain/openai": "^1.1.0", + "deepagents": "^1.5.1", "langchain": "^1.1.2", "lodash": "^4.17.21", "pluralize": "^8.0.0" diff --git a/core/langchain-decorator/src/qualifier/FileSystemQualifier.ts b/core/langchain-decorator/src/qualifier/FileSystemQualifier.ts new file mode 100644 index 00000000..678a813a --- /dev/null +++ b/core/langchain-decorator/src/qualifier/FileSystemQualifier.ts @@ -0,0 +1,10 @@ +import { QualifierUtil } from '@eggjs/tegg'; + +export const FileSystemQualifierAttribute = Symbol.for('Qualifier.FileSystem'); +export const FileSystemInjectName = 'teggFilesystem'; + +export function FileSystemQualifier(fsMiddlewareName: string) { + return function(target: any, propertyKey?: PropertyKey, parameterIndex?: number) { + QualifierUtil.addInjectQualifier(target, propertyKey, parameterIndex, FileSystemQualifierAttribute, fsMiddlewareName); + }; +} diff --git a/plugin/langchain/index.ts b/plugin/langchain/index.ts index 8d0be974..51a20591 100644 --- a/plugin/langchain/index.ts +++ b/plugin/langchain/index.ts @@ -8,4 +8,4 @@ export * from './lib/graph/GraphLoadUnitHook'; export * from './lib/graph/GraphObjectHook'; export * from './lib/graph/GraphPrototypeHook'; export * from './lib/tracing/LangGraphTracer'; - +export * from './lib/filesystem/MysqlFilesystem'; diff --git a/plugin/langchain/lib/filesystem/MysqlFilesystem.ts b/plugin/langchain/lib/filesystem/MysqlFilesystem.ts new file mode 100644 index 00000000..c1125a74 --- /dev/null +++ b/plugin/langchain/lib/filesystem/MysqlFilesystem.ts @@ -0,0 +1,713 @@ +import { AccessLevel, ConfigSourceQualifierAttribute, Inject, LifecyclePostInject, LoadUnitNameQualifierAttribute, ModuleConfig, MultiInstanceInfo, MultiInstanceProto, MultiInstancePrototypeGetObjectsContext, ObjectInfo, ObjectInitType } from '@eggjs/tegg'; +import { + BackendProtocol, + EditResult, + FileData, + FileDownloadResponse, + FileInfo, + FileUploadResponse, + GrepMatch, + WriteResult, +} from 'deepagents'; +import micromatch from 'micromatch'; +import { FileSystemInjectName, FileSystemQualifierAttribute } from '@eggjs/tegg-langchain-decorator'; +import { ModuleConfigUtil } from '@eggjs/tegg/helper'; +import { MysqlDataSourceManager } from '@eggjs/tegg/dal'; +import { getClientNames, getFileSystemConfig } from '../util'; +import assert from 'node:assert'; +import { AgentFS } from './agentfs/agentfs'; +import path, { basename } from 'node:path'; +import { MysqlDataSource } from '@eggjs/dal-runtime'; +import fsSync from 'node:fs'; +import { Stats } from './agentfs/interface'; + +const SUPPORTS_NOFOLLOW = fsSync.constants.O_NOFOLLOW !== undefined; + +export const EMPTY_CONTENT_WARNING = + 'System reminder: File exists but has empty contents'; +export const MAX_LINE_LENGTH = 10000; +export const LINE_NUMBER_WIDTH = 6; +export const TOOL_RESULT_TOKEN_LIMIT = 20000; // Same threshold as eviction +export const TRUNCATION_GUIDANCE = + '... [results truncated, try being more specific with your parameters]'; + +@MultiInstanceProto({ + accessLevel: AccessLevel.PUBLIC, + initType: ObjectInitType.SINGLETON, + // 从 module.yml 中动态获取配置来决定需要初始化几个对象 + getObjects(ctx: MultiInstancePrototypeGetObjectsContext) { + const config: ModuleConfig = ModuleConfigUtil.loadModuleConfigSync( + ctx.unitPath, + ); + const moduleName = ModuleConfigUtil.readModuleNameSync(ctx.unitPath); + return getClientNames(config, 'filesystem').map(name => { + return { + name: FileSystemInjectName, + qualifiers: + MysqlFilesystem.getFsQualifier(name)[ + FileSystemInjectName + ], + properQualifiers: { + moduleConfig: [{ + attribute: ConfigSourceQualifierAttribute, + value: moduleName, + }], + }, + }; + }); + }, +}) +export class MysqlFilesystem implements BackendProtocol { + moduleName: string; + dataSourceName: string; + cwd: string; + virtualMode = false; + agentFs: AgentFS; + mysql: MysqlDataSource; + + constructor( + @Inject() readonly moduleConfig: ModuleConfig, + @Inject() mysqlDataSourceManager: MysqlDataSourceManager, + + @MultiInstanceInfo([ + FileSystemQualifierAttribute, + LoadUnitNameQualifierAttribute, + ]) + objInfo: ObjectInfo, + ) { + this.moduleName = objInfo.qualifiers.find( + t => t.attribute === LoadUnitNameQualifierAttribute, + )?.value as string; + assert(this.moduleName, 'not found FsMiddleware name'); + + const fsConfig = getFileSystemConfig(moduleConfig, objInfo); + + this.dataSourceName = fsConfig.dataSource; + + this.cwd = fsConfig.cwd || '/'; + this.virtualMode = fsConfig.virtualMode === true; + + const mysql = mysqlDataSourceManager.get( + this.moduleName, + this.dataSourceName, + ); + if (!mysql) { + throw new Error( + `not found mysql datasource for module: ${this.moduleName}, dataSource: ${this.dataSourceName}`, + ); + } + this.mysql = mysql; + } + + @LifecyclePostInject() + async init() { + this.agentFs = await AgentFS.fromDatabase(this.mysql); + } + + + private resolvePath(key: string): string { + if (this.virtualMode) { + const vpath = key.startsWith('/') ? key : '/' + key; + if (vpath.includes('..') || vpath.startsWith('~')) { + throw new Error('Path traversal not allowed'); + } + const full = path.resolve(this.cwd, vpath.substring(1)); + const relative = path.relative(this.cwd, full); + if (relative.startsWith('..') || path.isAbsolute(relative)) { + throw new Error(`Path: ${full} outside root directory: ${this.cwd}`); + } + return full; + } + + if (path.isAbsolute(key)) { + return key; + } + return path.resolve(this.cwd, key); + } + + async lsInfo(dirPath: string): Promise { + try { + const resolvedPath = this.resolvePath(dirPath); + const stat = await this.agentFs.stat(resolvedPath); + + if (!stat.isDirectory()) { + return []; + } + + const entries = await this.agentFs.readdir(resolvedPath); + const results: FileInfo[] = []; + + const cwdStr = this.cwd.endsWith(path.sep) + ? this.cwd + : this.cwd + path.sep; + + for (const entry of entries) { + const fullPath = path.join(resolvedPath, entry); + + try { + const entryStat = await this.agentFs.stat(fullPath); + const isFile = entryStat.isFile(); + const isDir = entryStat.isDirectory(); + + if (!this.virtualMode) { + // Non-virtual mode: use absolute paths + if (isFile) { + results.push({ + path: fullPath, + is_dir: false, + size: entryStat.size, + modified_at: new Date(entryStat.mtime).toString(), + }); + } else if (isDir) { + results.push({ + path: fullPath + path.sep, + is_dir: true, + size: 0, + modified_at: new Date(entryStat.mtime).toString(), + }); + } + } else { + let relativePath: string; + if (fullPath.startsWith(cwdStr)) { + relativePath = fullPath.substring(cwdStr.length); + } else if (fullPath.startsWith(this.cwd)) { + relativePath = fullPath + .substring(this.cwd.length) + .replace(/^[/\\]/, ''); + } else { + relativePath = fullPath; + } + + relativePath = relativePath.split(path.sep).join('/'); + const virtPath = '/' + relativePath; + + if (isFile) { + results.push({ + path: virtPath, + is_dir: false, + size: entryStat.size, + modified_at: new Date(entryStat.mtime).toString(), + }); + } else if (isDir) { + results.push({ + path: virtPath + '/', + is_dir: true, + size: 0, + modified_at: new Date(entryStat.mtime).toString(), + }); + } + } + } catch { + continue; + } + } + + results.sort((a, b) => a.path.localeCompare(b.path)); + return results; + } catch { + return []; + } + } + async read( + filePath: string, + offset = 0, + limit = 500, + ): Promise { + try { + const resolvedPath = this.resolvePath(filePath); + + let content: string; + + if (SUPPORTS_NOFOLLOW) { + const stat = await this.agentFs.stat(resolvedPath); + if (!stat.isFile()) { + return `Error: File '${filePath}' not found`; + } + // const fd = await this.agentFs.open( + // resolvedPath, + // ); + // try { + // content = await fd.readFile({ encoding: 'utf-8' }); + // } finally { + // await fd.close(); + // } + content = await this.agentFs.readFile(resolvedPath, 'utf-8'); + } else { + const stat = await this.agentFs.lstat(resolvedPath); + if (stat.isSymbolicLink()) { + return `Error: Symlinks are not allowed: ${filePath}`; + } + if (!stat.isFile()) { + return `Error: File '${filePath}' not found`; + } + content = await this.agentFs.readFile(resolvedPath, 'utf-8'); + } + + const emptyMsg = this.checkEmptyContent(content); + if (emptyMsg) { + return emptyMsg; + } + + const lines = content.split('\n'); + const startIdx = offset; + const endIdx = Math.min(startIdx + limit, lines.length); + + if (startIdx >= lines.length) { + return `Error: Line offset ${offset} exceeds file length (${lines.length} lines)`; + } + + const selectedLines = lines.slice(startIdx, endIdx); + return this.formatContentWithLineNumbers(selectedLines, startIdx + 1); + } catch (e: any) { + return `Error reading file '${filePath}': ${e.message}`; + } + } + + checkEmptyContent(content: string): string | null { + if (!content || content.trim() === '') { + return EMPTY_CONTENT_WARNING; + } + return null; + } + + formatContentWithLineNumbers( + content: string | string[], + startLine = 1, + ): string { + let lines: string[]; + if (typeof content === 'string') { + lines = content.split('\n'); + if (lines.length > 0 && lines[lines.length - 1] === '') { + lines = lines.slice(0, -1); + } + } else { + lines = content; + } + + const resultLines: string[] = []; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineNum = i + startLine; + + if (line.length <= MAX_LINE_LENGTH) { + resultLines.push( + `${lineNum.toString().padStart(LINE_NUMBER_WIDTH)}\t${line}`, + ); + } else { + // Split long line into chunks with continuation markers + const numChunks = Math.ceil(line.length / MAX_LINE_LENGTH); + for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) { + const start = chunkIdx * MAX_LINE_LENGTH; + const end = Math.min(start + MAX_LINE_LENGTH, line.length); + const chunk = line.substring(start, end); + if (chunkIdx === 0) { + // First chunk: use normal line number + resultLines.push( + `${lineNum.toString().padStart(LINE_NUMBER_WIDTH)}\t${chunk}`, + ); + } else { + // Continuation chunks: use decimal notation (e.g., 5.1, 5.2) + const continuationMarker = `${lineNum}.${chunkIdx}`; + resultLines.push( + `${continuationMarker.padStart(LINE_NUMBER_WIDTH)}\t${chunk}`, + ); + } + } + } + } + + return resultLines.join('\n'); + } + async readRaw(filePath: string): Promise { + const resolvedPath = this.resolvePath(filePath); + + let content: string; + let stat: Stats; + + if (SUPPORTS_NOFOLLOW) { + stat = await this.agentFs.stat(resolvedPath); + if (!stat.isFile()) throw new Error(`File '${filePath}' not found`); + content = await this.agentFs.readFile(resolvedPath, 'utf-8'); + } else { + stat = await this.agentFs.lstat(resolvedPath); + if (stat.isSymbolicLink()) { + throw new Error(`Symlinks are not allowed: ${filePath}`); + } + if (!stat.isFile()) throw new Error(`File '${filePath}' not found`); + content = await this.agentFs.readFile(resolvedPath, 'utf-8'); + } + + return { + content: content.split('\n'), + created_at: new Date(stat.ctime).toString(), + modified_at: new Date(stat.mtime).toString(), + }; + } + async grepRaw( + pattern: string, + path?: string | null, + glob?: string | null, + ): Promise { + const files = await this.agentFs.readallFiles(this.cwd); + return this.grepMatchesFromFiles(files, pattern, path, glob); + } + grepMatchesFromFiles( + files: Record, + pattern: string, + path: string | null = null, + glob: string | null = null, + ): GrepMatch[] | string { + let regex: RegExp; + try { + regex = new RegExp(pattern); + } catch (e: any) { + return `Invalid regex pattern: ${e.message}`; + } + + let normalizedPath: string; + try { + normalizedPath = this.validatePath(path); + } catch { + return []; + } + + let filtered = Object.fromEntries( + Object.entries(files).filter(([ fp ]) => fp.startsWith(normalizedPath)), + ); + + if (glob) { + filtered = Object.fromEntries( + Object.entries(filtered).filter(([ fp ]) => + micromatch.isMatch(basename(fp), glob, { dot: true, nobrace: false }), + ), + ); + } + + const matches: GrepMatch[] = []; + for (const [ filePath, fileData ] of Object.entries(filtered)) { + for (let i = 0; i < fileData.content.length; i++) { + const line = fileData.content[i]; + const lineNum = i + 1; + if (regex.test(line)) { + matches.push({ path: filePath, line: lineNum, text: line }); + } + } + } + + return matches; + } + validatePath(path: string | null | undefined): string { + const pathStr = path || '/'; + if (!pathStr || pathStr.trim() === '') { + throw new Error('Path cannot be empty'); + } + + let normalized = pathStr.startsWith('/') ? pathStr : '/' + pathStr; + + if (!normalized.endsWith('/')) { + normalized += '/'; + } + + return normalized; + } + async globInfo(pattern: string, path?: string): Promise { + const files = await this.agentFs.readallFiles(this.cwd); + const result = this.globSearchFiles(files, pattern, path); + + if (result === 'No files found') { + return []; + } + + const paths = result.split('\n'); + const infos: FileInfo[] = []; + for (const p of paths) { + const fd = files[p]; + const size = fd ? fd.content.join('\n').length : 0; + infos.push({ + path: p, + is_dir: false, + size, + modified_at: fd?.modified_at || '', + }); + } + return infos; + } + globSearchFiles( + files: Record, + pattern: string, + path = '/', + ): string { + let normalizedPath: string; + try { + normalizedPath = this.validatePath(path); + } catch { + return 'No files found'; + } + + const filtered = Object.fromEntries( + Object.entries(files).filter(([ fp ]) => fp.startsWith(normalizedPath)), + ); + + // Respect standard glob semantics: + // - Patterns without path separators (e.g., '*.py') match only in the current + // directory (non-recursive) relative to `path`. + // - Use '**' explicitly for recursive matching. + const effectivePattern = pattern; + + const matches: Array<[string, string]> = []; + for (const [ filePath, fileData ] of Object.entries(filtered)) { + let relative = filePath.substring(normalizedPath.length); + if (relative.startsWith('/')) { + relative = relative.substring(1); + } + if (!relative) { + const parts = filePath.split('/'); + relative = parts[parts.length - 1] || ''; + } + + if ( + micromatch.isMatch(relative, effectivePattern, { + dot: true, + nobrace: false, + }) + ) { + matches.push([ filePath, fileData.modified_at ]); + } + } + + matches.sort((a, b) => b[1].localeCompare(a[1])); // Sort by modified_at descending + + if (matches.length === 0) { + return 'No files found'; + } + + return matches.map(([ fp ]) => fp).join('\n'); + } + async write(filePath: string, content: string): Promise { + try { + const resolvedPath = this.resolvePath(filePath); + + try { + const stat = await this.agentFs.lstat(resolvedPath); + if (stat.isSymbolicLink()) { + return { + error: `Cannot write to ${filePath} because it is a symlink. Symlinks are not allowed.`, + }; + } + return { + error: `Cannot write to ${filePath} because it already exists. Read and then make an edit, or write to a new path.`, + }; + } catch { + // File doesn't exist, good to proceed + } + + try { + await this.agentFs.mkdir(path.dirname(resolvedPath), { recursive: true }); + } catch (e) { + if (e.code !== 'EEXIST') { + throw e; + } + } + + // if (SUPPORTS_NOFOLLOW) { + // const flags = + // fsSync.constants.O_WRONLY | + // fsSync.constants.O_CREAT | + // fsSync.constants.O_TRUNC | + // fsSync.constants.O_NOFOLLOW; + + // const fd = await fs.open(resolvedPath, flags, 0o644); + // try { + // await fd.writeFile(content, 'utf-8'); + // } finally { + // await fd.close(); + // } + // } else { + // await fs.writeFile(resolvedPath, content, 'utf-8'); + // } + await this.agentFs.writeFile(resolvedPath, content, 'utf-8'); + + return { path: filePath, filesUpdate: null }; + } catch (e: any) { + return { error: `Error writing file '${filePath}': ${e.message}` }; + } + } + async edit( + filePath: string, + oldString: string, + newString: string, + replaceAll = false, + ): Promise { + try { + const resolvedPath = this.resolvePath(filePath); + + let content: string; + + if (SUPPORTS_NOFOLLOW) { + const stat = await this.agentFs.stat(resolvedPath); + if (!stat.isFile()) { + return { error: `Error: File '${filePath}' not found` }; + } + + // const fd = await fs.open( + // resolvedPath, + // fsSync.constants.O_RDONLY | fsSync.constants.O_NOFOLLOW, + // ); + // try { + // content = await fd.readFile({ encoding: 'utf-8' }); + // } finally { + // await fd.close(); + // } + content = await this.agentFs.readFile(resolvedPath, 'utf-8'); + } else { + const stat = await this.agentFs.lstat(resolvedPath); + if (stat.isSymbolicLink()) { + return { error: `Error: Symlinks are not allowed: ${filePath}` }; + } + if (!stat.isFile()) { + return { error: `Error: File '${filePath}' not found` }; + } + content = await this.agentFs.readFile(resolvedPath, 'utf-8'); + } + + const result = this.performStringReplacement( + content, + oldString, + newString, + replaceAll, + ); + + if (typeof result === 'string') { + return { error: result }; + } + + const [ newContent, occurrences ] = result; + + // // Write securely + // if (SUPPORTS_NOFOLLOW) { + // const flags = + // fsSync.constants.O_WRONLY | + // fsSync.constants.O_TRUNC | + // fsSync.constants.O_NOFOLLOW; + + // const fd = await fs.open(resolvedPath, flags); + // try { + // await fd.writeFile(newContent, 'utf-8'); + // } finally { + // await fd.close(); + // } + // } else { + // await fs.writeFile(resolvedPath, newContent, 'utf-8'); + // } + await this.agentFs.writeFile(resolvedPath, newContent, 'utf-8'); + + return { path: filePath, filesUpdate: null, occurrences }; + } catch (e: any) { + return { error: `Error editing file '${filePath}': ${e.message}` }; + } + } + + performStringReplacement( + content: string, + oldString: string, + newString: string, + replaceAll: boolean, + ): [string, number] | string { + // Use split to count occurrences (simpler than regex) + const occurrences = content.split(oldString).length - 1; + + if (occurrences === 0) { + return `Error: String not found in file: '${oldString}'`; + } + + if (occurrences > 1 && !replaceAll) { + return `Error: String '${oldString}' appears ${occurrences} times in file. Use replace_all=True to replace all instances, or provide a more specific string with surrounding context.`; + } + + // Python's str.replace() replaces ALL occurrences + // Use split/join for consistent behavior + const newContent = content.split(oldString).join(newString); + + return [ newContent, occurrences ]; + } + async uploadFiles( + files: Array<[string, Buffer]>, + ): Promise { + const responses: FileUploadResponse[] = []; + + for (const [ filePath, content ] of files) { + try { + const resolvedPath = this.resolvePath(filePath); + + // Ensure parent directory exists + await this.agentFs.mkdir(path.dirname(resolvedPath), { recursive: true }); + + // Write file + await this.agentFs.writeFile(resolvedPath, content); + responses.push({ path: filePath, error: null }); + } catch (e: any) { + if (e.code === 'ENOENT') { + responses.push({ path: filePath, error: 'file_not_found' }); + } else if (e.code === 'EACCES') { + responses.push({ path: filePath, error: 'permission_denied' }); + } else if (e.code === 'EISDIR') { + responses.push({ path: filePath, error: 'is_directory' }); + } else { + responses.push({ path: filePath, error: 'invalid_path' }); + } + } + } + + return responses; + } + async downloadFiles(paths: string[]): Promise { + const responses: FileDownloadResponse[] = []; + + for (const filePath of paths) { + try { + const resolvedPath = this.resolvePath(filePath); + const content = await this.agentFs.readFile(resolvedPath); + responses.push({ path: filePath, content, error: null }); + } catch (e: any) { + if (e.code === 'ENOENT') { + responses.push({ + path: filePath, + content: null, + error: 'file_not_found', + }); + } else if (e.code === 'EACCES') { + responses.push({ + path: filePath, + content: null, + error: 'permission_denied', + }); + } else if (e.code === 'EISDIR') { + responses.push({ + path: filePath, + content: null, + error: 'is_directory', + }); + } else { + responses.push({ + path: filePath, + content: null, + error: 'invalid_path', + }); + } + } + } + + return responses; + } + + static getFsQualifier(clientName: string) { + return { + [FileSystemInjectName]: [ + { + attribute: FileSystemQualifierAttribute, + value: clientName, + }, + ], + }; + } +} diff --git a/plugin/langchain/lib/filesystem/agentfs/agentfs.ts b/plugin/langchain/lib/filesystem/agentfs/agentfs.ts new file mode 100644 index 00000000..26afa5a6 --- /dev/null +++ b/plugin/langchain/lib/filesystem/agentfs/agentfs.ts @@ -0,0 +1,1328 @@ +/* eslint-disable no-bitwise */ +import { MysqlDataSource } from '@eggjs/dal-runtime'; +import SqlString from 'sqlstring'; +import { createFsError, type FsSyscall } from './errors'; +import { + assertInodeIsDirectory, + assertNotRoot, + assertNotSymlinkMode, + assertReadableExistingInode, + assertReaddirTargetInode, + assertUnlinkTargetInode, + assertWritableExistingInode, + getInodeModeOrThrow, + normalizeRmOptions, + throwENOENTUnlessForce, +} from './guards'; +import { + S_IFMT, + // S_IFREG, + S_IFDIR, + S_IFLNK, + DEFAULT_FILE_MODE, + DEFAULT_DIR_MODE, + createStats, + type Stats, + type DirEntry, + type FilesystemStats, + type FileHandle, + type FileSystem, +} from './interface'; +import { FileData } from 'deepagents'; + +const DEFAULT_CHUNK_SIZE = 4096; + +/** + * An open file handle for AgentFS. + */ +class AgentFSFile implements FileHandle { + private db: MysqlDataSource; + private bufferCtor: BufferConstructor; + private ino: number; + private chunkSize: number; + + constructor(db: MysqlDataSource, bufferCtor: BufferConstructor, ino: number, chunkSize: number) { + this.db = db; + this.bufferCtor = bufferCtor; + this.ino = ino; + this.chunkSize = chunkSize; + } + + async pread(offset: number, size: number): Promise { + const startChunk = Math.floor(offset / this.chunkSize); + const endChunk = Math.floor((offset + size - 1) / this.chunkSize); + + const sql = SqlString.format(` + SELECT chunk_index, data FROM fs_data + WHERE ino = ? AND chunk_index >= ? AND chunk_index <= ? + ORDER BY chunk_index ASC + `, + [ this.ino, startChunk, endChunk ], + ); + const rows = await this.db.query(sql) as { chunk_index: number; data: Buffer }[]; + if (!rows || rows.length === 0) { + return this.bufferCtor.alloc(0); + } + + const buffers: Buffer[] = []; + let bytesCollected = 0; + const startOffsetInChunk = offset % this.chunkSize; + + for (const row of rows) { + const skip = buffers.length === 0 ? startOffsetInChunk : 0; + if (skip >= row.data.length) { + continue; + } + const remaining = size - bytesCollected; + const take = Math.min(row.data.length - skip, remaining); + buffers.push(row.data.subarray(skip, skip + take)); + bytesCollected += take; + } + + if (buffers.length === 0) { + return this.bufferCtor.alloc(0); + } + + return this.bufferCtor.concat(buffers); + } + + async pwrite(offset: number, data: Buffer): Promise { + if (data.length === 0) { + return; + } + + const sql = SqlString.format( + 'SELECT size FROM fs_inode WHERE ino = ?', + [ this.ino ], + ); + const sizeRow = (await this.db.query(sql) as { size: number }[])[0]; + const currentSize = sizeRow?.size ?? 0; + + if (offset > currentSize) { + const zeros = this.bufferCtor.alloc(offset - currentSize); + await this.writeDataAtOffset(currentSize, zeros); + } + + await this.writeDataAtOffset(offset, data); + + const newSize = Math.max(currentSize, offset + data.length); + const now = Math.floor(Date.now() / 1000); + const updateSql = SqlString.format( + 'UPDATE fs_inode SET size = ?, mtime = ? WHERE ino = ?', + [ newSize, now, this.ino ], + ); + await this.db.query(updateSql, [], { executeType: 'execute' }); + } + + private async writeDataAtOffset(offset: number, data: Buffer): Promise { + const startChunk = Math.floor(offset / this.chunkSize); + const endChunk = Math.floor((offset + data.length - 1) / this.chunkSize); + + for (let chunkIdx = startChunk; chunkIdx <= endChunk; chunkIdx++) { + const chunkStart = chunkIdx * this.chunkSize; + const chunkEnd = chunkStart + this.chunkSize; + + const dataStart = Math.max(0, chunkStart - offset); + const dataEnd = Math.min(data.length, chunkEnd - offset); + const writeOffset = Math.max(0, offset - chunkStart); + + const selectSql = SqlString.format( + 'SELECT data FROM fs_data WHERE ino = ? AND chunk_index = ?', + [ this.ino, chunkIdx ], + ); + const existingRow = (await this.db.query(selectSql) as { data: Buffer }[])[0]; + + let chunkData: Buffer; + if (existingRow) { + chunkData = this.bufferCtor.from(existingRow.data); + if (writeOffset + (dataEnd - dataStart) > chunkData.length) { + const newChunk = this.bufferCtor.alloc(writeOffset + (dataEnd - dataStart)); + chunkData.copy(newChunk); + chunkData = newChunk; + } + } else { + chunkData = this.bufferCtor.alloc(writeOffset + (dataEnd - dataStart)); + } + + data.copy(chunkData, writeOffset, dataStart, dataEnd); + + const upsertSql = SqlString.format(` + INSERT INTO fs_data (ino, chunk_index, data) VALUES (?, ?, ?) + ON CONFLICT(ino, chunk_index) DO UPDATE SET data = excluded.data + `, [ this.ino, chunkIdx, chunkData ]); + await this.db.query(upsertSql, [], { executeType: 'execute' }); + } + } + + async truncate(newSize: number): Promise { + const sql = SqlString.format( + 'SELECT size FROM fs_inode WHERE ino = ?', + [ this.ino ], + ); + const sizeRow = (await this.db.query(sql) as { size: number }[])[0]; + const currentSize = sizeRow?.size ?? 0; + + + await this.db.beginTransactionScope(async () => { + if (newSize === 0) { + const deleteSql = SqlString.format( + 'DELETE FROM fs_data WHERE ino = ?', + [ this.ino ], + ); + await this.db.query(deleteSql, [], { executeType: 'execute' }); + } else if (newSize < currentSize) { + const lastChunkIdx = Math.floor((newSize - 1) / this.chunkSize); + + const deleteSql = SqlString.format( + 'DELETE FROM fs_data WHERE ino = ? AND chunk_index > ?', + [ this.ino, lastChunkIdx ], + { executeType: 'execute' }, + ); + await this.db.query(deleteSql, [], { executeType: 'execute' }); + + const offsetInChunk = newSize % this.chunkSize; + if (offsetInChunk > 0) { + const selectSql = SqlString.format( + 'SELECT data FROM fs_data WHERE ino = ? AND chunk_index = ?', + [ this.ino, lastChunkIdx ], + ); + const row = (await this.db.query(selectSql) as { data: Buffer }[])[0]; + + if (row && row.data.length > offsetInChunk) { + const truncatedChunk = row.data.subarray(0, offsetInChunk); + const updateSql = SqlString.format( + 'UPDATE fs_data SET data = ? WHERE ino = ? AND chunk_index = ?', + [ truncatedChunk, this.ino, lastChunkIdx ], + ); + await this.db.query(updateSql, [], { executeType: 'execute' }); + } + } + } + + const now = Math.floor(Date.now() / 1000); + const updateSql = SqlString.format( + 'UPDATE fs_inode SET size = ?, mtime = ? WHERE ino = ?', + [ newSize, now, this.ino ], + ); + await this.db.query(updateSql, [], { executeType: 'execute' }); + }); + } + + async fsync(): Promise { + await this.db.query('PRAGMA synchronous = FULL', [], { executeType: 'execute' }); + await this.db.query('PRAGMA wal_checkpoint(TRUNCATE)', [], { executeType: 'execute' }); + } + + async fstat(): Promise { + const sql = SqlString.format(` + SELECT ino, mode, nlink, uid, gid, size, atime, mtime, ctime + FROM fs_inode WHERE ino = ? + `, [ this.ino ]); + const row = await this.db.query(sql) as { + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + size: number; + atime: number; + mtime: number; + ctime: number; + } | undefined; + + if (!row) { + throw new Error('File handle refers to deleted inode'); + } + + return createStats(row); + } +} + +/** + * A filesystem backed by SQLite, implementing the FileSystem interface. + */ +export class AgentFS implements FileSystem { + private db: MysqlDataSource; + private bufferCtor: BufferConstructor; + private rootIno = 1; + private chunkSize: number = DEFAULT_CHUNK_SIZE; + + private constructor(db: MysqlDataSource, b: BufferConstructor) { + this.db = db; + this.bufferCtor = b; + } + + static async fromDatabase(db: MysqlDataSource, b?: BufferConstructor): Promise { + const fs = new AgentFS(db, b ?? Buffer); + await fs.initialize(); + return fs; + } + + getChunkSize(): number { + return this.chunkSize; + } + + private async initialize(): Promise { + await this.db.query(` + CREATE TABLE IF NOT EXISTS fs_config ( + id INTEGER AUTO_INCREMENT PRIMARY KEY, + \`key\` TEXT NOT NULL, + value TEXT NOT NULL + ) + `, [], { executeType: 'execute' }); + + await this.db.query(` + CREATE TABLE IF NOT EXISTS fs_inode ( + ino INTEGER AUTO_INCREMENT PRIMARY KEY, + mode INTEGER NOT NULL, + nlink INTEGER NOT NULL DEFAULT 0, + uid INTEGER NOT NULL DEFAULT 0, + gid INTEGER NOT NULL DEFAULT 0, + size INTEGER NOT NULL DEFAULT 0, + atime INTEGER NOT NULL, + mtime INTEGER NOT NULL, + ctime INTEGER NOT NULL + ) + `, [], { executeType: 'execute' }); + + await this.db.query(` + CREATE TABLE IF NOT EXISTS fs_dentry ( + id INTEGER AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + parent_ino INTEGER NOT NULL, + ino INTEGER NOT NULL + ) + `, [], { executeType: 'execute' }); + + // await this.db.query(` + // CREATE INDEX idx_fs_dentry_parent ON fs_dentry(parent_ino, name); + // `, [], { executeType: 'execute' }); + + await this.db.query(` + CREATE TABLE IF NOT EXISTS fs_data ( + ino INTEGER NOT NULL, + chunk_index INTEGER NOT NULL, + data BLOB NOT NULL, + PRIMARY KEY (ino, chunk_index) + ) + `, [], { executeType: 'execute' }); + + await this.db.query(` + CREATE TABLE IF NOT EXISTS fs_symlink ( + ino INTEGER PRIMARY KEY, + target TEXT NOT NULL + ) + `, [], { executeType: 'execute' }); + + this.chunkSize = await this.ensureRoot(); + } + + private async ensureRoot(): Promise { + const sql = SqlString.format( + 'SELECT value FROM fs_config WHERE `key` = ?', + [ 'chunk_size' ], + ); + const config = (await this.db.query(sql) as { value: string }[])[0]; + + let chunkSize: number; + if (!config) { + const sql = SqlString.format( + 'INSERT INTO fs_config (`key`, value) VALUES (?, ?)', + [ 'chunk_size', DEFAULT_CHUNK_SIZE.toString() ], + ); + await this.db.query(sql, [], { executeType: 'execute' }); + chunkSize = DEFAULT_CHUNK_SIZE; + } else { + chunkSize = parseInt(config.value, 10) || DEFAULT_CHUNK_SIZE; + } + + // const stmt = this.db.prepare('SELECT ino FROM fs_inode WHERE ino = ?'); + // const root = await stmt.get(this.rootIno); + const sqlRoot = SqlString.format( + 'SELECT ino FROM fs_inode WHERE ino = ?', + [ this.rootIno ], + ); + const root = (await this.db.query(sqlRoot) as { ino: number }[])[0]; + + if (!root) { + const now = Math.floor(Date.now() / 1000); + const insertSql = SqlString.format(` + INSERT INTO fs_inode (ino, mode, nlink, uid, gid, size, atime, mtime, ctime) + VALUES (?, ?, 1, 0, 0, 0, ?, ?, ?) + `, [ this.rootIno, DEFAULT_DIR_MODE, now, now, now ]); + await this.db.query(insertSql, [], { executeType: 'execute' }); + } + + return chunkSize; + } + + private normalizePath(path: string): string { + const normalized = path.replace(/\/+$/, '') || '/'; + return normalized.startsWith('/') ? normalized : '/' + normalized; + } + + private splitPath(path: string): string[] { + const normalized = this.normalizePath(path); + if (normalized === '/') return []; + return normalized.split('/').filter(p => p); + } + + private async resolvePathOrThrow( + path: string, + syscall: FsSyscall, + ): Promise<{ normalizedPath: string; ino: number }> { + const normalizedPath = this.normalizePath(path); + const ino = await this.resolvePath(normalizedPath); + if (ino === null) { + throw createFsError({ + code: 'ENOENT', + syscall, + path: normalizedPath, + message: 'no such file or directory', + }); + } + return { normalizedPath, ino }; + } + + private async resolvePath(path: string): Promise { + const normalized = this.normalizePath(path); + + if (normalized === '/') { + return this.rootIno; + } + + const parts = this.splitPath(normalized); + let currentIno = this.rootIno; + + for (const name of parts) { + const sql = SqlString.format(` + SELECT ino FROM fs_dentry + WHERE parent_ino = ? AND name = ? + `, [ currentIno, name ]); + const result = (await this.db.query(sql) as { ino: number }[])[0]; + + if (!result) { + return null; + } + + currentIno = result.ino; + } + + return currentIno; + } + + private async resolveParent(path: string): Promise<{ parentIno: number; name: string } | null> { + const normalized = this.normalizePath(path); + + if (normalized === '/') { + return null; + } + + const parts = this.splitPath(normalized); + const name = parts[parts.length - 1]; + const parentPath = parts.length === 1 ? '/' : '/' + parts.slice(0, -1).join('/'); + + const parentIno = await this.resolvePath(parentPath); + + if (parentIno === null) { + return null; + } + + return { parentIno, name }; + } + + private async createInode(mode: number, uid = 0, gid = 0): Promise { + const now = Math.floor(Date.now() / 1000); + const insertSql = SqlString.format(` + INSERT INTO fs_inode (mode, uid, gid, size, atime, mtime, ctime) + VALUES (?, ?, ?, 0, ?, ?, ?) + `, [ mode, uid, gid, now, now, now ]); + // const result = (await this.db.query(insertSql, [], { executeType: 'execute' }))[0]; + // const { ino } = result as { ino: number }; + // return Number(ino); + const insertRes = await this.db.query(insertSql, [], { executeType: 'execute' }); + return Number(insertRes.insertId); + } + + private async createDentry(parentIno: number, name: string, ino: number): Promise { + const sql = SqlString.format(` + INSERT INTO fs_dentry (name, parent_ino, ino) + VALUES (?, ?, ?) + `, [ name, parentIno, ino ]); + await this.db.query(sql); + + const updateSql = SqlString.format( + 'UPDATE fs_inode SET nlink = nlink + 1 WHERE ino = ?', + [ ino ], + ); + await this.db.query(updateSql, [], { executeType: 'execute' }); + } + + private async ensureParentDirs(path: string): Promise { + const parts = this.splitPath(path); + parts.pop(); + + let currentIno = this.rootIno; + + for (const name of parts) { + const sql = SqlString.format(` + SELECT ino FROM fs_dentry + WHERE parent_ino = ? AND name = ? + `, [ currentIno, name ]); + const result = (await this.db.query(sql) as { ino: number })[0]; + + if (!result) { + const dirIno = await this.createInode(DEFAULT_DIR_MODE); + await this.createDentry(currentIno, name, dirIno); + currentIno = dirIno; + } else { + await assertInodeIsDirectory(this.db, result.ino, 'open', this.normalizePath(path)); + currentIno = result.ino; + } + } + } + + private async getLinkCount(ino: number): Promise { + const sql = SqlString.format( + 'SELECT nlink FROM fs_inode WHERE ino = ?', + [ ino ], + ); + const result = (await this.db.query(sql) as { nlink: number }[])[0]; + return result?.nlink ?? 0; + } + + private async getInodeMode(ino: number): Promise { + const sql = SqlString.format( + 'SELECT mode FROM fs_inode WHERE ino = ?', + [ ino ], + ); + const row = (await this.db.query(sql) as { mode: number }[])[0]; + return row?.mode ?? null; + } + + // ==================== FileSystem Interface Implementation ==================== + + async writeFile( + path: string, + content: string | Buffer, + options?: BufferEncoding | { encoding?: BufferEncoding }, + ): Promise { + await this.ensureParentDirs(path); + + const ino = await this.resolvePath(path); + + const encoding = typeof options === 'string' + ? options + : options?.encoding; + + const normalizedPath = this.normalizePath(path); + if (ino !== null) { + await assertWritableExistingInode(this.db, ino, 'open', normalizedPath); + await this.updateFileContent(ino, content, encoding); + } else { + const parent = await this.resolveParent(path); + if (!parent) { + throw createFsError({ + code: 'ENOENT', + syscall: 'open', + path: normalizedPath, + message: 'no such file or directory', + }); + } + + await assertInodeIsDirectory(this.db, parent.parentIno, 'open', normalizedPath); + + const fileIno = await this.createInode(DEFAULT_FILE_MODE); + await this.createDentry(parent.parentIno, parent.name, fileIno); + await this.updateFileContent(fileIno, content, encoding); + } + } + + private async updateFileContent( + ino: number, + content: string | Buffer, + encoding?: BufferEncoding, + ): Promise { + const buffer = typeof content === 'string' + ? this.bufferCtor.from(content, encoding ?? 'utf8') + : content; + const now = Math.floor(Date.now() / 1000); + const deleteSql = SqlString.format( + 'DELETE FROM fs_data WHERE ino = ?', + [ ino ], + ); + await this.db.query(deleteSql, [], { executeType: 'execute' }); + + if (buffer.length > 0) { + let chunkIndex = 0; + for (let offset = 0; offset < buffer.length; offset += this.chunkSize) { + const chunk = buffer.subarray(offset, Math.min(offset + this.chunkSize, buffer.length)); + const insertSql = SqlString.format(` + INSERT INTO fs_data (ino, chunk_index, data) + VALUES (?, ?, ?) + `, [ ino, chunkIndex, chunk ]); + await this.db.query(insertSql, [], { executeType: 'execute' }); + chunkIndex++; + } + } + + const updateSql = SqlString.format(` + UPDATE fs_inode + SET size = ?, mtime = ? + WHERE ino = ? + `, [ buffer.length, now, ino ]); + await this.db.query(updateSql, [], { executeType: 'execute' }); + + } + + async readFile(path: string): Promise; + async readFile(path: string, encoding: BufferEncoding): Promise; + async readFile(path: string, options: { encoding: BufferEncoding }): Promise; + async readFile( + path: string, + options?: BufferEncoding | { encoding?: BufferEncoding }, + ): Promise { + const encoding = typeof options === 'string' + ? options + : options?.encoding; + + const { normalizedPath, ino } = await this.resolvePathOrThrow(path, 'open'); + + await assertReadableExistingInode(this.db, ino, 'open', normalizedPath); + + const sql = SqlString.format(` + SELECT data FROM fs_data + WHERE ino = ? + ORDER BY chunk_index ASC + `, [ ino ]); + const rows = await this.db.query(sql) as { data: Buffer }[]; + + let combined: Buffer; + if (rows.length === 0) { + combined = this.bufferCtor.alloc(0); + } else { + const buffers = rows.map(row => row.data); + combined = this.bufferCtor.concat(buffers); + } + + const now = Math.floor(Date.now() / 1000); + const updateSql = SqlString.format( + 'UPDATE fs_inode SET atime = ? WHERE ino = ?', + [ now, ino ], + ); + await this.db.query(updateSql, [], { executeType: 'execute' }); + + if (encoding) { + return combined.toString(encoding); + } + return combined; + } + + async readFileLines(path: string): Promise; + async readFileLines(path: string, encoding: BufferEncoding): Promise; + async readFileLines(path: string, options: { encoding: BufferEncoding }): Promise; + async readFileLines( + path: string, + options?: BufferEncoding | { encoding?: BufferEncoding }, + ): Promise { + const encoding = typeof options === 'string' + ? options + : options?.encoding; + + const { normalizedPath, ino } = await this.resolvePathOrThrow(path, 'open'); + + await assertReadableExistingInode(this.db, ino, 'open', normalizedPath); + + const sql = SqlString.format(` + SELECT data FROM fs_data + WHERE ino = ? + ORDER BY chunk_index ASC + `, [ ino ]); + const rows = await this.db.query(sql) as { data: Buffer }[]; + + const now = Math.floor(Date.now() / 1000); + const updateSql = SqlString.format( + 'UPDATE fs_inode SET atime = ? WHERE ino = ?', + [ now, ino ], + ); + await this.db.query(updateSql, [], { executeType: 'execute' }); + + if (encoding) { + return rows.map(data => data.data.toString(encoding)); + } + return rows.map(data => data.data); + } + + async readdir(path: string): Promise { + const { normalizedPath, ino } = await this.resolvePathOrThrow(path, 'scandir'); + + await assertReaddirTargetInode(this.db, ino, normalizedPath); + + const sql = SqlString.format(` + SELECT name FROM fs_dentry + WHERE parent_ino = ? + ORDER BY name ASC + `, [ ino ]); + const rows = await this.db.query(sql) as { name: string }[]; + + return rows.map(row => row.name); + } + + async readallFiles(path: string, files: Record = {}): Promise> { + const dirs = await this.readdirPlus(path); + + for (const entry of dirs) { + const { name, stats } = entry; + if (stats.isFile()) { + const content = await this.readFileLines(path + '/' + name, 'utf8'); + const fs = await this.stat(path + '/' + name); + files[path + '/' + name] = { content, created_at: new Date(fs.ctime).toISOString(), modified_at: new Date(fs.mtime).toISOString() }; + } else if (stats.isDirectory()) { + await this.readallFiles(path + '/' + name, files); + } + } + return files; + } + + async readdirPlus(path: string): Promise { + const { normalizedPath, ino } = await this.resolvePathOrThrow(path, 'scandir'); + + await assertReaddirTargetInode(this.db, ino, normalizedPath); + + const sql = SqlString.format(` + SELECT d.name, i.ino, i.mode, i.nlink, i.uid, i.gid, i.size, i.atime, i.mtime, i.ctime + FROM fs_dentry d + JOIN fs_inode i ON d.ino = i.ino + WHERE d.parent_ino = ? + ORDER BY d.name ASC + `, [ ino ]); + const rows = await this.db.query(sql) as { + name: string; + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + size: number; + atime: number; + mtime: number; + ctime: number; + }[]; + + + return rows.map(row => ({ + name: row.name, + stats: createStats({ + ino: row.ino, + mode: row.mode, + nlink: row.nlink, + uid: row.uid, + gid: row.gid, + size: row.size, + atime: row.atime, + mtime: row.mtime, + ctime: row.ctime, + }), + })); + } + + async stat(path: string): Promise { + const { normalizedPath, ino } = await this.resolvePathOrThrow(path, 'stat'); + + const sql = SqlString.format(` + SELECT ino, mode, nlink, uid, gid, size, atime, mtime, ctime + FROM fs_inode + WHERE ino = ? + `, [ ino ]); + const row = (await this.db.query(sql) as { + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + size: number; + atime: number; + mtime: number; + ctime: number; + }[])[0]; + + + if (!row) { + throw createFsError({ + code: 'ENOENT', + syscall: 'stat', + path: normalizedPath, + message: 'no such file or directory', + }); + } + + return createStats(row); + } + + async lstat(path: string): Promise { + // For now, lstat is the same as stat since we don't follow symlinks in stat yet + return this.stat(path); + } + + async mkdir(path: string, options?: { recursive?: boolean; }): Promise { + const normalizedPath = this.normalizePath(path); + + const existing = await this.resolvePath(normalizedPath); + if (existing !== null) { + throw createFsError({ + code: 'EEXIST', + syscall: 'mkdir', + path: normalizedPath, + message: 'file already exists', + }); + } + + const parent = await this.resolveParent(normalizedPath); + if (!parent) { + if (options?.recursive) { + await this.ensureParentDirs(normalizedPath); + return this.mkdir(normalizedPath, {}); + } + throw createFsError({ + code: 'ENOENT', + syscall: 'mkdir', + path: normalizedPath, + message: 'no such file or directory', + }); + } + + await assertInodeIsDirectory(this.db, parent.parentIno, 'mkdir', normalizedPath); + + const dirIno = await this.createInode(DEFAULT_DIR_MODE); + try { + await this.createDentry(parent.parentIno, parent.name, dirIno); + } catch { + throw createFsError({ + code: 'EEXIST', + syscall: 'mkdir', + path: normalizedPath, + message: 'file already exists', + }); + } + } + + async rmdir(path: string): Promise { + const normalizedPath = this.normalizePath(path); + assertNotRoot(normalizedPath, 'rmdir'); + + const { ino } = await this.resolvePathOrThrow(normalizedPath, 'rmdir'); + + const mode = await getInodeModeOrThrow(this.db, ino, 'rmdir', normalizedPath); + assertNotSymlinkMode(mode, 'rmdir', normalizedPath); + if ((mode & S_IFMT) !== S_IFDIR) { + throw createFsError({ + code: 'ENOTDIR', + syscall: 'rmdir', + path: normalizedPath, + message: 'not a directory', + }); + } + const sql = SqlString.format(` + SELECT 1 as one FROM fs_dentry + WHERE parent_ino = ? + LIMIT 1 + `, [ ino ]); + const child = (await this.db.query(sql) as { one: number }[])[0]; + if (child) { + throw createFsError({ + code: 'ENOTEMPTY', + syscall: 'rmdir', + path: normalizedPath, + message: 'directory not empty', + }); + } + + const parent = await this.resolveParent(normalizedPath); + if (!parent) { + throw createFsError({ + code: 'EPERM', + syscall: 'rmdir', + path: normalizedPath, + message: 'operation not permitted', + }); + } + + await this.removeDentryAndMaybeInode(parent.parentIno, parent.name, ino); + } + + async unlink(path: string): Promise { + const normalizedPath = this.normalizePath(path); + assertNotRoot(normalizedPath, 'unlink'); + const { ino } = await this.resolvePathOrThrow(normalizedPath, 'unlink'); + + await assertUnlinkTargetInode(this.db, ino, normalizedPath); + + const parent = (await this.resolveParent(normalizedPath))!; + + const sql = SqlString.format(` + DELETE FROM fs_dentry + WHERE parent_ino = ? AND name = ? + `, [ parent.parentIno, parent.name ]); + await this.db.query(sql, [], { executeType: 'execute' }); + + const decrementSql = SqlString.format( + 'UPDATE fs_inode SET nlink = nlink - 1 WHERE ino = ?', + [ ino ], + ); + await this.db.query(decrementSql, [], { executeType: 'execute' }); + + const linkCount = await this.getLinkCount(ino); + if (linkCount === 0) { + const deleteInodeSql = SqlString.format( + 'DELETE FROM fs_inode WHERE ino = ?', + [ ino ], + ); + await this.db.query(deleteInodeSql, [], { executeType: 'execute' }); + + const deleteDataSql = SqlString.format( + 'DELETE FROM fs_data WHERE ino = ?', + [ ino ], + ); + await this.db.query(deleteDataSql, [], { executeType: 'execute' }); + } + } + + async rm( + path: string, + options?: { force?: boolean; recursive?: boolean }, + ): Promise { + const normalizedPath = this.normalizePath(path); + const { force, recursive } = normalizeRmOptions(options); + assertNotRoot(normalizedPath, 'rm'); + + const ino = await this.resolvePath(normalizedPath); + if (ino === null) { + throwENOENTUnlessForce(normalizedPath, 'rm', force); + return; + } + + const mode = await getInodeModeOrThrow(this.db, ino, 'rm', normalizedPath); + assertNotSymlinkMode(mode, 'rm', normalizedPath); + + const parent = await this.resolveParent(normalizedPath); + if (!parent) { + throw createFsError({ + code: 'EPERM', + syscall: 'rm', + path: normalizedPath, + message: 'operation not permitted', + }); + } + + if ((mode & S_IFMT) === S_IFDIR) { + if (!recursive) { + throw createFsError({ + code: 'EISDIR', + syscall: 'rm', + path: normalizedPath, + message: 'illegal operation on a directory', + }); + } + + await this.rmDirContentsRecursive(ino); + await this.removeDentryAndMaybeInode(parent.parentIno, parent.name, ino); + return; + } + + await this.removeDentryAndMaybeInode(parent.parentIno, parent.name, ino); + } + + private async rmDirContentsRecursive(dirIno: number): Promise { + const sql = SqlString.format(` + SELECT name, ino FROM fs_dentry + WHERE parent_ino = ? + ORDER BY name ASC + `, [ dirIno ]); + const children = await this.db.query(sql) as { name: string; ino: number }[]; + + for (const child of children) { + const mode = await this.getInodeMode(child.ino); + if (mode === null) { + continue; + } + + if ((mode & S_IFMT) === S_IFDIR) { + await this.rmDirContentsRecursive(child.ino); + await this.removeDentryAndMaybeInode(dirIno, child.name, child.ino); + } else { + assertNotSymlinkMode(mode, 'rm', ''); + await this.removeDentryAndMaybeInode(dirIno, child.name, child.ino); + } + } + } + + private async removeDentryAndMaybeInode(parentIno: number, name: string, ino: number): Promise { + const sql = SqlString.format(` + DELETE FROM fs_dentry + WHERE parent_ino = ? AND name = ? + `, [ parentIno, name ]); + await this.db.query(sql, [], { executeType: 'execute' }); + + const decrementSql = SqlString.format( + 'UPDATE fs_inode SET nlink = nlink - 1 WHERE ino = ?', + [ ino ], + ); + await this.db.query(decrementSql, [], { executeType: 'execute' }); + + const linkCount = await this.getLinkCount(ino); + if (linkCount === 0) { + const deleteInodeSql = SqlString.format( + 'DELETE FROM fs_inode WHERE ino = ?', + [ ino ], + ); + await this.db.query(deleteInodeSql, [], { executeType: 'execute' }); + + const deleteDataSql = SqlString.format( + 'DELETE FROM fs_data WHERE ino = ?', + [ ino ], + ); + await this.db.query(deleteDataSql, [], { executeType: 'execute' }); + + const deleteSymlinkSql = SqlString.format( + 'DELETE FROM fs_symlink WHERE ino = ?', + [ ino ], + ); + await this.db.query(deleteSymlinkSql, [], { executeType: 'execute' }); + } + } + + async rename(oldPath: string, newPath: string): Promise { + const oldNormalized = this.normalizePath(oldPath); + const newNormalized = this.normalizePath(newPath); + + if (oldNormalized === newNormalized) return; + + assertNotRoot(oldNormalized, 'rename'); + assertNotRoot(newNormalized, 'rename'); + + const oldParent = await this.resolveParent(oldNormalized); + if (!oldParent) { + throw createFsError({ + code: 'EPERM', + syscall: 'rename', + path: oldNormalized, + message: 'operation not permitted', + }); + } + + const newParent = await this.resolveParent(newNormalized); + if (!newParent) { + throw createFsError({ + code: 'ENOENT', + syscall: 'rename', + path: newNormalized, + message: 'no such file or directory', + }); + } + + await assertInodeIsDirectory(this.db, newParent.parentIno, 'rename', newNormalized); + + await this.db.beginTransactionScope(async () => { + const oldResolved = await this.resolvePathOrThrow(oldNormalized, 'rename'); + const oldIno = oldResolved.ino; + const oldMode = await getInodeModeOrThrow(this.db, oldIno, 'rename', oldNormalized); + assertNotSymlinkMode(oldMode, 'rename', oldNormalized); + const oldIsDir = (oldMode & S_IFMT) === S_IFDIR; + + if (oldIsDir && newNormalized.startsWith(oldNormalized + '/')) { + throw createFsError({ + code: 'EINVAL', + syscall: 'rename', + path: newNormalized, + message: 'invalid argument', + }); + } + + const newIno = await this.resolvePath(newNormalized); + if (newIno !== null) { + const newMode = await getInodeModeOrThrow(this.db, newIno, 'rename', newNormalized); + assertNotSymlinkMode(newMode, 'rename', newNormalized); + const newIsDir = (newMode & S_IFMT) === S_IFDIR; + + if (newIsDir && !oldIsDir) { + throw createFsError({ + code: 'EISDIR', + syscall: 'rename', + path: newNormalized, + message: 'illegal operation on a directory', + }); + } + if (!newIsDir && oldIsDir) { + throw createFsError({ + code: 'ENOTDIR', + syscall: 'rename', + path: newNormalized, + message: 'not a directory', + }); + } + + if (newIsDir) { + const sql = SqlString.format(` + SELECT 1 as one FROM fs_dentry + WHERE parent_ino = ? + LIMIT 1 + `, [ newIno ]); + const child = (await this.db.query(sql) as { one: number }[])[0]; + if (child) { + throw createFsError({ + code: 'ENOTEMPTY', + syscall: 'rename', + path: newNormalized, + message: 'directory not empty', + }); + } + } + + await this.removeDentryAndMaybeInode(newParent.parentIno, newParent.name, newIno); + } + + const updateDentrySql = SqlString.format(` + UPDATE fs_dentry + SET parent_ino = ?, name = ? + WHERE parent_ino = ? AND name = ? + `, [ newParent.parentIno, newParent.name, oldParent.parentIno, oldParent.name ]); + await this.db.query(updateDentrySql, [], { executeType: 'execute' }); + + const now = Math.floor(Date.now() / 1000); + const updateInodeCtimeSql = SqlString.format(` + UPDATE fs_inode + SET ctime = ? + WHERE ino = ? + `, [ now, oldIno ]); + await this.db.query(updateInodeCtimeSql, [], { executeType: 'execute' }); + + if (newParent.parentIno !== oldParent.parentIno) { + const sql = SqlString.format(` + UPDATE fs_inode + SET mtime = ?, ctime = ? + WHERE ino = ? + `, [ now, now, newParent.parentIno ]); + await this.db.query(sql, [], { executeType: 'execute' }); + } else { + const sql = SqlString.format(` + UPDATE fs_inode + SET mtime = ?, ctime = ? + WHERE ino = ? + `, [ now, now, oldParent.parentIno ]); + await this.db.query(sql, [], { executeType: 'execute' }); + } + }); + } + + async copyFile(src: string, dest: string): Promise { + const srcNormalized = this.normalizePath(src); + const destNormalized = this.normalizePath(dest); + + if (srcNormalized === destNormalized) { + throw createFsError({ + code: 'EINVAL', + syscall: 'copyfile', + path: destNormalized, + message: 'invalid argument', + }); + } + + const { ino: srcIno } = await this.resolvePathOrThrow(srcNormalized, 'copyfile'); + await assertReadableExistingInode(this.db, srcIno, 'copyfile', srcNormalized); + + const sql = SqlString.format(` + SELECT mode, uid, gid, size FROM fs_inode WHERE ino = ? + `, [ srcIno ]); + const srcRow = (await this.db.query(sql) as + | { mode: number; uid: number; gid: number; size: number }[])[0]; + if (!srcRow) { + throw createFsError({ + code: 'ENOENT', + syscall: 'copyfile', + path: srcNormalized, + message: 'no such file or directory', + }); + } + + const destParent = await this.resolveParent(destNormalized); + if (!destParent) { + throw createFsError({ + code: 'ENOENT', + syscall: 'copyfile', + path: destNormalized, + message: 'no such file or directory', + }); + } + await assertInodeIsDirectory(this.db, destParent.parentIno, 'copyfile', destNormalized); + + await this.db.beginTransactionScope(async () => { + const now = Math.floor(Date.now() / 1000); + + const destIno = await this.resolvePath(destNormalized); + if (destIno !== null) { + const destMode = await getInodeModeOrThrow(this.db, destIno, 'copyfile', destNormalized); + assertNotSymlinkMode(destMode, 'copyfile', destNormalized); + if ((destMode & S_IFMT) === S_IFDIR) { + throw createFsError({ + code: 'EISDIR', + syscall: 'copyfile', + path: destNormalized, + message: 'illegal operation on a directory', + }); + } + + const deleteSql = SqlString.format( + 'DELETE FROM fs_data WHERE ino = ?', + [ destIno ], + ); + await this.db.query(deleteSql, [], { executeType: 'execute' }); + + const copySql = SqlString.format(` + INSERT INTO fs_data (ino, chunk_index, data) + SELECT ?, chunk_index, data + FROM fs_data + WHERE ino = ? + ORDER BY chunk_index ASC + `, [ destIno, srcIno ]); + await this.db.query(copySql, [], { executeType: 'execute' }); + + const updateSql = SqlString.format(` + UPDATE fs_inode + SET mode = ?, uid = ?, gid = ?, size = ?, mtime = ?, ctime = ? + WHERE ino = ? + `, [ srcRow.mode, srcRow.uid, srcRow.gid, srcRow.size, now, now, destIno ]); + await this.db.query(updateSql, [], { executeType: 'execute' }); + } else { + const destInoCreated = await this.createInode(srcRow.mode, srcRow.uid, srcRow.gid); + await this.createDentry(destParent.parentIno, destParent.name, destInoCreated); + + const copySql = SqlString.format(` + INSERT INTO fs_data (ino, chunk_index, data) + SELECT ?, chunk_index, data + FROM fs_data + WHERE ino = ? + ORDER BY chunk_index ASC + `, [ destInoCreated, srcIno ]); + await this.db.query(copySql, [], { executeType: 'execute' }); + + const updateSql = SqlString.format(` + UPDATE fs_inode + SET size = ?, mtime = ?, ctime = ? + WHERE ino = ? + `, [ srcRow.size, now, now, destInoCreated ]); + await this.db.query(updateSql, [], { executeType: 'execute' }); + } + }); + } + + async symlink(target: string, linkpath: string): Promise { + const normalizedLinkpath = this.normalizePath(linkpath); + + const existing = await this.resolvePath(normalizedLinkpath); + if (existing !== null) { + throw createFsError({ + code: 'EEXIST', + syscall: 'open', + path: normalizedLinkpath, + message: 'file already exists', + }); + } + + const parent = await this.resolveParent(normalizedLinkpath); + if (!parent) { + throw createFsError({ + code: 'ENOENT', + syscall: 'open', + path: normalizedLinkpath, + message: 'no such file or directory', + }); + } + + await assertInodeIsDirectory(this.db, parent.parentIno, 'open', normalizedLinkpath); + + const mode = S_IFLNK | 0o777; + const symlinkIno = await this.createInode(mode); + await this.createDentry(parent.parentIno, parent.name, symlinkIno); + + const insertSql = SqlString.format( + 'INSERT INTO fs_symlink (ino, target) VALUES (?, ?)', + [ symlinkIno, target ], + ); + await this.db.query(insertSql, [], { executeType: 'execute' }); + + const updateSql = SqlString.format( + 'UPDATE fs_inode SET size = ? WHERE ino = ?', + [ target.length, symlinkIno ], + ); + await this.db.query(updateSql, [], { executeType: 'execute' }); + } + + async readlink(path: string): Promise { + const { normalizedPath, ino } = await this.resolvePathOrThrow(path, 'open'); + + const mode = await this.getInodeMode(ino); + if (mode === null || (mode & S_IFMT) !== S_IFLNK) { + throw createFsError({ + code: 'EINVAL', + syscall: 'open', + path: normalizedPath, + message: 'invalid argument', + }); + } + + const sql = SqlString.format( + 'SELECT target FROM fs_symlink WHERE ino = ?', + [ ino ], + ); + const row = (await this.db.query(sql) as { target: string }[])[0]; + + if (!row) { + throw createFsError({ + code: 'ENOENT', + syscall: 'open', + path: normalizedPath, + message: 'no such file or directory', + }); + } + + return row.target; + } + + async access(path: string): Promise { + const normalizedPath = this.normalizePath(path); + const ino = await this.resolvePath(normalizedPath); + if (ino === null) { + throw createFsError({ + code: 'ENOENT', + syscall: 'access', + path: normalizedPath, + message: 'no such file or directory', + }); + } + } + + async statfs(): Promise { + const inodeSql = 'SELECT COUNT(*) as count FROM fs_inode'; + const inodeRow = (await this.db.query(inodeSql) as { count: number }[])[0]; + + const bytesSql = 'SELECT COALESCE(SUM(LENGTH(data)), 0) as total FROM fs_data'; + const bytesRow = (await this.db.query(bytesSql) as { total: number }[])[0]; + + return { + inodes: inodeRow.count, + bytesUsed: bytesRow.total, + }; + } + + async open(path: string): Promise { + const { normalizedPath, ino } = await this.resolvePathOrThrow(path, 'open'); + await assertReadableExistingInode(this.db, ino, 'open', normalizedPath); + + return new AgentFSFile(this.db, this.bufferCtor, ino, this.chunkSize); + } + + // Legacy alias + async deleteFile(path: string): Promise { + return await this.unlink(path); + } +} diff --git a/plugin/langchain/lib/filesystem/agentfs/errors.ts b/plugin/langchain/lib/filesystem/agentfs/errors.ts new file mode 100644 index 00000000..988ff35d --- /dev/null +++ b/plugin/langchain/lib/filesystem/agentfs/errors.ts @@ -0,0 +1,52 @@ +/** + * POSIX-style error codes for filesystem operations. + */ +export type FsErrorCode = + | 'ENOENT' // No such file or directory + | 'EEXIST' // File already exists + | 'EISDIR' // Is a directory (when file expected) + | 'ENOTDIR' // Not a directory (when directory expected) + | 'ENOTEMPTY' // Directory not empty + | 'EPERM' // Operation not permitted + | 'EINVAL' // Invalid argument + | 'ENOSYS'; // Function not implemented (use for symlinks) + +/** + * Filesystem syscall names for error reporting. + * rm, scandir and copyFile are not actual syscall but used for convenience + */ +export type FsSyscall = + | 'open' + | 'stat' + | 'mkdir' + | 'rmdir' + | 'rm' + | 'unlink' + | 'rename' + | 'scandir' + | 'copyfile' + | 'access'; + +export interface ErrnoException extends Error { + code?: FsErrorCode; + syscall?: FsSyscall; + path?: string; +} + +export function createFsError(params: { + code: FsErrorCode; + syscall: FsSyscall; + path?: string; + message?: string; +}): ErrnoException { + const { code, syscall, path, message } = params; + const base = message ?? code; + const suffix = path !== undefined ? ` '${path}'` : ''; + const err = new Error( + `${code}: ${base}, ${syscall}${suffix}`, + ) as ErrnoException; + err.code = code; + err.syscall = syscall; + if (path !== undefined) err.path = path; + return err; +} diff --git a/plugin/langchain/lib/filesystem/agentfs/guards.ts b/plugin/langchain/lib/filesystem/agentfs/guards.ts new file mode 100644 index 00000000..70e68d95 --- /dev/null +++ b/plugin/langchain/lib/filesystem/agentfs/guards.ts @@ -0,0 +1,216 @@ +/* eslint-disable no-bitwise */ +import SqlString from 'sqlstring'; +import { createFsError, type FsSyscall } from './errors'; +import { S_IFDIR, S_IFLNK, S_IFMT } from './interface'; +import { MysqlDataSource } from '@eggjs/dal-runtime'; + +async function getInodeMode( + db: MysqlDataSource, + ino: number, +): Promise { + const sql = SqlString.format( + 'SELECT mode FROM fs_inode WHERE ino = ?', + [ ino ], + ); + const result = await db.query(sql); + if (!result || result.length === 0) { + return null; + } + return (result[0] as { mode: number }).mode; +} + +function isDirMode(mode: number): boolean { + return (mode & S_IFMT) === S_IFDIR; +} + +export async function getInodeModeOrThrow( + db: MysqlDataSource, + ino: number, + syscall: FsSyscall, + path: string, +): Promise { + const mode = await getInodeMode(db, ino); + if (mode === null) { + throw createFsError({ + code: 'ENOENT', + syscall, + path, + message: 'no such file or directory', + }); + } + return mode; +} + +export function assertNotRoot(path: string, syscall: FsSyscall): void { + if (path === '/') { + throw createFsError({ + code: 'EPERM', + syscall, + path, + message: 'operation not permitted on root directory', + }); + } +} + +export function normalizeRmOptions(options?: { + force?: boolean; + recursive?: boolean; +}): { + force: boolean; + recursive: boolean; + } { + return { + force: options?.force === true, + recursive: options?.recursive === true, + }; +} + +export function throwENOENTUnlessForce( + path: string, + syscall: FsSyscall, + force: boolean, +): void { + if (force) return; + throw createFsError({ + code: 'ENOENT', + syscall, + path, + message: 'no such file or directory', + }); +} + +export function assertNotSymlinkMode( + mode: number, + syscall: FsSyscall, + path: string, +): void { + if ((mode & S_IFMT) === S_IFLNK) { + throw createFsError({ + code: 'ENOSYS', + syscall, + path, + message: 'symbolic links not supported yet', + }); + } +} + +async function assertExistingNonDirNonSymlinkInode( + db: MysqlDataSource, + ino: number, + syscall: FsSyscall, + fullPathForError: string, +): Promise { + const mode = await getInodeMode(db, ino); + if (mode === null) { + throw createFsError({ + code: 'ENOENT', + syscall, + path: fullPathForError, + message: 'no such file or directory', + }); + } + if (isDirMode(mode)) { + throw createFsError({ + code: 'EISDIR', + syscall, + path: fullPathForError, + message: 'illegal operation on a directory', + }); + } + assertNotSymlinkMode(mode, syscall, fullPathForError); +} + +export async function assertInodeIsDirectory( + db: MysqlDataSource, + ino: number, + syscall: FsSyscall, + fullPathForError: string, +): Promise { + const mode = await getInodeMode(db, ino); + if (mode === null) { + throw createFsError({ + code: 'ENOENT', + syscall, + path: fullPathForError, + message: 'no such file or directory', + }); + } + if (!isDirMode(mode)) { + throw createFsError({ + code: 'ENOTDIR', + syscall, + path: fullPathForError, + message: 'not a directory', + }); + } +} + +export async function assertWritableExistingInode( + db: MysqlDataSource, + ino: number, + syscall: FsSyscall, + fullPathForError: string, +): Promise { + await assertExistingNonDirNonSymlinkInode(db, ino, syscall, fullPathForError); +} + +export async function assertReadableExistingInode( + db: MysqlDataSource, + ino: number, + syscall: FsSyscall, + fullPathForError: string, +): Promise { + await assertExistingNonDirNonSymlinkInode(db, ino, syscall, fullPathForError); +} + +export async function assertReaddirTargetInode( + db: MysqlDataSource, + ino: number, + fullPathForError: string, +): Promise { + const syscall = 'scandir'; + const mode = await getInodeMode(db, ino); + if (mode === null) { + throw createFsError({ + code: 'ENOENT', + syscall, + path: fullPathForError, + message: 'no such file or directory', + }); + } + assertNotSymlinkMode(mode, syscall, fullPathForError); + if (!isDirMode(mode)) { + throw createFsError({ + code: 'ENOTDIR', + syscall, + path: fullPathForError, + message: 'not a directory', + }); + } +} + +export async function assertUnlinkTargetInode( + db: MysqlDataSource, + ino: number, + fullPathForError: string, +): Promise { + const syscall = 'unlink'; + const mode = await getInodeMode(db, ino); + if (mode === null) { + throw createFsError({ + code: 'ENOENT', + syscall, + path: fullPathForError, + message: 'no such file or directory', + }); + } + if (isDirMode(mode)) { + throw createFsError({ + code: 'EISDIR', + syscall, + path: fullPathForError, + message: 'illegal operation on a directory', + }); + } + assertNotSymlinkMode(mode, syscall, fullPathForError); +} diff --git a/plugin/langchain/lib/filesystem/agentfs/interface.ts b/plugin/langchain/lib/filesystem/agentfs/interface.ts new file mode 100644 index 00000000..44de9c2f --- /dev/null +++ b/plugin/langchain/lib/filesystem/agentfs/interface.ts @@ -0,0 +1,243 @@ +/* eslint-disable no-bitwise */ +// File types for mode field +export const S_IFMT = 0o170000; // File type mask +export const S_IFREG = 0o100000; // Regular file +export const S_IFDIR = 0o040000; // Directory +export const S_IFLNK = 0o120000; // Symbolic link + +// Default permissions +export const DEFAULT_FILE_MODE = S_IFREG | 0o644; // Regular file, rw-r--r-- +export const DEFAULT_DIR_MODE = S_IFDIR | 0o755; // Directory, rwxr-xr-x + +/** + * File statistics (Node.js compatible) + */ +export interface Stats { + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + size: number; + atime: number; + mtime: number; + ctime: number; + isFile(): boolean; + isDirectory(): boolean; + isSymbolicLink(): boolean; +} + +/** + * Plain stats data without methods (for creating Stats objects) + */ +export interface StatsData { + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + size: number; + atime: number; + mtime: number; + ctime: number; +} + +/** + * Create a Stats object from raw data + */ +export function createStats(data: StatsData): Stats { + return { + ...data, + isFile: () => (data.mode & S_IFMT) === S_IFREG, + isDirectory: () => (data.mode & S_IFMT) === S_IFDIR, + isSymbolicLink: () => (data.mode & S_IFMT) === S_IFLNK, + }; +} + +/** + * Directory entry with full statistics + */ +export interface DirEntry { + /** Entry name (without path) */ + name: string; + /** Full statistics for this entry */ + stats: Stats; +} + +/** + * Filesystem statistics + */ +export interface FilesystemStats { + /** Total number of inodes (files, directories, symlinks) */ + inodes: number; + /** Total bytes used by file contents */ + bytesUsed: number; +} + +/** + * An open file handle for performing I/O operations. + * Similar to Node.js FileHandle from fs/promises. + */ +export interface FileHandle { + /** + * Read from the file at the given offset. + */ + pread(offset: number, size: number): Promise; + + /** + * Write to the file at the given offset. + */ + pwrite(offset: number, data: Buffer): Promise; + + /** + * Truncate the file to the specified size. + */ + truncate(size: number): Promise; + + /** + * Synchronize file data to persistent storage. + */ + fsync(): Promise; + + /** + * Get file statistics. + */ + fstat(): Promise; +} + +/** + * FileSystem interface following Node.js fs/promises API conventions. + * + * This interface abstracts over different filesystem backends, + * allowing implementations like AgentFS (SQLite-backed), HostFS (native filesystem), + * and OverlayFS (layered filesystem). + * + * Methods throw errors on failure (ENOENT, EEXIST, etc.) like Node.js fs/promises. + */ +export interface FileSystem { + /** + * Get file statistics. + * @throws {ErrnoException} ENOENT if path does not exist + */ + stat(path: string): Promise; + + /** + * Get file statistics without following symlinks. + * @throws {ErrnoException} ENOENT if path does not exist + */ + lstat(path: string): Promise; + + /** + * Read entire file contents. + * @throws {ErrnoException} ENOENT if file does not exist + * @throws {ErrnoException} EISDIR if path is a directory + */ + readFile(path: string): Promise; + readFile(path: string, encoding: BufferEncoding): Promise; + readFile( + path: string, + options: { encoding: BufferEncoding }, + ): Promise; + readFile( + path: string, + options?: BufferEncoding | { encoding?: BufferEncoding }, + ): Promise; + + /** + * Write data to a file (creates or overwrites). + * Creates parent directories if they don't exist. + */ + writeFile( + path: string, + data: string | Buffer, + options?: BufferEncoding | { encoding?: BufferEncoding }, + ): Promise; + + /** + * List directory contents. + * @throws {ErrnoException} ENOENT if directory does not exist + * @throws {ErrnoException} ENOTDIR if path is not a directory + */ + readdir(path: string): Promise; + + /** + * List directory contents with full statistics for each entry. + * Optimized version that avoids N+1 queries. + * @throws {ErrnoException} ENOENT if directory does not exist + * @throws {ErrnoException} ENOTDIR if path is not a directory + */ + readdirPlus(path: string): Promise; + + /** + * Create a directory. + * @throws {ErrnoException} EEXIST if path already exists + * @throws {ErrnoException} ENOENT if parent does not exist + */ + mkdir(path: string): Promise; + + /** + * Remove an empty directory. + * @throws {ErrnoException} ENOENT if path does not exist + * @throws {ErrnoException} ENOTEMPTY if directory is not empty + * @throws {ErrnoException} ENOTDIR if path is not a directory + */ + rmdir(path: string): Promise; + + /** + * Remove a file. + * @throws {ErrnoException} ENOENT if path does not exist + * @throws {ErrnoException} EISDIR if path is a directory + */ + unlink(path: string): Promise; + + /** + * Remove a file or directory. + */ + rm( + path: string, + options?: { force?: boolean; recursive?: boolean }, + ): Promise; + + /** + * Rename/move a file or directory. + * @throws {ErrnoException} ENOENT if source does not exist + */ + rename(oldPath: string, newPath: string): Promise; + + /** + * Copy a file. + * @throws {ErrnoException} ENOENT if source does not exist + * @throws {ErrnoException} EISDIR if source or dest is a directory + */ + copyFile(src: string, dest: string): Promise; + + /** + * Create a symbolic link. + * @throws {ErrnoException} EEXIST if linkpath already exists + */ + symlink(target: string, linkpath: string): Promise; + + /** + * Read the target of a symbolic link. + * @throws {ErrnoException} ENOENT if path does not exist + * @throws {ErrnoException} EINVAL if path is not a symlink + */ + readlink(path: string): Promise; + + /** + * Test file access (existence check). + * @throws {ErrnoException} ENOENT if path does not exist + */ + access(path: string): Promise; + + /** + * Get filesystem statistics. + */ + statfs(): Promise; + + /** + * Open a file and return a file handle for I/O operations. + * @throws {ErrnoException} ENOENT if file does not exist + */ + open(path: string): Promise; +} diff --git a/plugin/langchain/lib/util.ts b/plugin/langchain/lib/util.ts index be1a4a24..5a1d2d90 100644 --- a/plugin/langchain/lib/util.ts +++ b/plugin/langchain/lib/util.ts @@ -1,10 +1,11 @@ import { ObjectInfo, ModuleConfig } from '@eggjs/tegg'; import { ChatModelQualifierAttribute, + FileSystemQualifierAttribute, } from '@eggjs/tegg-langchain-decorator'; import assert from 'node:assert'; -export type ConfigTypeHelper = +export type ConfigTypeHelper = Required[T]['clients'][keyof Required[T]['clients']]; export function getClientNames(config: ModuleConfig | undefined, key: string): string[] { @@ -22,3 +23,13 @@ export function getChatModelConfig(config: ModuleConfig, objectInfo: ObjectInfo) } return chatModelConfig!; } + +export function getFileSystemConfig(config: ModuleConfig, objectInfo: ObjectInfo): ConfigTypeHelper<'filesystem'> { + const filesystemName = objectInfo.qualifiers.find(t => t.attribute === FileSystemQualifierAttribute)?.value; + assert(filesystemName, 'not found filesystem name'); + const fsConfig = config.filesystem?.clients[filesystemName]; + if (!config) { + throw new Error(`not found FsMiddleware config for ${filesystemName}`); + } + return fsConfig!; +} diff --git a/plugin/langchain/package.json b/plugin/langchain/package.json index c6be7e28..70e6fcae 100644 --- a/plugin/langchain/package.json +++ b/plugin/langchain/package.json @@ -71,9 +71,13 @@ "@langchain/mcp-adapters": "^1.0.0", "@langchain/openai": "^1.0.0", "@types/koa-router": "^7.0.40", + "agentfs-sdk": "^0.5.3", + "deepagents": "^1.5.1", "koa-compose": "^3.2.1", "langchain": "^1.1.2", + "micromatch": "^4.0.8", "sdk-base": "^4.2.0", + "sqlstring": "^2.3.3", "urllib": "^4.4.0" }, "devDependencies": { @@ -83,6 +87,7 @@ "@eggjs/tegg-controller-plugin": "^3.70.1", "@eggjs/tegg-plugin": "^3.70.1", "@types/mocha": "^10.0.1", + "@types/micromatch": "^4.0.10", "@types/node": "^20.2.4", "cross-env": "^7.0.3", "egg": "^3.9.1", diff --git a/plugin/langchain/test/fixtures/apps/langchain/app/modules/bar/controller/AppController.ts b/plugin/langchain/test/fixtures/apps/langchain/app/modules/bar/controller/AppController.ts index 39541eb6..a308a849 100644 --- a/plugin/langchain/test/fixtures/apps/langchain/app/modules/bar/controller/AppController.ts +++ b/plugin/langchain/test/fixtures/apps/langchain/app/modules/bar/controller/AppController.ts @@ -4,10 +4,11 @@ import { HTTPMethodEnum, Inject, } from '@eggjs/tegg'; -import { ChatModelQualifier, TeggBoundModel, TeggCompiledStateGraph } from '@eggjs/tegg-langchain-decorator'; +import { ChatModelQualifier, FileSystemQualifier, TeggBoundModel, TeggCompiledStateGraph } from '@eggjs/tegg-langchain-decorator'; import { ChatOpenAIModel } from '../../../../../../../../lib/ChatOpenAI'; import { BoundChatModel } from '../service/BoundChatModel'; import { FooGraph } from '../service/Graph'; +import { MysqlFilesystem } from '../../../../../../../../lib/filesystem/MysqlFilesystem'; import { AIMessage } from 'langchain'; @HTTPController({ @@ -24,6 +25,10 @@ export class AppController { @Inject() compiledFooGraph: TeggCompiledStateGraph; + @Inject() + @FileSystemQualifier('foo') + teggFilesystem: MysqlFilesystem; + @HTTPMethod({ method: HTTPMethodEnum.GET, path: '/hello', diff --git a/plugin/langchain/test/fixtures/apps/langchain/app/modules/bar/module.yml b/plugin/langchain/test/fixtures/apps/langchain/app/modules/bar/module.yml index 46b277ac..01ba25b9 100644 --- a/plugin/langchain/test/fixtures/apps/langchain/app/modules/bar/module.yml +++ b/plugin/langchain/test/fixtures/apps/langchain/app/modules/bar/module.yml @@ -16,4 +16,21 @@ mcp: clientName: barSse version: 1.0.0 transportType: SSE - type: http \ No newline at end of file + type: http + +dataSource: + foo: + connectionLimit: 100 + database: 'test_dal_plugin' + host: '127.0.0.1' + user: root + port: 3306 + timezone: '+08:00' + forkDb: true + +filesystem: + clients: + foo: + dataSource: foo + cwd: '/langchain-plugin' + virtualMode: false diff --git a/plugin/langchain/test/fixtures/apps/langchain/app/modules/bar/service/Graph.ts b/plugin/langchain/test/fixtures/apps/langchain/app/modules/bar/service/Graph.ts index 5e9991aa..e85eff27 100644 --- a/plugin/langchain/test/fixtures/apps/langchain/app/modules/bar/service/Graph.ts +++ b/plugin/langchain/test/fixtures/apps/langchain/app/modules/bar/service/Graph.ts @@ -42,6 +42,8 @@ export const ToolType = { }) export class FooTool implements IGraphTool { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore async execute(@ToolArgsSchema(ToolType) args: ToolArgs) { console.log('query: ', args.query); return `hello ${args.query}`; diff --git a/plugin/langchain/test/fixtures/apps/langchain/config/plugin.js b/plugin/langchain/test/fixtures/apps/langchain/config/plugin.js index 7329d547..d28784ce 100644 --- a/plugin/langchain/test/fixtures/apps/langchain/config/plugin.js +++ b/plugin/langchain/test/fixtures/apps/langchain/config/plugin.js @@ -33,4 +33,9 @@ exports.tracer = { enable: true, }; +exports.teggDal = { + package: '@eggjs/tegg-dal-plugin', + enable: true, +}; + exports.watcher = false; diff --git a/plugin/langchain/test/fixtures/sse-mcp-server/http.ts b/plugin/langchain/test/fixtures/sse-mcp-server/http.ts index 3392bb56..c5bf9b1c 100644 --- a/plugin/langchain/test/fixtures/sse-mcp-server/http.ts +++ b/plugin/langchain/test/fixtures/sse-mcp-server/http.ts @@ -13,6 +13,8 @@ const server = new McpServer({ // Add an addition tool server.registerTool('add', { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore inputSchema: { a: z.number(), b: z.number() }, }, async ({ a, b }) => ({ diff --git a/plugin/langchain/test/llm.test.ts b/plugin/langchain/test/llm.test.ts index cfd43e62..d160f801 100644 --- a/plugin/langchain/test/llm.test.ts +++ b/plugin/langchain/test/llm.test.ts @@ -2,6 +2,12 @@ import mm from 'egg-mock'; import path from 'path'; import assert from 'assert'; import Tracer from 'egg-tracer/lib/tracer'; +import type { MysqlFilesystem } from '../lib/filesystem/MysqlFilesystem'; +import type { AgentFS } from '../lib/filesystem/agentfs/agentfs'; +import { MysqlDataSource } from '@eggjs/dal-runtime'; +import SqlString from 'sqlstring'; +import os from 'node:os'; + describe('plugin/langchain/test/llm.test.ts', () => { // https://github.com/langchain-ai/langchainjs/blob/main/libs/langchain/package.json#L9 @@ -12,22 +18,12 @@ describe('plugin/langchain/test/llm.test.ts', () => { const { ChatOpenAIModel } = require('../lib/ChatOpenAI'); // eslint-disable-next-line @typescript-eslint/no-var-requires const { BaseChatOpenAI } = require('@langchain/openai'); + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { FileSystemInjectName, FileSystemQualifierAttribute } = require('@eggjs/tegg-langchain-decorator'); let app; before(async () => { await startSSEServer(17283); - }); - - after(async () => { - await app.close(); - await stopSSEServer(); - }); - - afterEach(() => { - mm.restore(); - }); - - before(async () => { mm(process.env, 'EGG_TYPESCRIPT', true); mm(process, 'cwd', () => { return path.join(__dirname, '..'); @@ -39,8 +35,28 @@ describe('plugin/langchain/test/llm.test.ts', () => { await app.ready(); }); - after(() => { - return app.close(); + after(async () => { + const mysqlFilesystem: MysqlFilesystem = await app.getEggObjectFromName(FileSystemInjectName, [{ + attribute: FileSystemQualifierAttribute, + value: 'foo', + }]); + const mysql = mysqlFilesystem.mysql; + const delConfig = 'DELETE FROM fs_config'; + await mysql.query(delConfig, [], { executeType: 'execute' }); + const delData = 'DELETE FROM fs_data'; + await mysql.query(delData, [], { executeType: 'execute' }); + const delDentry = 'DELETE FROM fs_dentry'; + await mysql.query(delDentry, [], { executeType: 'execute' }); + const delInode = 'DELETE FROM fs_inode'; + await mysql.query(delInode, [], { executeType: 'execute' }); + const delSymlink = 'DELETE FROM fs_symlink'; + await mysql.query(delSymlink, [], { executeType: 'execute' }); + await app.close(); + await stopSSEServer(); + }); + + afterEach(async () => { + mm.restore(); }); it('should work', async () => { @@ -81,5 +97,1299 @@ describe('plugin/langchain/test/llm.test.ts', () => { app.expectLog(/Executing FooNode thread_id is 1/); app.expectLog(/traceId=test-trace-id/); }); + + describe('filesystem should work', () => { + let fs: AgentFS; + let mysql: MysqlDataSource; + before(async () => { + const mysqlFilesystem: MysqlFilesystem = await app.getEggObjectFromName(FileSystemInjectName, [{ + attribute: FileSystemQualifierAttribute, + value: 'foo', + }]); + fs = mysqlFilesystem.agentFs; + mysql = mysqlFilesystem.mysql; + }); + // File Write Operations + it('should write and read a simple text file', async () => { + await fs.writeFile('/test.txt', 'Hello, World!'); + const content = await fs.readFile('/test.txt', 'utf8'); + assert.deepStrictEqual(content, 'Hello, World!'); + await fs.rm('/test.txt'); + }); + + it('should respect encoding option when writing string data', async () => { + await fs.writeFile('/enc.txt', 'hello', { encoding: 'utf16le' }); + const data = await fs.readFile('/enc.txt') as Buffer; + assert.deepStrictEqual(data.toString('utf16le'), 'hello'); + await fs.rm('/enc.txt'); + }); + + it('should throw EISDIR when attempting to write to a directory path', async () => { + await fs.writeFile('/dir/file.txt', 'content'); // create directory + await assert.rejects(fs.writeFile('/dir', 'nope'), { code: 'EISDIR' }); + }); + + it('should throw ENOTDIR when a parent path component is a file', async () => { + await fs.writeFile('/a', 'file-content'); // create file + await assert.rejects(fs.writeFile('/a/b.txt', 'child'), { code: 'ENOTDIR' }); + await fs.rm('/a', { recursive: true }); + }); + + it('should write and read files in subdirectories', async () => { + await fs.writeFile('/dir/subdir/file.txt', 'nested content'); + const content = await fs.readFile('/dir/subdir/file.txt', 'utf8'); + assert.deepStrictEqual(content, 'nested content'); + await fs.rm('/dir', { recursive: true }); + }); + + it('should overwrite existing file', async () => { + await fs.writeFile('/overwrite.txt', 'original content'); + await fs.writeFile('/overwrite.txt', 'new content'); + const content = await fs.readFile('/overwrite.txt', 'utf8'); + assert.deepStrictEqual(content, 'new content'); + await fs.rm('/overwrite.txt'); + }); + + it('should handle empty file content', async () => { + await fs.writeFile('/empty.txt', ''); + const content = await fs.readFile('/empty.txt', 'utf8'); + assert.deepStrictEqual(content, ''); + await fs.rm('/empty.txt'); + }); + + it('should handle large file content', async () => { + const largeContent = 'x'.repeat(100000); + await fs.writeFile('/large.txt', largeContent); + const content = await fs.readFile('/large.txt', 'utf8'); + assert.deepStrictEqual(content, largeContent); + await fs.rm('/large.txt'); + }); + + // it('should handle files with special characters in content', async () => { + // const specialContent = 'Special chars: \n\t\r\''\\'; + // await fs.writeFile('/special.txt', specialContent); + // const content = await fs.readFile('/special.txt', 'utf8'); + // assert.deepStrictEqual(content, specialContent); + // await fs.rm('/special.txt'); + // }); + + // File Read Operations + it('should throw error when reading non-existent file', async () => { + await assert.rejects(fs.readFile('/non-existent.txt'), { code: 'ENOENT' }); + }); + + it('should throw EISDIR when attempting to read a directory path', async () => { + await fs.writeFile('/dir/file.txt', 'content'); + await assert.rejects(fs.readFile('/dir'), { code: 'EISDIR' }); + await fs.rm('/dir', { recursive: true }); + }); + + it('should read multiple different files', async () => { + await fs.writeFile('/file1.txt', 'content 1'); + await fs.writeFile('/file2.txt', 'content 2'); + await fs.writeFile('/file3.txt', 'content 3'); + + assert.deepStrictEqual(await fs.readFile('/file1.txt', 'utf8'), 'content 1'); + assert.deepStrictEqual(await fs.readFile('/file2.txt', 'utf8'), 'content 2'); + assert.deepStrictEqual(await fs.readFile('/file3.txt', 'utf8'), 'content 3'); + + await fs.rm('/file1.txt'); + await fs.rm('/file2.txt'); + await fs.rm('/file3.txt'); + }); + + // Directory Operations + it('should create a directory with mkdir()', async () => { + await fs.mkdir('/newdir'); + const entries = await fs.readdir('/'); + assert.deepStrictEqual(entries.includes('newdir'), true); + await fs.rm('/newdir', { recursive: true }); + }); + + it('should throw EEXIST when mkdir() is called on an existing directory', async () => { + await fs.mkdir('/exists'); + await assert.rejects(fs.mkdir('/exists'), { code: 'EEXIST' }); + await fs.rm('/exists', { recursive: true }); + }); + + it('should throw ENOENT when parent directory does not exist', async () => { + await assert.rejects(fs.mkdir('/missing-parent/child'), { code: 'ENOENT' }); + }); + + it('should list files in root directory', async () => { + await fs.writeFile('/file1.txt', 'content 1'); + await fs.writeFile('/file2.txt', 'content 2'); + await fs.writeFile('/file3.txt', 'content 3'); + + const files = await fs.readdir('/'); + assert.deepStrictEqual(files, [ 'file1.txt', 'file2.txt', 'file3.txt' ]); + + await fs.rm('/file1.txt'); + await fs.rm('/file2.txt'); + await fs.rm('/file3.txt'); + }); + + it('should list files in subdirectory', async () => { + await fs.writeFile('/dir/file1.txt', 'content 1'); + await fs.writeFile('/dir/file2.txt', 'content 2'); + await fs.writeFile('/other/file3.txt', 'content 3'); + + const files = await fs.readdir('/dir'); + + assert.deepStrictEqual(files, [ 'file1.txt', 'file2.txt' ]); + + await fs.rm('/dir', { recursive: true }); + await fs.rm('/other', { recursive: true }); + }); + + it('should return empty array for empty directory', async () => { + await fs.writeFile('/dir/file.txt', 'content'); + // /dir exists but is not empty, root exists and should be empty except for 'dir' + const files = await fs.readdir('/'); + assert.deepStrictEqual(files.includes('dir'), true); + await fs.rm('/dir', { recursive: true }); + }); + + it('should distinguish between files in different directories', async () => { + await fs.writeFile('/dir1/file.txt', 'content 1'); + await fs.writeFile('/dir2/file.txt', 'content 2'); + + const files1 = await fs.readdir('/dir1'); + const files2 = await fs.readdir('/dir2'); + + assert.deepStrictEqual(files1, [ 'file.txt' ]); + assert.deepStrictEqual(files2, [ 'file.txt' ]); + + await fs.rm('/dir1', { recursive: true }); + await fs.rm('/dir2', { recursive: true }); + }); + + it('should list subdirectories within a directory', async () => { + await fs.writeFile('/parent/child1/file.txt', 'content'); + await fs.writeFile('/parent/child2/file.txt', 'content'); + await fs.writeFile('/parent/file.txt', 'content'); + + const entries = await fs.readdir('/parent'); + assert.deepStrictEqual(entries.includes('file.txt'), true); + assert.deepStrictEqual(entries.includes('child1'), true); + assert.deepStrictEqual(entries.includes('child2'), true); + + await fs.rm('/parent', { recursive: true }); + }); + + it('should handle nested directory structures', async () => { + await fs.writeFile('/a/b/c/d/file.txt', 'deep content'); + const files = await fs.readdir('/a/b/c/d'); + assert.deepStrictEqual(files.includes('file.txt'), true); + + await fs.rm('/a', { recursive: true }); + }); + + it('should throw ENOTDIR when attempting to readdir a file path', async () => { + await fs.writeFile('/notadir.txt', 'content'); + await assert.rejects(fs.readdir('/notadir.txt'), { code: 'ENOTDIR' }); + await fs.rm('/notadir.txt'); + }); + + // File Delete Operations + it('should delete an existing file', async () => { + await fs.writeFile('/delete-me.txt', 'content'); + await fs.unlink('/delete-me.txt'); + await assert.rejects(fs.readFile('/delete-me.txt')); + }); + + it('should handle deleting non-existent file', async () => { + await assert.rejects(fs.unlink('/non-existent.txt'), { code: 'ENOENT' }); + }); + + it('should delete file and update directory listing', async () => { + await fs.writeFile('/dir/file1.txt', 'content 1'); + await fs.writeFile('/dir/file2.txt', 'content 2'); + + await fs.unlink('/dir/file1.txt'); + + const files = await fs.readdir('/dir'); + assert.deepStrictEqual(files, [ 'file2.txt' ]); + + await fs.rm('/dir', { recursive: true }); + }); + + it('should allow recreating deleted file', async () => { + await fs.writeFile('/recreate.txt', 'original'); + await fs.unlink('/recreate.txt'); + await fs.writeFile('/recreate.txt', 'new content'); + const content = await fs.readFile('/recreate.txt', 'utf8'); + assert.deepStrictEqual(content, 'new content'); + await fs.rm('/recreate.txt'); + }); + + it('should throw EISDIR when attempting to unlink a directory', async () => { + await fs.writeFile('/adir/file.txt', 'content'); + await assert.rejects(fs.unlink('/adir'), { code: 'EISDIR' }); + await fs.rm('/adir', { recursive: true }); + }); + + // rm() Operations + it('should remove a file', async () => { + await fs.writeFile('/rmfile.txt', 'content'); + await fs.rm('/rmfile.txt'); + await assert.rejects(fs.readFile('/rmfile.txt'), { code: 'ENOENT' }); + }); + + it('should not throw when force=true and path does not exist', async () => { + await fs.rm('/does-not-exist', { force: true }); + }); + + it('should throw ENOENT when force=false and path does not exist', async () => { + await assert.rejects(fs.rm('/does-not-exist'), { code: 'ENOENT' }); + }); + + it('should throw EISDIR when trying to rm a directory without recursive', async () => { + await fs.mkdir('/rmdir'); + await assert.rejects(fs.rm('/rmdir'), { code: 'EISDIR' }); + await fs.rm('/rmdir', { recursive: true }); + }); + + it('should remove a directory recursively', async () => { + await fs.writeFile('/tree/a/b/c.txt', 'content'); + await fs.rm('/tree', { recursive: true }); + await assert.rejects(fs.readdir('/tree'), { code: 'ENOENT' }); + const root = await fs.readdir('/'); + assert.deepStrictEqual(root.includes('tree'), false); + }); + + // rmdir() Operations + it('should remove an empty directory', async () => { + await fs.mkdir('/emptydir'); + await fs.rmdir('/emptydir'); + await assert.rejects(fs.readdir('/emptydir'), { code: 'ENOENT' }); + const root = await fs.readdir('/'); + assert.deepStrictEqual(root.includes('emptydir'), false); + }); + + it('should throw ENOTEMPTY when directory is not empty', async () => { + await fs.writeFile('/nonempty/file.txt', 'content'); + await assert.rejects(fs.rmdir('/nonempty'), { code: 'ENOTEMPTY' }); + await fs.rm('/nonempty', { recursive: true }); + }); + + it('should throw ENOTDIR when path is a file', async () => { + await fs.writeFile('/afile', 'content'); + await assert.rejects(fs.rmdir('/afile'), { code: 'ENOTDIR' }); + await fs.rm('/afile'); + }); + + it('should throw EPERM when attempting to remove root', async () => { + await assert.rejects(fs.rmdir('/'), { code: 'EPERM' }); + }); + + // rename() Operations + it('should rename a file', async () => { + await fs.writeFile('/a.txt', 'hello'); + await fs.rename('/a.txt', '/b.txt'); + await assert.rejects(fs.readFile('/a.txt'), { code: 'ENOENT' }); + const content = await fs.readFile('/b.txt', 'utf8'); + assert.deepStrictEqual(content, 'hello'); + await fs.rm('/b.txt'); + }); + + it('should rename a directory and preserve its contents', async () => { + await fs.writeFile('/olddir/sub/file.txt', 'content'); + await fs.rename('/olddir', '/newdir'); + await assert.rejects(fs.readdir('/olddir'), { code: 'ENOENT' }); + const content = await fs.readFile('/newdir/sub/file.txt', 'utf8'); + assert.deepStrictEqual(content, 'content'); + await fs.rm('/newdir', { recursive: true }); + }); + + it('should overwrite destination file if it exists', async () => { + await fs.writeFile('/src.txt', 'src'); + await fs.writeFile('/dst.txt', 'dst'); + await fs.rename('/src.txt', '/dst.txt'); + await assert.rejects(fs.readFile('/src.txt'), { code: 'ENOENT' }); + const content = await fs.readFile('/dst.txt', 'utf8'); + assert.deepStrictEqual(content, 'src'); + await fs.rm('/dst.txt'); + }); + + it('should throw EISDIR when renaming a file onto a directory', async () => { + await fs.writeFile('/dir/file.txt', 'content'); + await fs.writeFile('/file.txt', 'content'); + await assert.rejects(fs.rename('/file.txt', '/dir'), { code: 'EISDIR' }); + await fs.rm('/dir', { recursive: true }); + await fs.rm('/file.txt'); + }); + + it('should throw ENOTDIR when renaming a directory onto a file', async () => { + await fs.mkdir('/somedir'); + await fs.writeFile('/somefile', 'content'); + await assert.rejects(fs.rename('/somedir', '/somefile'), { code: 'ENOTDIR' }); + await fs.rm('/somedir', { recursive: true }); + await fs.rm('/somefile'); + }); + + it('should replace an existing empty directory', async () => { + await fs.mkdir('/fromdir'); + await fs.mkdir('/todir'); + await fs.rename('/fromdir', '/todir'); + const root = await fs.readdir('/'); + assert.deepStrictEqual(root.includes('fromdir'), false); + assert.deepStrictEqual(root.includes('todir'), true); + await assert.rejects(fs.readdir('/fromdir'), { code: 'ENOENT' }); + await fs.rm('/todir', { recursive: true }); + }); + + it('should throw ENOTEMPTY when replacing a non-empty directory', async () => { + await fs.mkdir('/fromdir'); + await fs.writeFile('/todir/file.txt', 'content'); + await assert.rejects(fs.rename('/fromdir', '/todir'), { code: 'ENOTEMPTY' }); + await fs.rm('/fromdir', { recursive: true }); + await fs.rm('/todir', { recursive: true }); + }); + + it('should throw EPERM when attempting to rename root', async () => { + await assert.rejects(fs.rename('/', '/x'), { code: 'EPERM' }); + }); + + it('should throw EINVAL when renaming a directory into its own subdirectory', async () => { + await fs.writeFile('/cycle/sub/file.txt', 'content'); + await assert.rejects(fs.rename('/cycle', '/cycle/sub/moved'), { code: 'EINVAL' }); + await fs.rm('/cycle', { recursive: true }); + }); + + + // copyFile() Operations + it('should copy a file', async () => { + await fs.writeFile('/src.txt', 'hello'); + await fs.copyFile('/src.txt', '/dst.txt'); + const srcContent = await fs.readFile('/src.txt', 'utf8'); + const dstContent = await fs.readFile('/dst.txt', 'utf8'); + assert.deepStrictEqual(srcContent, dstContent); + await fs.rm('/src.txt'); + await fs.rm('/dst.txt'); + }); + + it('should overwrite destination if it exists', async () => { + await fs.writeFile('/src.txt', 'src'); + await fs.writeFile('/dst.txt', 'dst'); + await fs.copyFile('/src.txt', '/dst.txt'); + const dstContent = await fs.readFile('/dst.txt', 'utf8'); + assert.deepStrictEqual(dstContent, 'src'); + await fs.rm('/src.txt'); + await fs.rm('/dst.txt'); + }); + + it('should throw ENOENT when source does not exist', async () => { + await assert.rejects(fs.copyFile('/nope.txt', '/out.txt'), { code: 'ENOENT' }); + }); + + it('should throw ENOENT when destination parent does not exist', async () => { + await fs.writeFile('/src3.txt', 'content'); + await assert.rejects(fs.copyFile('/src3.txt', '/missing/child.txt'), { code: 'ENOENT' }); + await fs.rm('/src3.txt'); + }); + + it('should throw EISDIR when source is a directory', async () => { + await fs.mkdir('/asrcdir'); + await assert.rejects(fs.copyFile('/asrcdir', '/out2.txt'), { code: 'EISDIR' }); + await fs.rm('/asrcdir', { recursive: true }); + }); + + it('should throw EISDIR when destination is a directory', async () => { + await fs.writeFile('/src4.txt', 'content'); + await fs.mkdir('/adstdir'); + await assert.rejects(fs.copyFile('/src4.txt', '/adstdir'), { code: 'EISDIR' }); + await fs.rm('/src4.txt'); + await fs.rm('/adstdir', { recursive: true }); + }); + + it('should throw EINVAL when source and destination are the same', async () => { + await fs.writeFile('/same.txt', 'content'); + await assert.rejects(fs.copyFile('/same.txt', '/same.txt'), { code: 'EINVAL' }); + await fs.rm('/same.txt'); + }); + + // access() Operations + it('should resolve when a file exists', async () => { + await fs.writeFile('/exists.txt', 'content'); + await fs.rm('/exists.txt'); + }); + + it('should resolve when a directory exists', async () => { + await fs.mkdir('/existsdir'); + await fs.rm('/existsdir', { recursive: true }); + }); + + it('should throw ENOENT when path does not exist', async () => { + await assert.rejects(fs.access('/does-not-exist'), { code: 'ENOENT' }); + }); + + // Path Handling + it('should handle paths with trailing slashes', async () => { + await fs.writeFile('/dir/file.txt', 'content'); + const files1 = await fs.readdir('/dir'); + const files2 = await fs.readdir('/dir/'); + assert.deepStrictEqual(files1, files2); + await fs.rm('/dir', { recursive: true }); + }); + + it('should handle paths with special characters', async () => { + const specialPath = '/dir-with-dash/file_with_underscore.txt'; + await fs.writeFile(specialPath, 'content'); + const content = await fs.readFile(specialPath, 'utf8'); + assert.deepStrictEqual(content, 'content'); + await fs.rm('/dir-with-dash', { recursive: true }); + }); + + // Concurrent Operations + it('should handle concurrent writes to different files', async () => { + const operations = Array.from({ length: 10 }, (_, i) => + fs.writeFile(`/concurrent-${i}.txt`, `content ${i}`), + ); + await Promise.all(operations); + + // Verify all files were created + for (let i = 0; i < 10; i++) { + const content = await fs.readFile(`/concurrent-${i}.txt`, 'utf8'); + assert.deepStrictEqual(content, `content ${i}`); + await fs.rm(`/concurrent-${i}.txt`); + } + }); + + it('should handle concurrent reads', async () => { + await fs.writeFile('/concurrent-read.txt', 'shared content'); + + await Promise.all( + Array.from({ length: 10 }, () => + fs.readFile('/concurrent-read.txt', 'utf8'), + ), + ); + for (let i = 0; i < 10; i++) { + const content = await fs.readFile('/concurrent-read.txt', 'utf8'); + assert.deepStrictEqual(content, 'shared content'); + } + await fs.rm('/concurrent-read.txt'); + }); + + // File System Integrity + it('should maintain file hierarchy integrity', async () => { + await fs.writeFile('/root.txt', 'root'); + await fs.writeFile('/dir1/file.txt', 'dir1'); + await fs.writeFile('/dir2/file.txt', 'dir2'); + await fs.writeFile('/dir1/subdir/file.txt', 'subdir'); + assert.deepStrictEqual(await fs.readFile('/root.txt', 'utf8'), 'root'); + assert.deepStrictEqual(await fs.readFile('/dir1/file.txt', 'utf8'), 'dir1'); + assert.deepStrictEqual(await fs.readFile('/dir2/file.txt', 'utf8'), 'dir2'); + assert.deepStrictEqual(await fs.readFile('/dir1/subdir/file.txt', 'utf8'), 'subdir'); + + const rootFiles = await fs.readdir('/'); + assert.deepStrictEqual(rootFiles.includes('root.txt'), true); + assert.deepStrictEqual(rootFiles.includes('dir1'), true); + assert.deepStrictEqual(rootFiles.includes('dir2'), true); + await fs.rm('/root.txt'); + await fs.rm('/dir1', { recursive: true }); + await fs.rm('/dir2', { recursive: true }); + }); + + it('should support multiple files with same name in different directories', async () => { + await fs.writeFile('/dir1/config.json', '{\'version\': 1}'); + await fs.writeFile('/dir2/config.json', '{\'version\': 2}'); + assert.deepStrictEqual(await fs.readFile('/dir1/config.json', 'utf8'), '{\'version\': 1}'); + assert.deepStrictEqual(await fs.readFile('/dir2/config.json', 'utf8'), '{\'version\': 2}'); + + await fs.rm('/dir1', { recursive: true }); + await fs.rm('/dir2', { recursive: true }); + }); + + // Chunk Size Boundary Tests + async function getChunkCount(path: string): Promise { + // const stmt = mysql.prepare(` + // SELECT COUNT(*) as count FROM fs_data + // WHERE ino = (SELECT ino FROM fs_dentry WHERE parent_ino = 1 AND name = ?) + // `); + // const sql = SqlString.format(` + // SELECT COUNT(*) as count FROM fs_data + // WHERE ino = (SELECT ino FROM fs_dentry WHERE parent_ino = 1 AND name = ?) + // `, [ path ]); + const pathParts = path.split('/').filter(p => p); + // const name = pathParts[pathParts.length - 1]; + + // For simple paths, get the inode from the path + // const lookupStmt = db.prepare(` + // SELECT d.ino FROM fs_dentry d WHERE d.parent_ino = ? AND d.name = ? + // `); + + let currentIno = 1; // root + for (const part of pathParts) { + const sql = SqlString.format( + 'SELECT ino FROM fs_dentry WHERE parent_ino = ? AND name = ?', + [ currentIno, part ], + ); + const result = (await mysql.query(sql, [], { executeType: 'execute' }) as { ino: number }[])[0]; + if (!result) return 0; + currentIno = result.ino; + } + + const countSql = SqlString.format( + 'SELECT COUNT(*) as count FROM fs_data WHERE ino = ?', + [ currentIno ], + ); + const result = (await mysql.query(countSql, [], { executeType: 'execute' }) as { count: number }[])[0]; + return Number(result.count); + } + + it('should write file smaller than chunk size', async () => { + // Write a file smaller than chunk_size (100 bytes) + const data = 'x'.repeat(100); + await fs.writeFile('/small.txt', data); + + // Read it back + const readData = await fs.readFile('/small.txt', 'utf8'); + assert.deepStrictEqual(readData, data); + + // Verify only 1 chunk was created + const chunkCount = await getChunkCount('/small.txt'); + assert.deepStrictEqual(chunkCount, 1); + await fs.rm('/small.txt'); + }); + it('should write file exactly chunk size', async () => { + const chunkSize = fs.getChunkSize(); + // Write exactly chunk_size bytes + const data = Buffer.alloc(chunkSize); + for (let i = 0; i < chunkSize; i++) { + data[i] = i % 256; + } + await fs.writeFile('/exact.txt', data); + + // Read it back + const readData = (await fs.readFile('/exact.txt')) as Buffer; + assert.deepStrictEqual(readData.length, chunkSize); + + // Verify only 1 chunk was created + const chunkCount = await getChunkCount('/exact.txt'); + assert.deepStrictEqual(chunkCount, 1); + await fs.rm('/exact.txt'); + }); + + it('should write file one byte over chunk size', async () => { + const chunkSize = fs.getChunkSize(); + // Write chunk_size + 1 bytes + const data = Buffer.alloc(chunkSize + 1); + for (let i = 0; i <= chunkSize; i++) { + data[i] = i % 256; + } + await fs.writeFile('/overflow.txt', data); + + // Read it back + const readData = (await fs.readFile('/overflow.txt')) as Buffer; + assert.deepStrictEqual(readData.length, chunkSize + 1); + + // Verify 2 chunks were created + const chunkCount = await getChunkCount('/overflow.txt'); + assert.deepStrictEqual(chunkCount, 2); + await fs.rm('/overflow.txt'); + }); + + it('should write file spanning multiple chunks', async () => { + const chunkSize = fs.getChunkSize(); + // Write ~2.5 chunks worth of data + const dataSize = Math.floor(chunkSize * 2.5); + const data = Buffer.alloc(dataSize); + for (let i = 0; i < dataSize; i++) { + data[i] = i % 256; + } + await fs.writeFile('/multi.txt', data); + + // Read it back + const readData = (await fs.readFile('/multi.txt')) as Buffer; + assert.deepStrictEqual(readData.length, dataSize); + + // Verify 3 chunks were created + const chunkCount = await getChunkCount('/multi.txt'); + assert.deepStrictEqual(chunkCount, 3); + await fs.rm('/multi.txt'); + }); + + // Data Integrity Tests + it('should roundtrip data byte-for-byte', async () => { + const chunkSize = fs.getChunkSize(); + // Create data that spans chunk boundaries with identifiable patterns + const dataSize = chunkSize * 3 + 123; // Odd size spanning 4 chunks + + const data = Buffer.alloc(dataSize); + for (let i = 0; i < dataSize; i++) { + data[i] = i % 256; + } + await fs.writeFile('/roundtrip.bin', data); + + const readData = (await fs.readFile('/roundtrip.bin')) as Buffer; + assert.deepStrictEqual(readData.length, dataSize); + + // Verify chunk count + const chunkCount = await getChunkCount('/roundtrip.bin'); + assert.deepStrictEqual(chunkCount, 4); + await fs.rm('/roundtrip.bin'); + }); + + + it('should handle binary data with null bytes', async () => { + const chunkSize = fs.getChunkSize(); + // Create data with null bytes at chunk boundaries + const data = Buffer.alloc(chunkSize * 2 + 100); + // Put nulls at the chunk boundary + data[chunkSize - 1] = 0; + data[chunkSize] = 0; + data[chunkSize + 1] = 0; + // Put some non-null bytes around + data[chunkSize - 2] = 0xff; + data[chunkSize + 2] = 0xff; + + await fs.writeFile('/nulls.bin', data); + const readData = (await fs.readFile('/nulls.bin')) as Buffer; + + assert.deepStrictEqual(readData[chunkSize - 2], 0xff); + assert.deepStrictEqual(readData[chunkSize - 1], 0); + assert.deepStrictEqual(readData[chunkSize], 0); + assert.deepStrictEqual(readData[chunkSize + 1], 0); + assert.deepStrictEqual(readData[chunkSize + 2], 0xff); + await fs.rm('/nulls.bin'); + }); + + it('should preserve chunk ordering', async () => { + const chunkSize = fs.getChunkSize(); + // Create sequential bytes spanning multiple chunks + const dataSize = chunkSize * 5; + const data = Buffer.alloc(dataSize); + for (let i = 0; i < dataSize; i++) { + data[i] = i % 256; + } + await fs.writeFile('/sequential.bin', data); + + const readData = (await fs.readFile('/sequential.bin')) as Buffer; + + // Verify every byte is in the correct position + for (let i = 0; i < dataSize; i++) { + assert.deepStrictEqual(readData[i], i % 256); + } + await fs.rm('/sequential.bin'); + }); + + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + // async function getIno(path: string): Promise { + // const pathParts = path.split('/').filter(p => p); + + // let currentIno = 1; + // for (const part of pathParts) { + // const sql = SqlString.format( + // 'SELECT ino FROM fs_dentry WHERE parent_ino = ? AND name = ?', + // [ currentIno, part ], + // ); + // const result = (await mysql.query(sql, [], { executeType: 'execute' }) as { ino: number }[])[0]; + // if (!result) return 0; + // currentIno = result.ino; + // } + // return currentIno; + // } + // Edge Case Tests + it('should handle empty file with zero chunks', async () => { + // Write empty file + await fs.writeFile('/empty.txt', ''); + + // Read it back + const readData = await fs.readFile('/empty.txt', 'utf8'); + assert.deepStrictEqual(readData, ''); + + // Verify 0 chunks were created + const chunkCount = await getChunkCount('/empty.txt'); + assert.deepStrictEqual(chunkCount, 0); + + // Verify size is 0 + const stats = await fs.stat('/empty.txt'); + assert.deepStrictEqual(stats.size, 0); + await fs.rm('/empty.txt'); + }); + + it('should overwrite large file with smaller file and clean up chunks', async () => { + const chunkSize = fs.getChunkSize(); + + // Write initial large file (3 chunks) + const initialData = Buffer.alloc(chunkSize * 3); + for (let i = 0; i < chunkSize * 3; i++) { + initialData[i] = i % 256; + } + await fs.writeFile('/overwrite.txt', initialData); + + // const ino = await getIno('/overwrite.txt'); + const initialChunkCount = await getChunkCount('/overwrite.txt'); + assert.deepStrictEqual(initialChunkCount, 3); + + // Overwrite with smaller file (1 chunk) + const newData = 'x'.repeat(100); + await fs.writeFile('/overwrite.txt', newData); + + // Verify old chunks are gone and new data is correct + const readData = await fs.readFile('/overwrite.txt', 'utf8'); + assert.deepStrictEqual(readData, newData); + + const newChunkCount = await getChunkCount('/overwrite.txt'); + assert.deepStrictEqual(newChunkCount, 1); + + // Verify size is updated + const stats = await fs.stat('/overwrite.txt'); + assert.deepStrictEqual(stats.size, 100); + await fs.rm('/overwrite.txt'); + }); + + it('should overwrite small file with larger file', async () => { + const chunkSize = fs.getChunkSize(); + + // Write initial small file (1 chunk) + const initialData = 'x'.repeat(100); + await fs.writeFile('/grow.txt', initialData); + + assert.deepStrictEqual(await getChunkCount('/grow.txt'), 1); + + // Overwrite with larger file (3 chunks) + const newData = Buffer.alloc(chunkSize * 3); + for (let i = 0; i < chunkSize * 3; i++) { + newData[i] = i % 256; + } + await fs.writeFile('/grow.txt', newData); + + // Verify data is correct (no encoding = Buffer) + const readData = (await fs.readFile('/grow.txt')) as Buffer; + assert.deepStrictEqual(readData.length, chunkSize * 3); + assert.deepStrictEqual(await getChunkCount('/grow.txt'), 3); + + await fs.rm('/grow.txt'); + }); + + it('should handle very large file (1MB)', async () => { + // Write 1MB file + const dataSize = 1024 * 1024; + const data = Buffer.alloc(dataSize); + for (let i = 0; i < dataSize; i++) { + data[i] = i % 256; + } + await fs.writeFile('/large.bin', data); + + const readData = (await fs.readFile('/large.bin')) as Buffer; + assert.deepStrictEqual(readData.length, dataSize); + + // Verify correct number of chunks + const chunkSize = fs.getChunkSize(); + const expectedChunks = Math.ceil(dataSize / chunkSize); + const actualChunks = await getChunkCount('/large.bin'); + assert.deepStrictEqual(actualChunks, expectedChunks); + }); + + // Configuration Tests + it('should have default chunk size of 4096', async () => { + const chunkSize = fs.getChunkSize(); + assert.deepStrictEqual(chunkSize, 4096); + }); + + it('should verify chunk_size accessor works correctly', async () => { + const chunkSize = fs.getChunkSize(); + + // Write data and verify chunks match expected based on chunk_size + const data = Buffer.alloc(chunkSize * 2 + 1); + await fs.writeFile('/test.bin', data); + + const pathParts = '/test.bin'.split('/').filter(p => p); + let currentIno = 1; + for (const part of pathParts) { + const sql = SqlString.format( + 'SELECT ino FROM fs_dentry WHERE parent_ino = ? AND name = ?', + [ currentIno, part ], + ); + const result = (await mysql.query(sql, [], { executeType: 'execute' }) as { ino: number }[])[0]; + if (result) currentIno = result.ino; + } + const countStmt = SqlString.format('SELECT COUNT(*) as count FROM fs_data WHERE ino = ?', + [ currentIno ], + ); + const result = (await mysql.query(countStmt, [], { executeType: 'execute' }) as { count: number }[])[0]; + assert.deepStrictEqual(result!.count, '3'); + await fs.rm('/test.bin'); + }); + + it('should persist chunk_size in fs_config table', async () => { + + const sql = SqlString.format( + 'SELECT value FROM fs_config WHERE `key` = ?', + [ 'chunk_size' ], + ); + const result = (await mysql.query(sql) as { value: string }[])[0]; + assert.deepStrictEqual(result!.value, '4096'); + }); + + // Schema Tests + it('should enforce chunk index uniqueness', async () => { + const chunkSize = fs.getChunkSize(); + // Write a file to create chunks + const data = Buffer.alloc(chunkSize * 2); + await fs.writeFile('/unique.txt', data); + const sql = SqlString.format( + 'SELECT ino FROM fs_dentry WHERE parent_ino = ? AND name = ?', + [ 1, 'unique.txt' ], + ); + const result = (await mysql.query(sql, [], { executeType: 'execute' }) as { ino: number }[])[0]; + const ino = result.ino; + const insertSql = SqlString.format( + 'INSERT INTO fs_data (ino, chunk_index, data) VALUES (?, ?, ?)', + [ ino, 0, Buffer.from([ 1, 2, 3 ]) ], + ); + assert.rejects(() => mysql.query(insertSql, [], { executeType: 'execute' })); + await fs.rm('/unique.txt'); + }); + + it('should store chunks with correct ordering in database', async () => { + const chunkSize = fs.getChunkSize(); + // Create 5 chunks with identifiable data + const dataSize = chunkSize * 5; + const data = Buffer.alloc(dataSize); + for (let i = 0; i < dataSize; i++) { + data[i] = i % 256; + } + await fs.writeFile('/ordered.bin', data); + + const sql = SqlString.format( + 'SELECT ino FROM fs_dentry WHERE parent_ino = ? AND name = ?', + [ 1, 'ordered.bin' ], + ); + const result = (await mysql.query(sql, [], { executeType: 'execute' }) as { ino: number }[])[0]; + const ino = result.ino; + + // Query chunks in order + const querySql = SqlString.format( + 'SELECT chunk_index FROM fs_data WHERE ino = ? ORDER BY chunk_index', + [ ino ], + ); + const rows = (await mysql.query(querySql, [], { executeType: 'execute' }) as { chunk_index: number }[]); + + const indices = rows.map(r => r.chunk_index); + assert.deepStrictEqual(indices, [ 0, 1, 2, 3, 4 ]); + await fs.rm('/ordered.bin'); + }); + + // Cleanup Tests + it('should delete all chunks when file is removed', async () => { + const chunkSize = fs.getChunkSize(); + // Create multi-chunk file + const data = Buffer.alloc(chunkSize * 4); + await fs.writeFile('/deleteme.txt', data); + const sql = SqlString.format( + 'SELECT ino FROM fs_dentry WHERE parent_ino = ? AND name = ?', + [ 1, 'deleteme.txt' ], + ); + const result = (await mysql.query(sql, [], { executeType: 'execute' }) as { ino: number }[])[0]; + const ino = result.ino; + + // const countStmt = db.prepare( + // 'SELECT COUNT(*) as count FROM fs_data WHERE ino = ?' + // ); + // const beforeResult = (await countStmt.get(ino)) as { count: number }; + // expect(beforeResult.count).toBe(4); + + const countStmt = SqlString.format('SELECT COUNT(*) as count FROM fs_data WHERE ino = ?', + [ ino ], + ); + const beforeResult = (await mysql.query(countStmt, [], { executeType: 'execute' }) as { count: number }[])[0]; + assert.deepStrictEqual(Number(beforeResult.count), 4); + + // Delete the file + await fs.unlink('/deleteme.txt'); + + // Verify all chunks are gone + + const afterResult = (await mysql.query(countStmt, [], { executeType: 'execute' }) as { count: number }[])[0]; + assert.deepStrictEqual(Number(afterResult.count), 0); + }); + + it('should handle multiple files with different sizes correctly', async () => { + const chunkSize = fs.getChunkSize(); + + // Create files of various sizes + const files: [string, number][] = [ + [ '/tiny.txt', 10 ], + [ '/small.txt', Math.floor(chunkSize / 2) ], + [ '/exact.txt', chunkSize ], + [ '/medium.txt', chunkSize * 2 + 100 ], + [ '/large.txt', chunkSize * 5 ], + ]; + + for (const [ path, size ] of files) { + const data = Buffer.alloc(size); + for (let i = 0; i < size; i++) { + data[i] = i % 256; + } + await fs.writeFile(path, data); + } + + // Verify each file has correct data and chunk count + for (const [ path, size ] of files) { + const readData = (await fs.readFile(path)) as Buffer; + // expect(readData.length).toBe(size); + assert.deepStrictEqual(readData.length, size); + + const expectedChunks = size === 0 ? 0 : Math.ceil(size / chunkSize); + + const name = path.split('/').pop()!; + const sql = SqlString.format( + 'SELECT d.ino FROM fs_dentry d WHERE d.parent_ino = ? AND d.name = ?', + [ 1, name ], + ); + const result = (await mysql.query(sql, [], { executeType: 'execute' }) as { ino: number }[])[0]; + const countSql = SqlString.format('SELECT COUNT(*) as count FROM fs_data WHERE ino = ?', + [ result.ino ], + ); + const countResult = (await mysql.query(countSql, [], { executeType: 'execute' }) as { count: number }[])[0]; + + assert.deepStrictEqual(Number(countResult.count), expectedChunks); + } + for (const [ path ] of files) { + await fs.unlink(path); + } + }); + }); + + describe('filesystemBackend should work', () => { + let backend: MysqlFilesystem; + let fs: AgentFS; + async function writeFile(filePath: string, content: string) { + try { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + } catch (e) { + if (e.code !== 'EEXIST') { + throw e; + } + } + await fs.writeFile(filePath, content, { encoding: 'utf-8' }); + } + + /** + * Helper to create a unique temporary directory for each test + */ + async function createTempDir(): Promise { + const tempdir = path.join(os.tmpdir(), 'deepagents-test'); + await fs.mkdir(tempdir, { recursive: true }); + return tempdir; + } + + async function removeDir(dirPath: string) { + try { + await fs.rm(dirPath, { recursive: true, force: true }); + } catch { + // Ignore errors during cleanup + } + } + before(async () => { + backend = await app.getEggObjectFromName(FileSystemInjectName, [{ + attribute: FileSystemQualifierAttribute, + value: 'foo', + }]); + fs = backend.agentFs; + }); + let tmpDir: string; + + beforeEach(async () => { + tmpDir = await createTempDir(); + }); + + afterEach(async () => { + await removeDir(tmpDir); + }); + it('should work in normal mode with absolute paths', async () => { + const root = tmpDir; + const f1 = path.join(root, 'a.txt'); + const f2 = path.join(root, 'dir', 'b.py'); + await writeFile(f1, 'hello fs'); + await writeFile(f2, 'print(\'x\')\nhello'); + + backend.cwd = root; + + const infos = await backend.lsInfo(root); + const paths = new Set(infos.map(i => i.path)); + assert.deepStrictEqual(paths.has(f1), true); + assert.deepStrictEqual(paths.has(f2), false); + assert.deepStrictEqual(paths.has(path.join(root, 'dir') + path.sep), true); + + const txt = await backend.read(f1); + assert.deepStrictEqual(txt.includes('hello fs'), true); + + const editMsg = await backend.edit(f1, 'fs', 'filesystem', false); + assert.deepStrictEqual(editMsg.error, undefined); + assert.deepStrictEqual(editMsg.occurrences, 1); + + const writeMsg = await backend.write( + path.join(root, 'new.txt'), + 'new content', + ); + assert.deepStrictEqual(writeMsg.error, undefined); + assert.deepStrictEqual(writeMsg.path, path.join(root, 'new.txt')); + + const matches = await backend.grepRaw('hello', root); + assert.deepStrictEqual(Array.isArray(matches), true); + if (Array.isArray(matches)) { + assert.deepStrictEqual(matches.some(m => m.path.endsWith('a.txt')), true); + } + + const globResults = await backend.globInfo('**/*.py', root); + assert.deepStrictEqual(globResults.some(i => i.path === f2), true); + }); + + + it('should work in virtual mode with sandboxed paths', async () => { + const root = tmpDir; + const f1 = path.join(root, 'a.txt'); + const f2 = path.join(root, 'dir', 'b.md'); + await writeFile(f1, 'hello virtual'); + await writeFile(f2, 'content'); + + backend.cwd = root; + backend.virtualMode = true; + + const infos = await backend.lsInfo('/'); + const paths = new Set(infos.map(i => i.path)); + assert.deepStrictEqual(paths.has('/a.txt'), true); + assert.deepStrictEqual(paths.has('/dir/b.md'), false); + assert.deepStrictEqual(paths.has('/dir/'), true); + + const txt = await backend.read('/a.txt'); + assert.deepStrictEqual(txt.includes('hello virtual'), true); + + const editMsg = await backend.edit('/a.txt', 'virtual', 'virt', false); + assert.deepStrictEqual(editMsg.error, undefined); + assert.deepStrictEqual(editMsg.occurrences, 1); + + const writeMsg = await backend.write('/new.txt', 'x'); + assert.deepStrictEqual(writeMsg.error, undefined); + + const matches = await backend.grepRaw('virt', '/'); + assert.deepStrictEqual(Array.isArray(matches), true); + if (Array.isArray(matches)) { + assert.deepStrictEqual(matches.some(m => m.path.includes('/a.txt')), true); + } + + const globResults = await backend.globInfo('**/*.md', '/'); + assert.deepStrictEqual(globResults.some(i => i.path.includes('/dir/b.md')), true); + + const err = await backend.grepRaw('[', '/'); + assert.deepStrictEqual(typeof err, 'string'); + + const traversalError = await backend.read('/../a.txt'); + assert.deepStrictEqual(traversalError.includes('Error'), true); + assert.deepStrictEqual(traversalError.includes('Path traversal not allowed'), true); + }); + + it('should list nested directories correctly in virtual mode', async () => { + const root = tmpDir; + + const files: Record = { + [path.join(root, 'config.json')]: 'config', + [path.join(root, 'src', 'main.py')]: 'code', + [path.join(root, 'src', 'utils', 'helper.py')]: 'utils code', + [path.join(root, 'src', 'utils', 'common.py')]: 'common utils', + [path.join(root, 'docs', 'readme.md')]: 'documentation', + [path.join(root, 'docs', 'api', 'reference.md')]: 'api docs', + }; + + for (const [ filePath, content ] of Object.entries(files)) { + await writeFile(filePath, content); + } + + + backend.cwd = root; + backend.virtualMode = true; + + const rootListing = await backend.lsInfo('/'); + const rootPaths = rootListing.map(fi => fi.path); + assert.deepStrictEqual(rootPaths.some(rootPath => rootPath.includes('/config.json')), true); + assert.deepStrictEqual(rootPaths.some(rootPath => rootPath.includes('/src/')), true); + assert.deepStrictEqual(rootPaths.some(rootPath => rootPath.includes('/docs/')), true); + assert.deepStrictEqual(rootPaths.some(rootPath => rootPath.includes('/src/main.py')), false); + assert.deepStrictEqual(rootPaths.some(rootPath => rootPath.includes('/src/utils/helper.py')), false); + + const srcListing = await backend.lsInfo('/src/'); + const srcPaths = srcListing.map(fi => fi.path); + assert.deepStrictEqual(srcPaths.some(srcPath => srcPath.includes('/src/main.py')), true); + assert.deepStrictEqual(srcPaths.some(srcPath => srcPath.includes('/src/utils/')), true); + assert.deepStrictEqual(srcPaths.some(srcPath => srcPath.includes('/src/utils/helper.py')), false); + + const utilsListing = await backend.lsInfo('/src/utils/'); + const utilsPaths = utilsListing.map(fi => fi.path); + assert.deepStrictEqual(utilsPaths.some(utilsPath => utilsPath.includes('/src/utils/helper.py')), true); + assert.deepStrictEqual(utilsPaths.some(utilsPath => utilsPath.includes('/src/utils/common.py')), true); + assert.deepStrictEqual(utilsPaths.length, 2); + + const emptyListing = await backend.lsInfo('/nonexistent/'); + assert.deepStrictEqual(emptyListing.length, 0); + }); + + + it('should list nested directories correctly in normal mode', async () => { + const root = tmpDir; + + const files: Record = { + [path.join(root, 'file1.txt')]: 'content1', + [path.join(root, 'subdir', 'file2.txt')]: 'content2', + [path.join(root, 'subdir', 'nested', 'file3.txt')]: 'content3', + }; + + for (const [ filePath, content ] of Object.entries(files)) { + await writeFile(filePath, content); + } + + backend.cwd = root; + backend.virtualMode = false; + + const rootListing = await backend.lsInfo(root); + const rootPaths = rootListing.map(fi => fi.path); + assert.deepStrictEqual(rootPaths.some(rootPath => rootPath.includes(path.join(root, 'file1.txt'))), true); + assert.deepStrictEqual(rootPaths.some(rootPath => rootPath.includes(path.join(root, 'subdir'))), true); + assert.deepStrictEqual(rootPaths.some(rootPath => rootPath.includes(path.join(root, 'subdir', 'file2.txt'))), false); + const subdirListing = await backend.lsInfo(path.join(root, 'subdir')); + const subdirPaths = subdirListing.map(fi => fi.path); + assert.deepStrictEqual(subdirPaths.some(subdirPath => subdirPath.includes(path.join(root, 'subdir', 'file2.txt'))), true); + assert.deepStrictEqual(subdirPaths.some(subdirPath => subdirPath.includes(path.join(root, 'subdir', 'nested'))), true); + assert.deepStrictEqual(subdirPaths.some(subdirPath => subdirPath.includes(path.join(root, 'subdir', 'nested', 'file3.txt'))), false); + }); + it('should handle trailing slashes consistently', async () => { + const root = tmpDir; + + const files: Record = { + [path.join(root, 'file.txt')]: 'content', + [path.join(root, 'dir', 'nested.txt')]: 'nested', + }; + + for (const [ filePath, content ] of Object.entries(files)) { + await writeFile(filePath, content); + } + + backend.cwd = root; + backend.virtualMode = true; + const listingWithSlash = await backend.lsInfo('/'); + assert.deepStrictEqual(listingWithSlash.length > 0, true); + + // const listing = await backend.lsInfo('/'); + // const paths = listing.map(fi => fi.path); + + const listing1 = await backend.lsInfo('/dir/'); + const listing2 = await backend.lsInfo('/dir'); + assert.deepStrictEqual(listing1.length, listing2.length); + assert.deepStrictEqual(listing1.map(fi => fi.path), listing2.map(fi => fi.path)); + + const empty = await backend.lsInfo('/nonexistent/'); + assert.deepStrictEqual(empty.length, 0); + }); + + it('should handle large file writes correctly', async () => { + const root = tmpDir; + backend.cwd = root; + backend.virtualMode = true; + + const largeContent = 'f'.repeat(10000); + const writeResult = await backend.write('/large_file.txt', largeContent); + + assert.deepStrictEqual(writeResult.error, undefined); + assert.deepStrictEqual(writeResult.path?.includes('/large_file.txt'), true); + + const readContent = await backend.read('/large_file.txt'); + assert.deepStrictEqual(readContent.includes(largeContent.substring(0, 100)), true); + + const savedFile = path.join(root, 'large_file.txt'); + await fs.access(savedFile); + }); + + it('should read multiline content', async () => { + const root = tmpDir; + const filePath = path.join(root, 'multiline.txt'); + await writeFile(filePath, 'line1\nline2\nline3'); + + backend.cwd = root; + backend.virtualMode = false; + + const txt = await backend.read(filePath); + assert.deepStrictEqual(txt.includes('line1'), true); + assert.deepStrictEqual(txt.includes('line2'), true); + assert.deepStrictEqual(txt.includes('line3'), true); + }); + + it('should handle empty files', async () => { + const root = tmpDir; + const filePath = path.join(root, 'empty.txt'); + await writeFile(filePath, ''); + + backend.cwd = root; + backend.virtualMode = false; + + const txt = await backend.read(filePath); + assert.deepStrictEqual(txt.includes('empty contents'), true); + }); + + it('should handle files with trailing newlines', async () => { + const root = tmpDir; + const filePath = path.join(root, 'trailing.txt'); + await writeFile(filePath, 'line1\nline2\n'); + backend.cwd = root; + backend.virtualMode = false; + + const txt = await backend.read(filePath); + assert.deepStrictEqual(txt.includes('line1'), true); + assert.deepStrictEqual(txt.includes('line2'), true); + }); + + it('should handle unicode content', async () => { + const root = tmpDir; + const filePath = path.join(root, 'unicode.txt'); + await writeFile(filePath, 'Hello 世界\n🚀 emoji\nΩ omega'); + + backend.cwd = root; + backend.virtualMode = false; + + const txt = await backend.read(filePath); + assert.deepStrictEqual(txt.includes('Hello 世界'), true); + assert.deepStrictEqual(txt.includes('🚀 emoji'), true); + assert.deepStrictEqual(txt.includes('Ω omega'), true); + }); + + it('should handle non-existent files consistently', async () => { + const root = tmpDir; + + backend.cwd = root; + backend.virtualMode = false; + + const nonexistentPath = path.join(root, 'nonexistent.txt'); + + const readResult = await backend.read(nonexistentPath); + assert.deepStrictEqual(readResult.includes('Error'), true); + }); + + it('should handle symlinks securely', async () => { + const root = tmpDir; + const targetFile = path.join(root, 'target.txt'); + const symlinkFile = path.join(root, 'symlink.txt'); + + await writeFile(targetFile, 'target content'); + try { + await fs.symlink(targetFile, symlinkFile); + } catch { + // Skip test if symlinks aren't supported (e.g., Windows without admin) + return; + } + + // const backend = new FilesystemBackend({ + // rootDir: root, + // virtualMode: false, + // }); + + backend.cwd = root; + backend.virtualMode = false; + const readResult = await backend.read(symlinkFile); + assert.deepStrictEqual(readResult.includes('Error'), true); + }); + }); } }); diff --git a/plugin/langchain/typings/index.d.ts b/plugin/langchain/typings/index.d.ts index d3673afa..012e6306 100644 --- a/plugin/langchain/typings/index.d.ts +++ b/plugin/langchain/typings/index.d.ts @@ -110,9 +110,29 @@ export const ChatModelConfigModuleConfigSchema = Type.Object({ export type ChatModelConfigModuleConfigType = Static; +export const FileSystemConfigModuleConfigSchema = Type.Object({ + clients: Type.Record(Type.String(), Type.Object({ + dataSource: Type.String({ + description: 'The name of the data source to use for the MySQL filesystem middleware.', + }), + cwd: Type.Optional(Type.String({ + description: 'The current working directory for the filesystem middleware. Defaults to "/"', + })), + virtualMode: Type.Optional(Type.Boolean({ + description: 'Whether to enable virtual mode, which restricts file operations to within the specified cwd.', + })), + })), +}, { + title: 'FsMiddleware 设置', + name: 'FsMiddleware', +}); + +export type FilesystemConfigModuleConfigType = Static; + declare module '@eggjs/tegg' { export type LangChainModuleConfig = { ChatModel?: ChatModelConfigModuleConfigType; + filesystem?: FilesystemConfigModuleConfigType; }; export interface ModuleConfig extends LangChainModuleConfig {