perf: s3 controller

Author: c121914yu

packages/service/common/s3/buckets/base.ts

@@ -15,6 +15,7 @@ import { addLog } from '../../system/log';
 import { addS3DelJob } from '../mq';
 import { type Readable } from 'node:stream';
 import { type UploadFileByBufferParams, UploadFileByBufferSchema } from '../type';
+import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
 
 export class S3BaseBucket {
   private _client: Client;
@@ -26,7 +27,7 @@ export class S3BaseBucket {
    * @param options the options for the s3 client
    */
   constructor(
-    private readonly bucketName: string,
+    public readonly bucketName: string,
     public options: Partial<S3OptionsType> = defaultS3Options
   ) {
     options = { ...defaultS3Options, ...options };
@@ -56,20 +57,18 @@ export class S3BaseBucket {
     }
 
     const init = async () => {
-      if (!(await this.exist())) {
+      // Not exists bucket, create it
+      if (!(await this.client.bucketExists(this.bucketName))) {
         await this.client.makeBucket(this.bucketName);
       }
       await this.options.afterInit?.();
-      console.log(`S3 init success: ${this.name}`);
+      console.log(`S3 init success: ${this.bucketName}`);
     };
     if (this.options.init) {
       init();
     }
   }
 
-  get name(): string {
-    return this.bucketName;
-  }
   get client(): Client {
     return this._client;
   }
@@ -110,21 +109,17 @@ export class S3BaseBucket {
       copyConditions?: CopyConditions;
     };
   }): ReturnType<Client['copyObject']> {
-    const bucket = this.name;
+    const bucket = this.bucketName;
     if (options?.temporary) {
       await MongoS3TTL.create({
         minioKey: to,
-        bucketName: this.name,
+        bucketName: this.bucketName,
         expiredTime: addHours(new Date(), 24)
       });
     }
     return this.client.copyObject(bucket, to, `${bucket}/${from}`, options?.copyConditions);
   }
 
-  exist(): Promise<boolean> {
-    return this.client.bucketExists(this.name);
-  }
-
   async delete(objectKey: string, options?: RemoveOptions): Promise<void> {
     try {
       if (!objectKey) return Promise.resolve();
@@ -133,39 +128,55 @@ export class S3BaseBucket {
       const fileParsedPrefix = `${path.dirname(objectKey)}/${path.basename(objectKey, path.extname(objectKey))}-parsed`;
       await this.addDeleteJob({ prefix: fileParsedPrefix });
 
-      return await this.client.removeObject(this.name, objectKey, options);
+      return await this.client.removeObject(this.bucketName, objectKey, options);
     } catch (error) {
       if (error instanceof S3Error) {
         if (error.code === 'InvalidObjectName') {
-          addLog.warn(`${this.name} delete object not found: ${objectKey}`, error);
+          addLog.warn(`${this.bucketName} delete object not found: ${objectKey}`, error);
           return Promise.resolve();
         }
       }
       return Promise.reject(error);
     }
   }
 
+  // 列出文件
   listObjectsV2(
     ...params: Parameters<Client['listObjectsV2']> extends [string, ...infer R] ? R : never
   ) {
-    return this.client.listObjectsV2(this.name, ...params);
+    return this.client.listObjectsV2(this.bucketName, ...params);
   }
 
+  // 上传文件
   putObject(...params: Parameters<Client['putObject']> extends [string, ...infer R] ? R : never) {
-    return this.client.putObject(this.name, ...params);
+    return this.client.putObject(this.bucketName, ...params);
   }
 
-  getObject(...params: Parameters<Client['getObject']> extends [string, ...infer R] ? R : never) {
-    return this.client.getObject(this.name, ...params);
+  // 获取文件流
+  getFileStream(
+    ...params: Parameters<Client['getObject']> extends [string, ...infer R] ? R : never
+  ) {
+    return this.client.getObject(this.bucketName, ...params);
   }
 
-  statObject(...params: Parameters<Client['statObject']> extends [string, ...infer R] ? R : never) {
-    return this.client.statObject(this.name, ...params);
+  // 获取文件状态
+  async statObject(
+    ...params: Parameters<Client['statObject']> extends [string, ...infer R] ? R : never
+  ) {
+    try {
+      return await this.client.statObject(this.bucketName, ...params);
+    } catch (error) {
+      if (error instanceof S3Error && error.message === 'Not Found') {
+        return null;
+      }
+      return Promise.reject(error);
+    }
   }
 
+  // 判断文件是否存在
   async isObjectExists(key: string): Promise<boolean> {
     try {
-      await this.client.statObject(this.name, key);
+      await this.client.statObject(this.bucketName, key);
       return true;
     } catch (err) {
       if (err instanceof S3Error && err.message === 'Not Found') {
@@ -175,6 +186,7 @@ export class S3BaseBucket {
     }
   }
 
+  // 将文件流转换为Buffer
   async fileStreamToBuffer(stream: Readable): Promise<Buffer> {
     const chunks: Buffer[] = [];
     for await (const chunk of stream) {
@@ -184,7 +196,7 @@ export class S3BaseBucket {
   }
 
   addDeleteJob(params: Omit<Parameters<typeof addS3DelJob>[0], 'bucketName'>) {
-    return addS3DelJob({ ...params, bucketName: this.name });
+    return addS3DelJob({ ...params, bucketName: this.bucketName });
   }
 
   async createPostPresignedUrl(
@@ -202,7 +214,7 @@ export class S3BaseBucket {
 
       const policy = this.externalClient.newPostPolicy();
       policy.setKey(key);
-      policy.setBucket(this.name);
+      policy.setBucket(this.bucketName);
       policy.setContentType(contentType);
       if (formatMaxFileSize) {
         policy.setContentLengthRange(1, formatMaxFileSize);
@@ -220,7 +232,7 @@ export class S3BaseBucket {
       if (expiredHours) {
         await MongoS3TTL.create({
           minioKey: key,
-          bucketName: this.name,
+          bucketName: this.bucketName,
           expiredTime: addHours(new Date(), expiredHours)
         });
       }
@@ -242,7 +254,7 @@ export class S3BaseBucket {
     const { key, expiredHours } = parsed;
     const expires = expiredHours ? expiredHours * 60 * 60 : 30 * 60; // expires 的单位是秒 默认 30 分钟
 
-    return await this.externalClient.presignedGetObject(this.name, key, expires);
+    return await this.externalClient.presignedGetObject(this.bucketName, key, expires);
   }
 
   async createPreviewUrl(params: createPreviewUrlParams) {
@@ -251,15 +263,15 @@ export class S3BaseBucket {
     const { key, expiredHours } = parsed;
     const expires = expiredHours ? expiredHours * 60 * 60 : 30 * 60; // expires 的单位是秒 默认 30 分钟
 
-    return await this.client.presignedGetObject(this.name, key, expires);
+    return await this.client.presignedGetObject(this.bucketName, key, expires);
   }
 
   async uploadFileByBuffer(params: UploadFileByBufferParams) {
     const { key, buffer, contentType } = UploadFileByBufferSchema.parse(params);
 
     await MongoS3TTL.create({
       minioKey: key,
-      bucketName: this.name,
+      bucketName: this.bucketName,
       expiredTime: addHours(new Date(), 1)
     });
     await this.putObject(key, buffer, undefined, {
@@ -274,4 +286,22 @@ export class S3BaseBucket {
       })
     };
   }
+
+  // 对外包装的方法
+  // 获取文件元数据
+  async getFileMetadata(key: string) {
+    const stat = await this.statObject(key);
+    if (!stat) return;
+
+    const contentLength = stat.size;
+    const filename: string = decodeURIComponent(stat.metaData['origin-filename']);
+    const extension = parseFileExtensionFromUrl(filename);
+    const contentType: string = stat.metaData['content-type'];
+    return {
+      filename,
+      extension,
+      contentType,
+      contentLength
+    };
+  }
 }

packages/service/common/s3/buckets/public.ts

@@ -7,7 +7,7 @@ export class S3PublicBucket extends S3BaseBucket {
     super(S3Buckets.public, {
       ...options,
       afterInit: async () => {
-        const bucket = this.name;
+        const bucket = this.bucketName;
         const policy = JSON.stringify({
           Version: '2012-10-17',
           Statement: [
@@ -34,7 +34,7 @@ export class S3PublicBucket extends S3BaseBucket {
     const protocol = this.options.useSSL ? 'https' : 'http';
     const hostname = this.options.endPoint;
     const port = this.options.port;
-    const bucket = this.name;
+    const bucket = this.bucketName;
 
     const url = new URL(`${protocol}://${hostname}:${port}/${bucket}/${objectKey}`);
 

packages/service/common/s3/sources/avatar.ts

@@ -5,11 +5,9 @@ import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants';
 import type { ClientSession } from 'mongoose';
 import { getFileS3Key } from '../utils';
 
-class S3AvatarSource {
-  private bucket: S3PublicBucket;
-
+class S3AvatarSource extends S3PublicBucket {
   constructor() {
-    this.bucket = new S3PublicBucket();
+    super();
   }
 
   get prefix(): string {
@@ -27,7 +25,7 @@ class S3AvatarSource {
   }) {
     const { fileKey } = getFileS3Key.avatar({ teamId, filename });
 
-    return this.bucket.createPostPresignedUrl(
+    return this.createPostPresignedUrl(
       { filename, rawKey: fileKey },
       {
         expiredHours: autoExpired ? 1 : undefined, // 1 Hours
@@ -36,19 +34,15 @@ class S3AvatarSource {
     );
   }
 
-  createPublicUrl(objectKey: string): string {
-    return this.bucket.createPublicUrl(objectKey);
-  }
-
   async removeAvatarTTL(avatar: string, session?: ClientSession): Promise<void> {
     const key = avatar.slice(this.prefix.length);
-    await MongoS3TTL.deleteOne({ minioKey: key, bucketName: this.bucket.name }, session);
+    await MongoS3TTL.deleteOne({ minioKey: key, bucketName: this.bucketName }, session);
   }
 
   async deleteAvatar(avatar: string, session?: ClientSession): Promise<void> {
     const key = avatar.slice(this.prefix.length);
-    await MongoS3TTL.deleteOne({ minioKey: key, bucketName: this.bucket.name }, session);
-    await this.bucket.delete(key);
+    await MongoS3TTL.deleteOne({ minioKey: key, bucketName: this.bucketName }, session);
+    await this.delete(key);
   }
 
   async refreshAvatar(newAvatar?: string, oldAvatar?: string, session?: ClientSession) {
@@ -78,7 +72,7 @@ class S3AvatarSource {
   }) {
     const from = key.slice(this.prefix.length);
     const to = `${S3Sources.avatar}/${teamId}/${filename}`;
-    await this.bucket.copy({ from, to, options: { temporary } });
+    await this.copy({ from, to, options: { temporary } });
     return this.prefix.concat(to);
   }
 }

packages/service/common/s3/sources/chat/index.ts

@@ -1,4 +1,3 @@
-import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
 import { S3PrivateBucket } from '../../buckets/private';
 import { S3Sources } from '../../type';
 import {
@@ -14,11 +13,9 @@ import { S3Buckets } from '../../constants';
 import path from 'path';
 import { getFileS3Key } from '../../utils';
 
-export class S3ChatSource {
-  private bucket: S3PrivateBucket;
-
+export class S3ChatSource extends S3PrivateBucket {
   constructor() {
-    this.bucket = new S3PrivateBucket();
+    super();
   }
 
   static parseChatUrl(url: string | URL) {
@@ -51,46 +48,19 @@ export class S3ChatSource {
     }
   }
 
-  // 获取文件流
-  getChatFileStream(key: string) {
-    return this.bucket.getObject(key);
-  }
-
-  // 获取文件状态
-  getChatFileStat(key: string) {
-    return this.bucket.statObject(key);
-  }
-
-  // 获取文件元数据
-  async getFileMetadata(key: string) {
-    const stat = await this.getChatFileStat(key);
-    if (!stat) return { filename: '', extension: '', contentLength: 0, contentType: '' };
-
-    const contentLength = stat.size;
-    const filename: string = decodeURIComponent(stat.metaData['origin-filename']);
-    const extension = parseFileExtensionFromUrl(filename);
-    const contentType: string = stat.metaData['content-type'];
-    return {
-      filename,
-      extension,
-      contentType,
-      contentLength
-    };
-  }
-
   async createGetChatFileURL(params: { key: string; expiredHours?: number; external: boolean }) {
     const { key, expiredHours = 1, external = false } = params; // 默认一个小时
 
     if (external) {
-      return await this.bucket.createExternalUrl({ key, expiredHours });
+      return await this.createExternalUrl({ key, expiredHours });
     }
-    return await this.bucket.createPreviewUrl({ key, expiredHours });
+    return await this.createPreviewUrl({ key, expiredHours });
   }
 
   async createUploadChatFileURL(params: CheckChatFileKeys) {
     const { appId, chatId, uId, filename, expiredTime } = ChatFileUploadSchema.parse(params);
     const { fileKey } = getFileS3Key.chat({ appId, chatId, uId, filename });
-    return await this.bucket.createPostPresignedUrl(
+    return await this.createPostPresignedUrl(
       { rawKey: fileKey, filename },
       { expiredHours: expiredTime ? differenceInHours(expiredTime, new Date()) : 24 }
     );
@@ -100,11 +70,11 @@ export class S3ChatSource {
     const { appId, chatId, uId } = DelChatFileByPrefixSchema.parse(params);
 
     const prefix = [S3Sources.chat, appId, uId, chatId].filter(Boolean).join('/');
-    return this.bucket.addDeleteJob({ prefix });
+    return this.addDeleteJob({ prefix });
   }
 
   deleteChatFileByKey(key: string) {
-    return this.bucket.addDeleteJob({ key });
+    return this.addDeleteJob({ key });
   }
 
   async uploadChatFileByBuffer(params: UploadFileParams) {
@@ -117,7 +87,7 @@ export class S3ChatSource {
       filename
     });
 
-    return this.bucket.uploadFileByBuffer({
+    return this.uploadFileByBuffer({
       key: fileKey,
       buffer,
       contentType