Merge pull request #1 from techiejd/multitables

Adds multiple tables ability for embedding

Author: techiejd

README.md

@@ -11,6 +11,7 @@ A Payload CMS plugin that adds vector search capabilities to your collections us
 - 🎯 **Flexible Chunking**: You provide the custom chunkers for different field types (text, rich text, etc.)
 - 🔧 **Configurable**: Choose which collections and fields to vectorize
 - 🌐 **REST API**: Built-in vector-search endpoint for querying vectorized content
+- 🏊 **Multiple Knowledge Pools**: Separate knowledge pools with independent configurations (dims, ivfflatLists, embedding functions)
 
 ## Prerequisites
 
@@ -65,12 +66,19 @@ const chunkRichText = async (richText: SerializedEditorState,
   return /* string array */
 }
 
-// Create the integration
+// Create the integration with static configs (dims, ivfflatLists)
 const { afterSchemaInitHook, payloadcmsVectorize } = createVectorizeIntegration({
   // Note limitation: Changing these values is currently not supported.
   // Migration is necessary.
-  dims: 1536, // Vector dimensions
-  ivfflatLists: 100, // IVFFLAT index parameter
+  main: {
+    dims: 1536, // Vector dimensions
+    ivfflatLists: 100, // IVFFLAT index parameter
+  },
+  // You can add more knowledge pools with different static configs
+  // products: {
+  //   dims: 384,
+  //   ivfflatLists: 50,
+  // },
 })
 
 export default buildConfig({
@@ -83,18 +91,37 @@ export default buildConfig({
   }),
   plugins: [
     payloadcmsVectorize({
-      // The collection-fields you want vectorized
-      collections: {
-        posts: {
-          fields: {
-            title: { chunker: chunkText },
-            content: { chunker: chunkRichText },
+      // Knowledge pools - dynamic configs (collections, embedding functions)
+      knowledgePools: {
+        main: {
+          // The collection-fields you want vectorized in this pool
+          collections: {
+            posts: {
+              fields: {
+                title: { chunker: chunkText },
+                content: { chunker: chunkRichText },
+              },
+            },
           },
+          embedDocs,
+          embedQuery,
+          embeddingVersion: 'v1.0.0',
         },
+        // You can add more knowledge pools with different dynamic configs
+        // products: {
+        //   collections: { ... },
+        //   embedDocs: differentEmbedDocs,
+        //   embedQuery: differentEmbedQuery,
+        //   embeddingVersion: 'v2.0.0',
+        // },
       },
-      embedDocs,
-      embedQuery,
-      embeddingVersion: 'v1.0.0',
+      // Optional plugin options:
+      // queueName: 'custom-queue',
+      // endpointOverrides: {
+      //   path: '/custom-vector-search',
+      //   enabled: true,
+      // },
+      // disabled: false,
     }),
   ],
 })
@@ -108,7 +135,10 @@ The plugin automatically creates a `/api/vector-search` endpoint:
 const response = await fetch('/api/vector-search', {
   method: 'POST',
   headers: { 'Content-Type': 'application/json' },
-  body: JSON.stringify({ query: 'What is machine learning?' }),
+  body: JSON.stringify({
+    query: 'What is machine learning?', // Required: query
+    knowledgePool: 'main', // Required: specify which knowledge pool to search
+  }),
 })
 
 const results = await response.json()
@@ -119,15 +149,30 @@ const results = await response.json()
 
 ### Plugin Options
 
-| Option              | Type                                        | Required | Description                               |
-| ------------------- | ------------------------------------------- | -------- | ----------------------------------------- |
-| `collections`       | `Record<string, CollectionVectorizeOption>` | ✅       | Collections and fields to vectorize       |
-| `embedDocs`         | `EmbedDocsFn`                               | ✅       | Function to embed multiple documents      |
-| `embedQuery`        | `EmbedQueryFn`                              | ✅       | Function to embed search queries          |
-| `embeddingVersion`  | `string`                                    | ✅       | Version string for tracking model changes |
-| `queueName`         | `string`                                    | ❌       | Custom queue name for background jobs     |
-| `endpointOverrides` | `object`                                    | ❌       | Customize the search endpoint             |
-| `disabled`          | `boolean`                                   | ❌       | Disable plugin while keeping schema       |
+| Option              | Type                                         | Required | Description                              |
+| ------------------- | -------------------------------------------- | -------- | ---------------------------------------- |
+| `knowledgePools`    | `Record<KnowledgePool, KnowledgePoolConfig>` | ✅       | Knowledge pools and their configurations |
+| `queueName`         | `string`                                     | ❌       | Custom queue name for background jobs    |
+| `endpointOverrides` | `object`                                     | ❌       | Customize the search endpoint            |
+| `disabled`          | `boolean`                                    | ❌       | Disable plugin while keeping schema      |
+
+### Knowledge Pool Config
+
+Knowledge pools are configured in two steps. The static configs define the database schema (migration required), while dynamic configs define runtime behavior (no migration required).
+
+**1. Static Config** (passed to `createVectorizeIntegration`):
+
+- `dims`: `number` - Vector dimensions for pgvector column
+- `ivfflatLists`: `number` - IVFFLAT index parameter
+
+The embeddings collection name will be the same as the knowledge pool name.
+
+**2. Dynamic Config** (passed to `payloadcmsVectorize`):
+
+- `collections`: `Record<string, CollectionVectorizeOption>` - Collections and fields to vectorize
+- `embedDocs`: `EmbedDocsFn` - Function to embed multiple documents
+- `embedQuery`: `EmbedQueryFn` - Function to embed search queries
+- `embeddingVersion`: `string` - Version string for tracking model changes
 
 ## Chunkers
 
@@ -187,10 +232,16 @@ Search for similar content using vector similarity.
 
 ```json
 {
-  "query": "Your search query"
+  "query": "Your search query",
+  "knowledgePool": "main"
 }
 ```
 
+**Parameters:**
+
+- `query` (required): The search query string
+- `knowledgePool` (required): The knowledge pool identifier to search in
+
 **Response:**
 
 ```json
@@ -210,6 +261,73 @@ Search for similar content using vector similarity.
 }
 ```
 
+## Migration from v0.1.0 to v0.2.0
+
+Version 0.2.0 introduces support for multiple knowledge pools. This is a **breaking change** that requires updating your configuration.
+
+### Before (v0.1.0):
+
+```typescript
+const { afterSchemaInitHook, payloadcmsVectorize } = createVectorizeIntegration({
+  dims: 1536,
+  ivfflatLists: 100,
+})
+
+payloadcmsVectorize({
+  collections: {
+    posts: { fields: { ... } },
+  },
+  embedDocs,
+  embedQuery,
+  embeddingVersion: 'v1.0.0',
+})
+```
+
+### After (v0.2.0):
+
+```typescript
+// Static configs (schema-related) passed to createVectorizeIntegration
+const { afterSchemaInitHook, payloadcmsVectorize } = createVectorizeIntegration({
+  main: {
+    dims: 1536,
+    ivfflatLists: 100,
+  },
+})
+
+// Dynamic configs (runtime behavior) passed to payloadcmsVectorize
+payloadcmsVectorize({
+  knowledgePools: {
+    main: {
+      collections: {
+        posts: { fields: { ... } },
+      },
+      embedDocs,
+      embedQuery,
+      embeddingVersion: 'v1.0.0',
+    },
+  },
+})
+```
+
+### API Changes
+
+The vector search endpoint now requires a `knowledgePool` parameter:
+
+```typescript
+// Before
+{ query: 'search term' }
+
+// After
+{ query: 'search term', knowledgePool: 'main' }
+```
+
+### Benefits of Multiple Knowledge Pools
+
+- **Separate knowledge domains**: Keep different types of content in separate pools
+- **Different technical requirements**: Each pool can have different `dims`, `ivfflatLists`, and embedding functions
+- **Flexible organization**: Collections can appear in multiple pools if needed
+- **Independent versioning**: Each pool can track its own embedding model version
+
 ## Requirements
 
 - Payload CMS ^3.37.0

dev/payload-types.ts

@@ -68,8 +68,7 @@ export interface Config {
   blocks: {};
   collections: {
     posts: Post;
-    media: Media;
-    embeddings: Embedding;
+    default: Default;
     users: User;
     'payload-jobs': PayloadJob;
     'payload-locked-documents': PayloadLockedDocument;
@@ -79,8 +78,7 @@ export interface Config {
   collectionsJoins: {};
   collectionsSelect: {
     posts: PostsSelect<false> | PostsSelect<true>;
-    media: MediaSelect<false> | MediaSelect<true>;
-    embeddings: EmbeddingsSelect<false> | EmbeddingsSelect<true>;
+    default: DefaultSelect<false> | DefaultSelect<true>;
     users: UsersSelect<false> | UsersSelect<true>;
     'payload-jobs': PayloadJobsSelect<false> | PayloadJobsSelect<true>;
     'payload-locked-documents': PayloadLockedDocumentsSelect<false> | PayloadLockedDocumentsSelect<true>;
@@ -150,31 +148,13 @@ export interface Post {
   updatedAt: string;
   createdAt: string;
 }
-/**
- * This interface was referenced by `Config`'s JSON-Schema
- * via the `definition` "media".
- */
-export interface Media {
-  id: number;
-  updatedAt: string;
-  createdAt: string;
-  url?: string | null;
-  thumbnailURL?: string | null;
-  filename?: string | null;
-  mimeType?: string | null;
-  filesize?: number | null;
-  width?: number | null;
-  height?: number | null;
-  focalX?: number | null;
-  focalY?: number | null;
-}
 /**
  * Vector embeddings for search and similarity queries. Created by the payloadcms-vectorize plugin. Embeddings cannot be added or modified, only deleted, through the admin panel. No other restrictions enforced.
  *
  * This interface was referenced by `Config`'s JSON-Schema
- * via the `definition` "embeddings".
+ * via the `definition` "default".
  */
-export interface Embedding {
+export interface Default {
   id: number;
   /**
    * The collection that this embedding belongs to
@@ -324,12 +304,8 @@ export interface PayloadLockedDocument {
         value: number | Post;
       } | null)
     | ({
-        relationTo: 'media';
-        value: number | Media;
-      } | null)
-    | ({
-        relationTo: 'embeddings';
-        value: number | Embedding;
+        relationTo: 'default';
+        value: number | Default;
       } | null)
     | ({
         relationTo: 'users';
@@ -393,26 +369,9 @@ export interface PostsSelect<T extends boolean = true> {
 }
 /**
  * This interface was referenced by `Config`'s JSON-Schema
- * via the `definition` "media_select".
- */
-export interface MediaSelect<T extends boolean = true> {
-  updatedAt?: T;
-  createdAt?: T;
-  url?: T;
-  thumbnailURL?: T;
-  filename?: T;
-  mimeType?: T;
-  filesize?: T;
-  width?: T;
-  height?: T;
-  focalX?: T;
-  focalY?: T;
-}
-/**
- * This interface was referenced by `Config`'s JSON-Schema
- * via the `definition` "embeddings_select".
+ * via the `definition` "default_select".
  */
-export interface EmbeddingsSelect<T extends boolean = true> {
+export interface DefaultSelect<T extends boolean = true> {
   sourceCollection?: T;
   docId?: T;
   fieldPath?: T;

dev/payload.config.ts

@@ -2,12 +2,11 @@ import { postgresAdapter } from '@payloadcms/db-postgres'
 import { lexicalEditor } from '@payloadcms/richtext-lexical'
 import path from 'path'
 import { buildConfig } from 'payload'
-import { createVectorizeIntegration, StaticIntegrationConfig } from 'payloadcms-vectorize'
+import { createVectorizeIntegration } from 'payloadcms-vectorize'
 import {
   makeDummyEmbedDocs,
   testEmbeddingVersion,
   voyageEmbedDocs,
-  voyageEmbedDims,
   voyageEmbedQuery,
   makeDummyEmbedQuery,
 } from './helpers/embed.js'
@@ -43,12 +42,12 @@ const ssl =
       }
     : undefined
 
-const integrationConfig: StaticIntegrationConfig = {
-  dims,
-  ivfflatLists, // Rule of thumb: ivfflatLists = sqrt(total_number_of_vectors). Helps with working memory usage.
-}
-
-const { afterSchemaInitHook, payloadcmsVectorize } = createVectorizeIntegration(integrationConfig)
+const { afterSchemaInitHook, payloadcmsVectorize } = createVectorizeIntegration({
+  default: {
+    dims,
+    ivfflatLists, // Rule of thumb: ivfflatLists = sqrt(total_number_of_vectors). Helps with working memory usage.
+  },
+})
 
 const buildConfigWithPostgres = async () => {
   return buildConfig({
@@ -100,17 +99,21 @@ const buildConfigWithPostgres = async () => {
     },
     plugins: [
       payloadcmsVectorize({
-        collections: {
-          posts: {
-            fields: {
-              title: { chunker: chunkText },
-              content: { chunker: chunkRichText },
+        knowledgePools: {
+          default: {
+            collections: {
+              posts: {
+                fields: {
+                  title: { chunker: chunkText },
+                  content: { chunker: chunkRichText },
+                },
+              },
             },
+            embedDocs,
+            embedQuery,
+            embeddingVersion: testEmbeddingVersion,
           },
         },
-        embedDocs,
-        embedQuery,
-        embeddingVersion: testEmbeddingVersion,
       }),
     ],
     secret: process.env.PAYLOAD_SECRET || 'test-secret_key',