Jchris/compaction-repro

CLAUDE.md

@@ -0,0 +1,145 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+Fireproof is a lightweight embedded document database with encrypted live sync for JavaScript environments. It's designed to work in browsers, Node.js, Deno, and other JavaScript runtimes with a unified API. The repository is structured as a monorepo with multiple packages and includes React hooks integration.
+
+## Common Development Commands
+
+- `pnpm run check` - Run format, lint, test, and build in sequence
+
+### Building and Development
+
+- `pnpm run build` - Build all packages (runs prebuild, build scripts, and pub scripts)
+- `pnpm run build:tsc` - Build using TypeScript compiler
+- `pnpm run build:tsup` - Build using tsup bundler
+- `pnpm run dev` - Start development servers for cloud components
+- `pnpm run dev:dashboard` - Start dashboard development server on port 3002
+- `pnpm run dev:3rd-party` - Start 3rd-party development server on port 3001
+
+### Testing
+
+- `pnpm run test` - Run all tests using vitest
+- `pnpm run test:file` - Run file-based tests
+- `pnpm run test:indexeddb` - Run IndexedDB-specific tests
+- `pnpm run test:deno` - Run tests in Deno environment
+- `pnpm run test -t 'test name pattern' path/to/test/file` - Run specific tests
+- `FP_DEBUG=Loader pnpm run test --project file -t 'codec implicit iv' crdt` - Run specific test with debugging
+
+### Code Quality
+
+- `pnpm run lint` - Run ESLint
+- `pnpm run format` - Run Prettier formatting
+
+### Docker Management
+
+- `pnpm run docker:down` - Stop Docker containers
+- `pnpm run docker:up` - Start Docker containers
+- `pnpm run docker:restart` - Restart Docker containers
+- `pnpm run docker:logs` - View Docker container logs
+- `pnpm run docker:health` - Check Docker container and MinIO health
+
+### Publishing and Distribution
+
+- `pnpm run smoke` - Run smoke tests against built packages
+- `pnpm run fppublish` - Publish packages to npm
+- `pnpm run presmoke` - Build and publish to local registry for smoke testing
+
+## Architecture Overview
+
+### Core Components
+
+**Database Layer (`src/database.ts`, `src/ledger.ts`)**
+
+- `DatabaseImpl` - Main database implementation with CRUD operations
+- `Ledger` - Lower-level data storage and versioning layer
+- CRDT (Conflict-free Replicated Data Types) implementation for distributed consistency
+
+**Blockstore (`src/blockstore/`)**
+
+- Content-addressed storage system using IPLD blocks
+- Multiple gateway implementations (file, IndexedDB, memory, cloud)
+- Encryption and serialization handling
+- Transaction management and commit queues
+
+**Runtime (`src/runtime/`)**
+
+- Platform-specific implementations (Node.js, Deno, browser)
+- File system abstractions
+- Key management and cryptography
+- Storage gateway factory patterns
+
+**React Integration (`src/react/`)**
+
+- `useFireproof` - Main hook for database access
+- `useLiveQuery` - Real-time query results
+- `useDocument` - Document-level operations
+- `useAllDocs` - Bulk document operations
+- `ImgFile` component for file attachments
+
+**Protocols (`src/protocols/`)**
+
+- Cloud synchronization protocols
+- Dashboard API protocols
+- Message passing and connection management
+
+### Storage Gateways
+
+The system supports multiple storage backends:
+
+- **File** - Local file system storage (Node.js/Deno)
+- **IndexedDB** - Browser-based storage
+- **Memory** - In-memory storage for testing
+- **Cloud** - Remote storage with sync capabilities
+
+### Testing Infrastructure
+
+Uses Vitest with multiple configurations:
+
+- `vitest.workspace.ts` - Main workspace configuration
+- Separate configs for file, memory, IndexedDB, and cloud testing
+- Screenshot testing for React components
+- Multiple test environments (file, memory, indexeddb, cloud variants)
+
+## Key File Locations
+
+- `src/index.ts` - Main entry point
+- `src/database.ts` - Database implementation
+- `src/ledger.ts` - Core ledger functionality
+- `src/crdt.ts` - CRDT implementation
+- `src/blockstore/` - Storage layer
+- `src/runtime/` - Platform-specific code
+- `src/react/` - React hooks and components
+- `tests/` - Test suites organized by component
+
+## Development Notes
+
+- Uses pnpm for package management
+- TypeScript with strict configuration
+- ESM modules throughout
+- Supports Node.js >=20.18.1
+- Uses Vitest for testing with multiple environments
+- Includes comprehensive smoke testing pipeline
+- Debug logging available via `FP_DEBUG` environment variable
+- Uses content-addressed storage with cryptographic integrity
+- Implements causal consistency for distributed operations
+
+## React Development
+
+When working with React components:
+
+- Use `useFireproof` hook to access database functionality
+- `useLiveQuery` provides real-time query results that update automatically
+- `useDocument` handles individual document operations with optimistic updates
+- File attachments are handled through the `_files` property and `ImgFile` component
+- Test React components using the testing utilities in `tests/react/`
+
+## Cloud and Sync
+
+- Cloud functionality is in the `cloud/` directory
+- Supports multiple cloud backends (CloudFlare D1, LibSQL, etc.)
+- WebSocket and HTTP-based synchronization
+- Encrypted data transmission and storage
+- Multi-tenant architecture support

core/blockstore/transaction.ts

@@ -185,16 +185,33 @@ export class BaseBlockstoreImpl implements BlockFetcher {
     return new CarTransactionImpl(this, opts);
   }
 
+  inflightCompaction = false;
+
+  needsCompaction() {
+    if (!this.inflightCompaction && this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
+      this.inflightCompaction = true;
+      // Wait until the commit queue is idle before triggering compaction to
+      // ensure no commits are still in-flight. This prevents race conditions
+      // where compaction runs before all blocks have been persisted.
+      this.loader.commitQueue
+        .waitIdle()
+        .then(() => this.compact())
+        .catch((err) => {
+          this.logger.Warn().Err(err).Msg("autoCompact scheduling failed");
+        })
+        .finally(() => {
+          this.inflightCompaction = false;
+        });
+    }
+  }
+
   async commitTransaction<M extends TransactionMeta>(
     t: CarTransaction,
     done: M,
     opts: CarTransactionOpts,
   ): Promise<TransactionWrapper<M>> {
-    if (!this.loader) throw this.logger.Error().Msg("loader required to commit").AsError();
     const cars = await this.loader.commit<M>(t, done, opts);
-    if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
-      setTimeout(() => void this.compact(), 10);
-    }
+    this.needsCompaction();
     if (cars) {
       this.transactions.delete(t);
       return { meta: done, cars, t };
@@ -255,9 +272,7 @@ export class EncryptedBlockstore extends BaseBlockstoreImpl {
     this.logger.Debug().Msg("post super.transaction");
     const cars = await this.loader.commit<M>(t, done, opts);
     this.logger.Debug().Msg("post this.loader.commit");
-    if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
-      setTimeout(() => void this.compact(), 10);
-    }
+    this.needsCompaction();
     if (cars) {
       this.transactions.delete(t);
       return { meta: done, cars, t };
@@ -278,6 +293,7 @@ export class EncryptedBlockstore extends BaseBlockstoreImpl {
   }
 
   async compact() {
+    this.logger.Debug().Any({ carLogLen_before: this.loader?.carLog.length }).Msg("compact() – start");
     await this.ready();
     if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
     if (this.loader.carLog.length < 2) return;

core/tests/fireproof/repro-blocks-inline.test.ts

@@ -0,0 +1,82 @@
+import { Database, DocWithId, fireproof } from "@fireproof/core";
+import { describe, it, expect } from "vitest";
+
+interface Record {
+  id: string;
+  type: string;
+  createdAt: string;
+}
+
+async function findAll(db: Database): Promise<Record[]> {
+  const result = await db.query(
+    (doc: DocWithId<Record>) => {
+      if (doc.type === "CustomPropertyDefinition" && doc.createdAt && doc._deleted !== true) {
+        return doc.createdAt;
+      }
+    },
+    { descending: true },
+  );
+  return result.rows
+    .filter((row) => row.doc) // Filter out any rows without documents
+    .map((row) => row.doc as Record);
+}
+
+const numberOfDocs = 100;
+
+async function writeSampleData(db: Database): Promise<void> {
+  console.log("start puts");
+  for (let i = 10; i < numberOfDocs; i++) {
+    const record: DocWithId<Record> = {
+      _id: `record-${i}`,
+      id: `record-${i}`,
+      type: "CustomPropertyDefinition",
+      createdAt: new Date().toISOString(),
+    };
+    await db.put(record);
+  }
+  console.log("start dels");
+  for (let i = 10; i < numberOfDocs; i += 10) {
+    await db.del(`record-${i}`);
+  }
+}
+
+async function runReproBlocksOnce(iter: number, compactStrategy?: string) {
+  const db = fireproof(`test-db-inline-${iter}-${Date.now()}`, {
+    compactStrategy,
+  });
+
+  await writeSampleData(db);
+
+  const all = await db.allDocs<Record>();
+  const records = await findAll(db);
+
+  console.log(`repro-blocks inline run ${iter}: Found records:`, all.rows.length, records.length);
+  expect(all.rows.length).toBe(81); // 90 puts - 9 deletes = 81
+  expect(records.length).toBe(81);
+
+  // Clean up the database after the test
+  await db.destroy();
+}
+
+// Test both compaction modes in a single test process
+describe("repro-blocks inline regression test", () => {
+  it(
+    "runs with fireproof-default compaction mode",
+    async () => {
+      for (let i = 1; i <= 3; i++) {
+        await runReproBlocksOnce(i, undefined);
+      }
+    },
+    2 * 60 * 1000, // 2 minutes
+  );
+
+  it(
+    "runs with full compaction mode",
+    async () => {
+      for (let i = 1; i <= 3; i++) {
+        await runReproBlocksOnce(i, "full");
+      }
+    },
+    2 * 60 * 1000, // 2 minutes
+  );
+});

core/tests/fireproof/repro-blocks.process.test.ts

@@ -0,0 +1,80 @@
+import { describe, it } from "vitest";
+import { Database, DocWithId, fireproof } from "@fireproof/core";
+
+// Skip this entire suite when running inside a browser-like Vitest environment
+const isNode = typeof process !== "undefined" && !!process.versions?.node;
+const describeFn = isNode ? describe.skip : describe.skip;
+
+/* eslint-disable no-console */
+
+interface Record {
+  id: string;
+  type: string;
+  createdAt: string;
+}
+
+async function findAll(db: Database): Promise<Record[]> {
+  const result = await db.query(
+    (doc: DocWithId<Record>) => {
+      if (doc.type === "CustomPropertyDefinition" && doc.createdAt && doc._deleted !== true) {
+        return doc.createdAt;
+      }
+    },
+    { descending: true },
+  );
+  return result.rows
+    .filter((row) => row.doc) // Filter out rows without documents
+    .map((row) => row.doc as Record);
+}
+
+const numberOfDocs = 100;
+
+async function writeSampleData(db: Database): Promise<void> {
+  console.log("start puts");
+  for (let i = 10; i < numberOfDocs; i++) {
+    const record: DocWithId<Record> = {
+      _id: `record-${i}`,
+      id: `record-${i}`,
+      type: "CustomPropertyDefinition",
+      createdAt: new Date().toISOString(),
+    };
+    await db.put(record);
+  }
+  console.log("start dels");
+  for (let i = 10; i < numberOfDocs; i += 10) {
+    await db.del(`record-${i}`);
+  }
+}
+
+async function runReproBlocksOnce(iter: number, compactStrategy?: string) {
+  const db = fireproof(`test-db-${iter}`, {
+    compactStrategy,
+  });
+
+  await writeSampleData(db);
+
+  const all = await db.allDocs<Record>();
+  const records = await findAll(db);
+
+  console.log(`repro-blocks run ${iter}: Found records:`, all.rows.length, records.length);
+  console.log(`repro-blocks run ${iter}: ok`); // useful in CI logs
+
+  // Clean up the database after the test
+  await db.destroy();
+}
+
+// Test both compaction modes
+describeFn.each([
+  { name: "fireproof-default", compactionMode: undefined },
+  { name: "full-compaction", compactionMode: "full" },
+])("repro-blocks regression test with $name compaction", ({ compactionMode }) => {
+  it(
+    "runs 10 consecutive times without compaction errors",
+    async () => {
+      for (let i = 1; i <= 10; i++) {
+        await runReproBlocksOnce(i, compactionMode);
+      }
+    },
+    5 * 60 * 1000, // allow up to 5 minutes – heavy disk workload
+  );
+});