From 0759b0369da200a6d6ca93711663fcb31963f4a4 Mon Sep 17 00:00:00 2001 From: Ada Date: Mon, 29 Dec 2025 08:20:32 -0500 Subject: [PATCH 1/2] =?UTF-8?q?DATA=20[SCMS]=20Add=20idempotent=20lab=5Fno?= =?UTF-8?q?tes=20schema=20migration=20with=20logging=20=F0=9F=A7=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Introduce migration helper to safely evolve lab_notes schema - Preserve existing data while adding translation-ready fields - Keep Lab Note attributes (category, excerpt, etc.) - Add lightweight migration logging (added columns only) - Make bootstrapDb thin and declarative co-authored-by: Lyric co-authored-by: Carmel --- src/db.ts | 165 ++++++++++++++++++++++++++------- src/db/migrateLabNotes.ts | 187 ++++++++++++++++++++++++++++++++++++++ src/seed/devSeed.ts | 172 +++++++++++++++++++++++++++-------- 3 files changed, 450 insertions(+), 74 deletions(-) create mode 100644 src/db/migrateLabNotes.ts diff --git a/src/db.ts b/src/db.ts index 6808599..e476c1b 100644 --- a/src/db.ts +++ b/src/db.ts @@ -3,6 +3,7 @@ import Database from "better-sqlite3"; import path from "path"; import { fileURLToPath } from "url"; import { env } from "./env.js"; +import { migrateLabNotesSchema } from "./db/migrateLabNotes.js"; export function resolveDbPath(): string { const __filename = fileURLToPath(import.meta.url); @@ -35,63 +36,158 @@ export function openDb(dbPath: string) { } export function bootstrapDb(db: Database.Database) { + const log = process.env.DB_MIGRATE_VERBOSE === "1" ? console.log : undefined; + migrateLabNotesSchema(db, log); db.exec(` CREATE TABLE IF NOT EXISTS lab_notes ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL, - slug TEXT UNIQUE NOT NULL, - category TEXT, - excerpt TEXT, - content_html TEXT, - content_md TEXT, - department_id TEXT DEFAULT 'SCMS', - shadow_density INTEGER DEFAULT 0, - coherence_score REAL DEFAULT 1.0, - safer_landing BOOLEAN DEFAULT 0, - read_time_minutes INTEGER, - published_at TEXT, - created_at TEXT, - updated_at TEXT - ); + id TEXT PRIMARY KEY, -- uuid per row + group_id TEXT NOT NULL, -- uuid shared across translations + slug TEXT NOT NULL, + locale TEXT NOT NULL DEFAULT 'en', - CREATE TABLE IF NOT EXISTS lab_note_tags ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - note_id TEXT NOT NULL, - tag TEXT NOT NULL, - UNIQUE(note_id, tag), - FOREIGN KEY (note_id) REFERENCES lab_notes(id) ON DELETE CASCADE + type TEXT NOT NULL DEFAULT 'labnote', -- labnote|paper|memo + title TEXT NOT NULL, + + -- MVP core (kept) + category TEXT, + excerpt TEXT, + department_id TEXT, + shadow_density REAL, + safer_landing INTEGER, -- 0/1 + read_time_minutes INTEGER, + + coherence_score REAL, + subtitle TEXT, + summary TEXT, + + tags_json TEXT, -- optional JSON array string + dept TEXT, -- optional convenience label + + status TEXT NOT NULL DEFAULT 'draft', -- draft|published|archived + published_at TEXT, + + author TEXT, + ai_author TEXT, + + -- Translation metadata + source_locale TEXT, + translation_status TEXT NOT NULL DEFAULT 'original', -- original|machine|human|needs_review + translation_provider TEXT, + translation_version INTEGER NOT NULL DEFAULT 1, + source_updated_at TEXT, + translation_meta_json TEXT, + + -- Canonical markdown + content_md TEXT NOT NULL, + content_html TEXT, + + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + + UNIQUE (group_id, locale), + UNIQUE (slug, locale) ); + CREATE INDEX IF NOT EXISTS idx_lab_notes_locale ON lab_notes(locale); + CREATE INDEX IF NOT EXISTS idx_lab_notes_status ON lab_notes(status); + CREATE INDEX IF NOT EXISTS idx_lab_notes_published_at ON lab_notes(published_at); + CREATE INDEX IF NOT EXISTS idx_lab_notes_group_id ON lab_notes(group_id); + CREATE INDEX IF NOT EXISTS idx_lab_notes_department_id ON lab_notes(department_id); + DROP VIEW IF EXISTS v_lab_notes; - CREATE VIEW v_lab_notes AS - SELECT * FROM lab_notes; - `); -} -export function seedMarkerNote(db: Database.Database) { - db.prepare(` - INSERT OR IGNORE INTO lab_notes ( + CREATE VIEW v_lab_notes AS + SELECT id, - title, + group_id, slug, + locale, + type, + title, + category, excerpt, department_id, + shadow_density, + safer_landing, + read_time_minutes, + coherence_score, + subtitle, + summary, + tags_json, + dept, + + status, published_at, + author, + ai_author, + + source_locale, + translation_status, + translation_provider, + translation_version, + source_updated_at, + translation_meta_json, + + content_md, + content_html, + created_at, updated_at + FROM lab_notes; + `); + + // Optional: keep the tag table if anything still uses it + db.exec(` + CREATE TABLE IF NOT EXISTS lab_note_tags ( + note_id TEXT NOT NULL, + tag TEXT NOT NULL, + UNIQUE(note_id, tag) + ); + CREATE INDEX IF NOT EXISTS idx_lab_note_tags_note_id ON lab_note_tags(note_id); + CREATE INDEX IF NOT EXISTS idx_lab_note_tags_tag ON lab_note_tags(tag); + `); +} + +export function seedMarkerNote(db: Database.Database) { + const now = new Date(); + const nowIso = now.toISOString(); + + db.prepare(` + INSERT OR IGNORE INTO lab_notes ( + id, group_id, title, slug, locale, type, + category, excerpt, department_id, + shadow_density, safer_landing, read_time_minutes, coherence_score, + status, published_at, + author, ai_author, + translation_status, + content_md, content_html, + created_at, updated_at ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( + "api-marker", "api-marker", "API Marker Note", "api-marker-note", + "en", + "memo", "Debug", "If you can see this in WebStorm, we are looking at the same DB.", "SCMS", - new Date().toISOString().slice(0, 10), - new Date().toISOString(), - new Date().toISOString() + 0.0, + 1, + 1, + 1.0, + "draft", + nowIso.slice(0, 10), + "Ada", + "Lyric", + "original", + "If you can see this in WebStorm, we are looking at the same DB.", + null, + nowIso, + nowIso ); } @@ -99,3 +195,4 @@ export function isDbEmpty(db: Database.Database): boolean { const row = db.prepare(`SELECT COUNT(*) as count FROM lab_notes`).get() as { count: number }; return row.count === 0; } + diff --git a/src/db/migrateLabNotes.ts b/src/db/migrateLabNotes.ts new file mode 100644 index 0000000..d9c3866 --- /dev/null +++ b/src/db/migrateLabNotes.ts @@ -0,0 +1,187 @@ +// src/db/migrateLabNotes.ts +import Database from "better-sqlite3"; + +type ColDef = { name: string; ddl: string }; +type MigrationLogFn = (msg: string) => void; + +export type MigrationResult = { + addedColumns: string[]; + createdFreshTable: boolean; +}; + +const LAB_NOTES_REQUIRED_COLS: ColDef[] = [ + // Core identity + { name: "group_id", ddl: "TEXT NOT NULL DEFAULT ''" }, + { name: "slug", ddl: "TEXT NOT NULL DEFAULT ''" }, + { name: "locale", ddl: "TEXT NOT NULL DEFAULT 'en'" }, + + // Basics + { name: "type", ddl: "TEXT NOT NULL DEFAULT 'labnote'" }, + { name: "title", ddl: "TEXT NOT NULL DEFAULT ''" }, + + // Kept attributes + { name: "category", ddl: "TEXT" }, + { name: "excerpt", ddl: "TEXT" }, + { name: "department_id", ddl: "TEXT" }, + { name: "shadow_density", ddl: "REAL" }, + { name: "safer_landing", ddl: "INTEGER" }, + { name: "read_time_minutes", ddl: "INTEGER" }, + + { name: "coherence_score", ddl: "REAL" }, + { name: "subtitle", ddl: "TEXT" }, + { name: "summary", ddl: "TEXT" }, + + { name: "tags_json", ddl: "TEXT" }, + { name: "dept", ddl: "TEXT" }, + + // Publishing + { name: "status", ddl: "TEXT NOT NULL DEFAULT 'draft'" }, + { name: "published_at", ddl: "TEXT" }, + + // Authors + { name: "author", ddl: "TEXT" }, + { name: "ai_author", ddl: "TEXT" }, + + // Translation metadata + { name: "source_locale", ddl: "TEXT" }, + { name: "translation_status", ddl: "TEXT NOT NULL DEFAULT 'original'" }, + { name: "translation_provider", ddl: "TEXT" }, + { name: "translation_version", ddl: "INTEGER NOT NULL DEFAULT 1" }, + { name: "source_updated_at", ddl: "TEXT" }, + { name: "translation_meta_json", ddl: "TEXT" }, + + // Content + { name: "content_md", ddl: "TEXT NOT NULL DEFAULT ''" }, + { name: "content_html", ddl: "TEXT" }, + + // Timestamps + { name: "created_at", ddl: "TEXT NOT NULL DEFAULT ''" }, + { name: "updated_at", ddl: "TEXT NOT NULL DEFAULT ''" }, +]; + +export function migrateLabNotesSchema( + db: Database.Database, + log?: MigrationLogFn +): MigrationResult { + // Did the table exist before? + const hadLabNotesTable = !!db + .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='lab_notes'`) + .get(); + + // Ensure table exists (minimal seed) + db.exec(` + CREATE TABLE IF NOT EXISTS lab_notes ( + id TEXT PRIMARY KEY + ); + `); + + const existingCols = new Set( + db.prepare(`PRAGMA table_info(lab_notes)`).all().map((r: any) => r.name) + ); + + const addedColumns: string[] = []; + + for (const col of LAB_NOTES_REQUIRED_COLS) { + if (!existingCols.has(col.name)) { + db.exec(`ALTER TABLE lab_notes ADD COLUMN ${col.name} ${col.ddl};`); + addedColumns.push(col.name); + } + } + + // Backfills for legacy rows + db.exec(` + UPDATE lab_notes + SET group_id = id + WHERE group_id IS NULL OR group_id = ''; + + UPDATE lab_notes SET slug = id WHERE slug IS NULL OR slug = ''; + UPDATE lab_notes SET title = slug WHERE title IS NULL OR title = ''; + UPDATE lab_notes SET content_md = '' WHERE content_md IS NULL; + + UPDATE lab_notes + SET created_at = COALESCE(created_at, updated_at, datetime('now')) + WHERE created_at IS NULL OR created_at = ''; + + UPDATE lab_notes + SET updated_at = COALESCE(updated_at, created_at, datetime('now')) + WHERE updated_at IS NULL OR updated_at = ''; + `); + + // Indexes + view (idempotent) + db.exec(` + CREATE INDEX IF NOT EXISTS idx_lab_notes_locale ON lab_notes(locale); + CREATE INDEX IF NOT EXISTS idx_lab_notes_status ON lab_notes(status); + CREATE INDEX IF NOT EXISTS idx_lab_notes_published_at ON lab_notes(published_at); + CREATE INDEX IF NOT EXISTS idx_lab_notes_group_id ON lab_notes(group_id); + CREATE INDEX IF NOT EXISTS idx_lab_notes_department_id ON lab_notes(department_id); + + DROP VIEW IF EXISTS v_lab_notes; + + CREATE VIEW v_lab_notes AS + SELECT + id, + group_id, + slug, + locale, + type, + title, + + category, + excerpt, + department_id, + shadow_density, + safer_landing, + read_time_minutes, + coherence_score, + subtitle, + summary, + tags_json, + dept, + + status, + published_at, + author, + ai_author, + + source_locale, + translation_status, + translation_provider, + translation_version, + source_updated_at, + translation_meta_json, + + content_md, + content_html, + + created_at, + updated_at + FROM lab_notes; + `); + + // Optional tag table + db.exec(` + CREATE TABLE IF NOT EXISTS lab_note_tags ( + note_id TEXT NOT NULL, + tag TEXT NOT NULL, + UNIQUE(note_id, tag) + ); + CREATE INDEX IF NOT EXISTS idx_lab_note_tags_note_id ON lab_note_tags(note_id); + CREATE INDEX IF NOT EXISTS idx_lab_note_tags_tag ON lab_note_tags(tag); + `); + + const result: MigrationResult = { + addedColumns, + createdFreshTable: !hadLabNotesTable, + }; + + if (log && (result.createdFreshTable || result.addedColumns.length > 0)) { + const colsPart = + result.addedColumns.length > 0 + ? `added ${result.addedColumns.length} column(s): ${result.addedColumns.join(", ")}` + : "no new columns"; + const tablePart = result.createdFreshTable ? "created lab_notes table" : "updated lab_notes table"; + log(`[db] lab_notes migration: ${tablePart}; ${colsPart}`); + } + + return result; +} diff --git a/src/seed/devSeed.ts b/src/seed/devSeed.ts index 3a4c027..083f994 100644 --- a/src/seed/devSeed.ts +++ b/src/seed/devSeed.ts @@ -2,69 +2,161 @@ import Database from "better-sqlite3"; type SeedNote = { - id: string; + id: string; // row id (uuid) + group_id?: string; // shared id for translated siblings (optional; will default to id) title: string; slug: string; - category: string; - excerpt: string; - department_id: string; - shadow_density: number; - safer_landing: boolean; - read_time_minutes: number; - published_at: string; - coherence_score?: number; + + locale?: string; // default 'en' + type?: "labnote" | "paper" | "memo"; + dept?: string; // optional label (e.g. "SCMS") + + status?: "draft" | "published" | "archived"; + published_at?: string | null; + + author?: string | null; + ai_author?: string | null; + + // ✅ Keep these + category?: string | null; + excerpt?: string | null; + department_id?: string | null; + shadow_density?: number | null; + safer_landing?: boolean; + read_time_minutes?: number | null; + coherence_score?: number | null; + + // Translation metadata (optional) + source_locale?: string | null; + translation_status?: "original" | "machine" | "human" | "needs_review"; + translation_provider?: string | null; + translation_version?: number; + source_updated_at?: string | null; + translation_meta_json?: string | null; + + subtitle?: string | null; + summary?: string | null; + + // You can keep this if other parts of the app use it, but DB canonical is read_time_minutes + reading_time?: number | null; + + tags?: string[]; content_html?: string | null; - content_md?: string | null; - tags: string[]; + + // ✅ Canonical markdown field for DB + content_md: string; }; const notes: SeedNote[] = [ - // paste your seed notes here (same as scripts/seed.ts) + // paste your seed notes here ]; export function seedDevDb(db: Database.Database) { const insertNote = db.prepare(` - INSERT OR IGNORE INTO lab_notes ( - id, title, slug, category, excerpt, - content_html, content_md, - department_id, shadow_density, coherence_score, - safer_landing, read_time_minutes, - published_at, created_at, updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `); - - const insertTag = db.prepare(` - INSERT OR IGNORE INTO lab_note_tags (note_id, tag) - VALUES (?, ?) - `); + INSERT OR IGNORE INTO lab_notes ( + id, + group_id, + slug, + locale, + type, + title, + + category, + excerpt, + department_id, + shadow_density, + safer_landing, + read_time_minutes, + coherence_score, + + subtitle, + summary, + tags_json, + dept, + status, + published_at, + author, + ai_author, + + source_locale, + translation_status, + translation_provider, + translation_version, + source_updated_at, + translation_meta_json, + + content_md, + content_html, + + created_at, + updated_at + ) + VALUES ( + ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, + ?, ?, + ?, ? + ) + `); const nowIso = () => new Date().toISOString(); const tx = db.transaction(() => { for (const note of notes) { const ts = nowIso(); + + const locale = note.locale ?? "en"; + const type = note.type ?? "labnote"; + const dept = note.dept ?? "SCMS"; + const status = note.status ?? "draft"; + const groupId = note.group_id ?? note.id; + + const tagsJson = JSON.stringify(note.tags ?? []); + + const readTimeMinutes = + note.read_time_minutes ?? + (note.reading_time != null ? Math.max(1, Math.round(note.reading_time)) : null); + insertNote.run( note.id, - note.title, + groupId, note.slug, - note.category, - note.excerpt, - note.content_html ?? null, - note.content_md ?? null, - note.department_id, - note.shadow_density, - note.coherence_score ?? 1.0, + locale, + type, + note.title, + + note.category ?? null, + note.excerpt ?? null, + note.department_id ?? "SCMS", + note.shadow_density ?? null, note.safer_landing ? 1 : 0, - note.read_time_minutes, - note.published_at, + readTimeMinutes, + note.coherence_score ?? 1.0, + + note.subtitle ?? null, + note.summary ?? null, + tagsJson, + dept, + status, + note.published_at ?? null, + note.author ?? "Ada", + note.ai_author ?? "Lyric", + + note.source_locale ?? null, + note.translation_status ?? "original", + note.translation_provider ?? null, + note.translation_version ?? 1, + note.source_updated_at ?? null, + note.translation_meta_json ?? null, + + note.content_md, + note.content_html ?? null, + ts, ts ); - - for (const tag of note.tags) { - insertTag.run(note.id, tag); - } } }); From 71ada5ccb3bae8db2bc0f9379ff88c0fe218ceb7 Mon Sep 17 00:00:00 2001 From: Ada Date: Mon, 29 Dec 2025 08:20:32 -0500 Subject: [PATCH 2/2] =?UTF-8?q?DATA=20[SCMS]=20Add=20idempotent=20lab=5Fno?= =?UTF-8?q?tes=20schema=20migration=20with=20logging=20=F0=9F=A7=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Introduce migration helper to safely evolve lab_notes schema - Preserve existing data while adding translation-ready fields - Keep Lab Note attributes (category, excerpt, etc.) - Add lightweight migration logging (added columns only) - Make bootstrapDb thin and declarative co-authored-by: Lyric co-authored-by: Carmel --- CHANGELOG.md | 14 ++++++++++++++ package.json | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ee41d24 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,14 @@ +## [0.1.1] – 2025-12-29 + +### Added +- Idempotent database migration for `lab_notes` +- Lightweight schema version tracking via `schema_meta` +- Startup logging for applied schema changes + +### Changed +- Centralized DB migration logic into a dedicated module +- Made database bootstrap resilient to existing / older schemas + +### Notes +- No API behavior changes +- Safe to deploy over existing databases diff --git a/package.json b/package.json index 7ca9cd1..191cad5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "the-human-pattern-lab-api", - "version": "0.1.0", + "version": "0.1.1", "type": "module", "private": true, "description": "API backend for The Human Pattern Lab",