From 114f94f461d0df7f6f6f52de4beafd7d63049fdd Mon Sep 17 00:00:00 2001 From: mcull Date: Tue, 21 Oct 2025 00:53:55 -0700 Subject: [PATCH] Add describe-based item intake with concept generation --- README.md | 5 + .../migration.sql | 48 ++ prisma/schema.prisma | 45 ++ scripts/animate-storyboard-transitions.ts | 254 +++++++++ scripts/generate-imagen-storyboard.ts | 196 +++++++ scripts/generate-veo-video.ts | 156 ++++++ src/app/api/items/[id]/route.ts | 17 +- .../__tests__/suggest-visuals-route.test.ts | 142 +++++ .../api/items/create-from-concept/route.ts | 240 +++++++++ src/app/api/items/suggest-visuals/route.ts | 114 ++++ src/components/AddItemClient.tsx | 402 +++++++++++++- .../__tests__/prompt-safety-service.test.ts | 28 + src/lib/item-concept-service.ts | 494 ++++++++++++++++++ src/lib/prompt-safety-service.ts | 113 ++++ 14 files changed, 2241 insertions(+), 13 deletions(-) create mode 100644 prisma/migrations/20251021055238_add_item_concepts/migration.sql create mode 100644 scripts/animate-storyboard-transitions.ts create mode 100644 scripts/generate-imagen-storyboard.ts create mode 100644 scripts/generate-veo-video.ts create mode 100644 src/app/api/items/__tests__/suggest-visuals-route.test.ts create mode 100644 src/app/api/items/create-from-concept/route.ts create mode 100644 src/app/api/items/suggest-visuals/route.ts create mode 100644 src/lib/__tests__/prompt-safety-service.test.ts create mode 100644 src/lib/item-concept-service.ts create mode 100644 src/lib/prompt-safety-service.ts diff --git a/README.md b/README.md index e252211..1675cd4 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,11 @@ All environment variables are documented in `.env.example` with examples and des - `npm run db:migrate` - Run database migrations - `npm run db:studio` - Open Prisma Studio +## Item Intake Modes + +- **Use Camera:** Capture an item photo for automatic recognition and watercolor rendering. Works best when the item is in front of you. +- **Describe Instead:** Provide a name, brand, and details to receive three safe watercolor illustrations generated from licensed references or de-identified AI renderings—no digging items out of storage required. + ## Tech Stack - **Framework**: Next.js 15 with App Router diff --git a/prisma/migrations/20251021055238_add_item_concepts/migration.sql b/prisma/migrations/20251021055238_add_item_concepts/migration.sql new file mode 100644 index 0000000..70fdd6f --- /dev/null +++ b/prisma/migrations/20251021055238_add_item_concepts/migration.sql @@ -0,0 +1,48 @@ +-- CreateEnum +CREATE TYPE "public"."ItemConceptStatus" AS ENUM ('PENDING', 'READY', 'CONSUMED', 'EXPIRED', 'DISCARDED'); + +-- CreateEnum +CREATE TYPE "public"."ConceptSourceType" AS ENUM ('OPENVERSE', 'GENERATIVE'); + +-- CreateTable +CREATE TABLE "public"."item_concepts" ( + "id" TEXT NOT NULL, + "batchId" TEXT NOT NULL, + "ownerId" TEXT NOT NULL, + "inputName" TEXT, + "inputDescription" TEXT, + "inputBrand" TEXT, + "generatedName" TEXT, + "generatedDetails" JSONB, + "sourceType" "public"."ConceptSourceType" NOT NULL, + "sourceAttribution" JSONB, + "originalImageUrl" TEXT, + "watercolorUrl" TEXT, + "watercolorThumbUrl" TEXT, + "status" "public"."ItemConceptStatus" NOT NULL DEFAULT 'PENDING', + "expiresAt" TIMESTAMP(3) NOT NULL, + "consumedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "itemId" TEXT, + + CONSTRAINT "item_concepts_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "item_concepts_ownerId_idx" ON "public"."item_concepts"("ownerId"); + +-- CreateIndex +CREATE INDEX "item_concepts_batchId_idx" ON "public"."item_concepts"("batchId"); + +-- CreateIndex +CREATE INDEX "item_concepts_status_idx" ON "public"."item_concepts"("status"); + +-- CreateIndex +CREATE INDEX "item_concepts_expiresAt_idx" ON "public"."item_concepts"("expiresAt"); + +-- AddForeignKey +ALTER TABLE "public"."item_concepts" ADD CONSTRAINT "item_concepts_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "public"."users"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."item_concepts" ADD CONSTRAINT "item_concepts_itemId_fkey" FOREIGN KEY ("itemId") REFERENCES "public"."items"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index e969ed1..bb1dfd7 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -85,6 +85,7 @@ model User { complianceReports ComplianceReport[] feedbackVotes FeedbackVote[] complianceAdmin ComplianceReport[] @relation("ComplianceAdmin") + itemConcepts ItemConcept[] @@map("users") } @@ -215,10 +216,41 @@ model Item { collections ItemCollection[] reports UserReport[] disputes Dispute[] + concepts ItemConcept[] @@map("items") } +model ItemConcept { + id String @id @default(cuid()) + batchId String + ownerId String + inputName String? + inputDescription String? + inputBrand String? + generatedName String? + generatedDetails Json? + sourceType ConceptSourceType + sourceAttribution Json? + originalImageUrl String? + watercolorUrl String? + watercolorThumbUrl String? + status ItemConceptStatus @default(PENDING) + expiresAt DateTime + consumedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + itemId String? + owner User @relation(fields: [ownerId], references: [id], onDelete: Cascade) + item Item? @relation(fields: [itemId], references: [id]) + + @@index([ownerId]) + @@index([batchId]) + @@index([status]) + @@index([expiresAt]) + @@map("item_concepts") +} + model BorrowRequest { id String @id @default(cuid()) status BorrowRequestStatus @default(PENDING) @@ -360,6 +392,19 @@ enum BorrowRequestStatus { CANCELLED } +enum ItemConceptStatus { + PENDING + READY + CONSUMED + EXPIRED + DISCARDED +} + +enum ConceptSourceType { + OPENVERSE + GENERATIVE +} + model UserReport { id String @id @default(cuid()) reason UserReportReason diff --git a/scripts/animate-storyboard-transitions.ts b/scripts/animate-storyboard-transitions.ts new file mode 100644 index 0000000..e4f5580 --- /dev/null +++ b/scripts/animate-storyboard-transitions.ts @@ -0,0 +1,254 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +import { GoogleGenAI } from '@google/genai'; +import type { + GenerateImagesParameters, + GenerateVideosParameters, +} from '@google/genai'; + +interface Transition { + startFrame: number; + endFrame: number; + title: string; + prompt: string; +} + +async function animateTransitions() { + const apiKey = process.env.GOOGLE_AI_API_KEY; + if (!apiKey) { + throw new Error('GOOGLE_AI_API_KEY environment variable is required'); + } + + const client = new GoogleGenAI({ apiKey }); + + const storyboardDir = path.join( + process.cwd(), + 'docs', + 'video', + 'storyboard_frames' + ); + const outputDir = path.join( + process.cwd(), + 'docs', + 'video', + 'animated_transitions' + ); + + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + // Define the 4 key transitions for the sharing sequence (frames 7-11) + const transitions: Transition[] = [ + { + startFrame: 7, + endFrame: 8, + title: 'Descending to First House', + prompt: `Camera descends from aerial view of curved residential street toward a red brick ranch house. A miniature diorama with stop-motion aesthetic. The camera moves smoothly downward, getting closer to the house where a wooden ladder leans against it and a tiny figurine stands on the ladder. Warm Kodachrome colors, soft naturalistic lighting, dusk atmosphere. Model railroad aesthetic with textured miniature houses and foam trees. Smooth crane-down camera movement maintaining miniature scale.`, + }, + { + startFrame: 8, + endFrame: 9, + title: 'Ladder Handoff', + prompt: `In a miniature diorama neighborhood, a tiny figurine climbs down a wooden ladder from a red brick house (now with completed glowing Christmas lights). The figurine carries the ladder across the driveway toward the neighboring olive green house. A second tiny figurine from the green house walks toward the first to help carry the ladder. The ladder moves between the two houses in a handoff motion. Stop-motion miniature aesthetic with warm Kodachrome colors, soft dusk lighting, model railroad details. Smooth natural movement of the figurines and ladder.`, + }, + { + startFrame: 9, + endFrame: 10, + title: 'Lights Accumulating', + prompt: `Time-lapse effect in a miniature diorama neighborhood showing the wooden ladder being moved from house to house along a curved residential street. As the ladder moves, Christmas lights progressively appear and glow on each house roofline (warm colored bulbs—red, green, yellow, blue). The scene shows four houses with completed lights glowing warmly, and the ladder now at the fifth cream-colored house where a tiny figurine strings lights. Stop-motion miniature aesthetic, warm Kodachrome colors, dusk lighting deepening to early evening. Model railroad details with glowing house lights creating festive atmosphere.`, + }, + { + startFrame: 10, + endFrame: 11, + title: 'Completion and Celebration', + prompt: `Final house in a miniature diorama neighborhood receives Christmas lights. A tiny figurine completes stringing the last lights on the sixth house, creating a complete display. The wooden ladder is set down peacefully against the garage or in the yard. Tiny neighbor figurines gather in small groups on the street and driveways, appearing to chat and admire the completed lights. All six houses glow warmly with colored Christmas lights (red, green, yellow, blue, white) against the deepening blue hour evening sky. Stop-motion miniature aesthetic with warm Kodachrome colors, model railroad details. A sense of completion and community warmth. Camera static or slow gentle pan to show all houses together.`, + }, + ]; + + console.log( + '🎬 Starting Veo animation generation for storyboard transitions...' + ); + console.log(`📁 Storyboard frames: ${storyboardDir}`); + console.log(`📁 Output directory: ${outputDir}`); + console.log(`🎞️ Generating ${transitions.length} animated transitions...`); + console.log(''); + + for (const transition of transitions) { + try { + // Find the start frame file + const frameFiles = fs.readdirSync(storyboardDir); + const startFrameFile = frameFiles.find((f) => + f.startsWith( + `frame_${transition.startFrame.toString().padStart(2, '0')}_` + ) + ); + + if (!startFrameFile) { + throw new Error( + `Start frame ${transition.startFrame} not found in ${storyboardDir}` + ); + } + + const startFramePath = path.join(storyboardDir, startFrameFile); + + console.log( + `[${transition.startFrame}→${transition.endFrame}] 🎥 Animating: ${transition.title}` + ); + console.log(` Start frame: ${startFrameFile}`); + console.log(` Prompt: ${transition.prompt.substring(0, 80)}...`); + console.log(''); + + // Read the start frame image + const imageBuffer = fs.readFileSync(startFramePath); + const imageBase64 = imageBuffer.toString('base64'); + + const startTime = Date.now(); + + // First, regenerate this frame with Imagen to get proper image object + console.log( + ` 📸 Re-generating start frame with Imagen for proper format...` + ); + const imagenParams: GenerateImagesParameters & { + referenceImages: Array<{ + bytesBase64Encoded: string; + mimeType: string; + }>; + } = { + model: 'imagen-4.0-generate-001', + prompt: `Recreate this exact miniature diorama scene with no changes.`, + referenceImages: [ + { + bytesBase64Encoded: imageBase64, + mimeType: 'image/png', + }, + ], + config: { + numberOfImages: 1, + aspectRatio: '16:9', + }, + }; + + const imagenResult = await client.models.generateImages( + imagenParams as GenerateImagesParameters + ); + + const imagenImage = imagenResult.generatedImages?.[0]?.image; + if (!imagenImage) { + throw new Error('Failed to obtain regenerated frame for animation'); + } + + // Generate video with Veo using the Imagen image object + const videoParams: GenerateVideosParameters = { + model: 'veo-3.0-generate-001', + prompt: transition.prompt, + image: imagenImage, + config: { + aspectRatio: '16:9', + personGeneration: 'allow_adult', // Only allow_adult supported for image-to-video + }, + }; + const operation = await client.models.generateVideos(videoParams); + + console.log(` ✅ Generation request submitted`); + console.log(` 📋 Operation ID: ${operation.name}`); + console.log(` ⏳ Waiting for video generation...`); + + // Poll for completion + let pollOperation = operation; + let pollCount = 0; + const pollStartTime = Date.now(); + + while (!pollOperation.done) { + await new Promise((resolve) => setTimeout(resolve, 10000)); + pollCount++; + const elapsed = Math.round((Date.now() - pollStartTime) / 1000); + console.log( + ` ⏳ Still generating... (check ${pollCount}, ${elapsed}s)` + ); + + pollOperation = await client.operations.get({ + operation: pollOperation, + }); + } + + const totalTime = Math.round((Date.now() - startTime) / 1000); + console.log(` 🎉 Video generated in ${totalTime}s`); + + // Download the video + const generatedVideos = + ( + pollOperation.response as + | { generatedVideos?: Array<{ video?: { uri?: string } }> } + | undefined + )?.generatedVideos ?? []; + if (generatedVideos.length === 0) { + throw new Error('No video generated in response'); + } + + const [generatedVideo] = generatedVideos; + const videoUri = generatedVideo?.video?.uri; + if (!videoUri) { + throw new Error('Generated video URI missing in response'); + } + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const outputPath = path.join( + outputDir, + `transition_${transition.startFrame}_to_${transition.endFrame}_${timestamp}.mp4` + ); + + console.log(` 💾 Downloading video...`); + + const videoResponse = await fetch(videoUri, { + headers: { + 'x-goog-api-key': apiKey, + }, + }); + + if (!videoResponse.ok) { + throw new Error( + `Failed to download: ${videoResponse.status} ${videoResponse.statusText}` + ); + } + + const videoBuffer = await videoResponse.arrayBuffer(); + fs.writeFileSync(outputPath, Buffer.from(videoBuffer)); + + console.log(` ✅ Saved: ${path.basename(outputPath)}`); + console.log(''); + } catch (error) { + console.error( + ` ❌ Error animating transition ${transition.startFrame}→${transition.endFrame}:`, + error + ); + console.error(''); + // Continue with next transition + continue; + } + } + + console.log(''); + console.log('✅ Animation generation complete!'); + console.log(`📁 All videos saved to: ${outputDir}`); + console.log(''); + console.log('🎬 Next steps:'); + console.log(' 1. Review the animated transitions'); + console.log(' 2. Edit them together in sequence'); + console.log(' 3. Add sound design and music'); + console.log( + ' 4. Decide whether to refine frames 1-6 or use alternative opening' + ); +} + +// Run the script +animateTransitions() + .then(() => { + console.log('🏁 Script completed successfully'); + process.exit(0); + }) + .catch((error) => { + console.error(''); + console.error('💥 Script failed:', error); + process.exit(1); + }); diff --git a/scripts/generate-imagen-storyboard.ts b/scripts/generate-imagen-storyboard.ts new file mode 100644 index 0000000..b00b13d --- /dev/null +++ b/scripts/generate-imagen-storyboard.ts @@ -0,0 +1,196 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +import { GoogleGenAI } from '@google/genai'; +import type { GenerateImagesParameters } from '@google/genai'; + +async function generateStoryboardFrames() { + const apiKey = process.env.GOOGLE_AI_API_KEY; + if (!apiKey) { + throw new Error('GOOGLE_AI_API_KEY environment variable is required'); + } + + const client = new GoogleGenAI({ apiKey }); + + // Define all 11 frame prompts + const frames = [ + { + number: 1, + title: 'Vans Arriving', + prompt: `A miniature diorama aerial view of a suburban street with six adjacent 1950s ranch-style houses (yellow, light blue, pale green, cream, grey, and beige). Six white delivery vans are driving up the street from the left side, approaching the houses in a neat convoy. The houses have small front yards with tiny model trees, driveways, and closed garages. Physical miniature diorama photographed with real camera, warm Kodachrome color palette, soft naturalistic lighting, shallow depth of field. Model railroad aesthetic with visible textures—plastic houses, foam bushes, painted details. Camera angle: 45-degree aerial view. Warm afternoon lighting with soft shadows.`, + }, + { + number: 2, + title: 'Vans Parked, Delivery Begins', + prompt: `A miniature diorama aerial view of the same suburban street. Six white delivery vans are now parked in front of six adjacent houses, doors open. Tiny figurines representing delivery people stand beside each van, holding or unloading small wooden ladders. The ladders are identical—simple wooden A-frame stepladders visible next to each van. Homeowners (tiny figurines) stand in driveways. Physical miniature diorama photographed with real camera, warm Kodachrome color palette, soft naturalistic lighting, shallow depth of field. Model railroad aesthetic—plastic delivery vans, painted figurines, foam yard details. Camera angle: 45-degree aerial view. Warm afternoon lighting.`, + }, + { + number: 3, + title: 'Ladders Going Into Garages', + prompt: `A miniature diorama aerial view of six suburban houses. The delivery vans are still parked. At each house, tiny figurines (delivery people and homeowners) are positioned near open garage doors, with wooden ladders being carried or positioned near the garage entrances. The garages are open, showing interior details—tiny bikes, boxes, and storage items visible inside. The ladders are in various stages of being moved: some halfway to the garage, some leaning against garage doors. Physical miniature diorama photographed with real camera, warm Kodachrome color palette, soft naturalistic lighting. Model railroad aesthetic with visible painted details. Camera angle: 45-degree aerial view.`, + }, + { + number: 4, + title: 'Garages Closing, Vans Leaving', + prompt: `A miniature diorama aerial view of six suburban houses. The garage doors are now closed (or mostly closed), with the wooden ladders stored inside. The six white delivery vans are pulling away from the curbs, driving off toward the right side of the frame in a departing convoy. The street is returning to quiet suburban stillness. Tiny homeowner figurines stand in driveways watching the vans leave. Physical miniature diorama photographed with real camera, warm Kodachrome color palette, soft naturalistic lighting, shallow depth of field. Model railroad aesthetic—plastic houses, foam trees, painted road. Camera angle: 45-degree aerial view. Warm afternoon light casting soft shadows.`, + }, + { + number: 5, + title: 'Pulling Up from Neighborhood #1', + prompt: `A high-altitude aerial view of a miniature suburban neighborhood diorama. The six houses from the previous scene are now visible as a small cluster in the lower portion of frame. The surrounding neighborhood grid is visible—multiple blocks of tiny houses, street grid, small parks with miniature trees, a main street with shops. Everything is a physical miniature model with visible textures. The depth of field is wider now, showing more of the town layout. Physical miniature diorama photographed from above with real camera, warm Kodachrome color palette, soft naturalistic lighting. Model railroad town aesthetic—foam terrain, plastic buildings, painted roads. Camera altitude: very high, almost bird's-eye view. Warm afternoon lighting.`, + }, + { + number: 6, + title: 'Sweeping Across Town', + prompt: `An aerial view of a different section of the miniature town diorama. The camera has moved laterally across the model landscape. This view shows a downtown main street area with tiny storefronts, a small park with a fountain and model trees, and residential neighborhoods in the background. Some buildings are brick, others are painted wood. The miniature town layout feels organic, not grid-perfect. Physical miniature diorama photographed with real camera, warm Kodachrome color palette, soft naturalistic lighting with gentle depth of field. Model railroad aesthetic—hand-painted buildings, foam trees, textured roads. Camera angle: 45-degree aerial view panning across landscape. Warm afternoon light.`, + }, + { + number: 7, + title: 'Descending Toward Neighborhood #2', + prompt: `A miniature diorama aerial view descending toward a second residential neighborhood. Six ranch-style houses are arranged along a curved residential street (not a closed circle—an open curve or L-shape). The houses are different colors than the first neighborhood—warm reds, oranges, browns, olive green, cream. Some houses already have tiny Christmas lights strung along rooflines (colored dots of light). A single wooden ladder is visible leaning against the first house on the left, where a tiny figurine stands on the roof. Physical miniature diorama photographed with real camera, warm Kodachrome color palette, soft naturalistic lighting, shallow depth of field. Model railroad aesthetic. Camera angle: 45-degree aerial view, slightly lower altitude. Warm afternoon transitioning to dusk lighting—soft golden light with hints of blue shadow.`, + }, + { + number: 8, + title: 'First House - Ladder in Use', + prompt: `A miniature diorama close-up aerial view of a red brick ranch house with a single wooden ladder leaning against it. A tiny figurine stands on the ladder near the roofline, appearing to string Christmas lights. Another tiny figurine stands in the driveway below, looking up. The house has warm glowing Christmas lights already strung along part of the roofline (small colored bulbs—red, green, yellow). Five other houses are visible in the background along the curved street, most without lights yet. Physical miniature diorama photographed with real camera, warm Kodachrome color palette, soft naturalistic lighting. Model railroad aesthetic—textured plastic house, foam yard, painted figurines. Camera angle: 45-degree aerial close-up. Dusk lighting—warm golden hour with house lights beginning to glow.`, + }, + { + number: 9, + title: 'Ladder Being Passed to Second House', + prompt: `A miniature diorama aerial view showing two adjacent houses along a curved residential street. A tiny figurine is carrying the wooden ladder from the first house (which now has complete Christmas lights glowing) toward the second house (an olive green ranch house). A second figurine from the olive green house walks toward the first figurine to help carry the ladder. The ladder is positioned between the two houses, mid-transfer. Three other houses are visible in the background, two without lights, one with lights partially strung. Physical miniature diorama photographed with real camera, warm Kodachrome color palette, soft naturalistic lighting. Model railroad aesthetic with visible textures. Camera angle: 45-degree aerial view. Dusk lighting—warm glowing house lights against deepening blue shadows.`, + }, + { + number: 10, + title: 'Multiple Houses Lighting Up', + prompt: `A miniature diorama aerial view of the curved residential street with six houses. Four houses now have glowing Christmas lights strung along their rooflines (warm colored bulbs—red, green, yellow, blue). The wooden ladder is currently leaning against the fifth house (cream-colored), where a tiny figurine is on the ladder stringing lights. Two more figurines stand in the driveway watching or waiting. The first four houses glow warmly with completed light displays. The sixth house in the background is still dark. Physical miniature diorama photographed with real camera, warm Kodachrome color palette, soft naturalistic lighting. Model railroad aesthetic—textured plastic houses, foam trees, painted details. Camera angle: 45-degree aerial view. Dusk/early evening lighting—houses glowing, blue evening shadows.`, + }, + { + number: 11, + title: 'All Houses Lit - Ladder at Rest', + prompt: `A miniature diorama aerial view of six houses along a curved residential street at dusk. All six houses now have glowing Christmas lights strung along their rooflines, creating a warm, festive scene. The lights twinkle with colored bulbs (red, green, yellow, blue, white). The single wooden ladder is leaning peacefully against the garage of the final house, or propped in a front yard. Tiny figurines (neighbors) are gathered in small groups on the street and driveways, appearing to chat or admire the lights. A warm communal feeling. Physical miniature diorama photographed with real camera, warm Kodachrome color palette, soft naturalistic lighting. Model railroad aesthetic—glowing house windows, textured details, foam trees with tiny lights. Camera angle: 45-degree aerial view. Evening lighting—rich blue hour with warm glowing lights creating inviting atmosphere.`, + }, + ]; + + // Create output directory + const outputDir = path.join( + process.cwd(), + 'docs', + 'video', + 'storyboard_frames' + ); + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + console.log('🎨 Starting Imagen 3 storyboard generation...'); + console.log(`📁 Output directory: ${outputDir}`); + console.log(`🖼️ Generating ${frames.length} frames...`); + console.log(''); + + for (const frame of frames) { + try { + console.log(`[${frame.number}/11] 🖌️ Generating: ${frame.title}`); + console.log(` Prompt: ${frame.prompt.substring(0, 80)}...`); + + const startTime = Date.now(); + + // Generate image with Imagen 4 + const request: GenerateImagesParameters = { + model: 'imagen-4.0-generate-001', + prompt: frame.prompt, + config: { + numberOfImages: 1, + aspectRatio: '16:9', + }, + }; + const result = await client.models.generateImages(request); + + const elapsed = Date.now() - startTime; + + // Debug: log response structure + console.log(' Response:', JSON.stringify(result).substring(0, 200)); + + // Save the generated image (check both snake_case and camelCase) + const generatedImages = result.generatedImages ?? []; + if (generatedImages.length === 0) { + console.error(' Full response:', JSON.stringify(result, null, 2)); + throw new Error('No images generated in response'); + } + const generatedImage = generatedImages[0]; + const image = generatedImage?.image as + | { + imageBytes?: string; + gcsUri?: string; + imageUrl?: string; + } + | undefined; + if (!image) { + throw new Error('No image data in generated image'); + } + + const outputPath = path.join( + outputDir, + `frame_${frame.number.toString().padStart(2, '0')}_${frame.title.toLowerCase().replace(/\s+/g, '_')}.png` + ); + + // Imagen returns imageBytes directly in base64, not a URL + if (image.imageBytes) { + // Decode base64 and save directly + const imageBuffer = Buffer.from(image.imageBytes, 'base64'); + fs.writeFileSync(outputPath, imageBuffer); + } else if (image.imageUrl || image.gcsUri) { + // Fallback: if it's a URL, download it + const remoteUri = image.imageUrl ?? image.gcsUri; + if (!remoteUri) { + throw new Error('Remote image URI missing'); + } + const imageResponse = await fetch(remoteUri, { + headers: { + 'x-goog-api-key': apiKey, + }, + }); + + if (!imageResponse.ok) { + throw new Error( + `Failed to download image: ${imageResponse.status} ${imageResponse.statusText}` + ); + } + + const imageBuffer = await imageResponse.arrayBuffer(); + fs.writeFileSync(outputPath, Buffer.from(imageBuffer)); + } else { + throw new Error('No imageBytes or imageUrl in response'); + } + + console.log(` ✅ Generated in ${elapsed}ms`); + console.log(` 💾 Saved: ${path.basename(outputPath)}`); + console.log(''); + } catch (error) { + console.error(` ❌ Error generating frame ${frame.number}:`, error); + console.error(''); + // Continue with next frame instead of failing completely + continue; + } + } + + console.log(''); + console.log('✅ Storyboard generation complete!'); + console.log(`📁 All frames saved to: ${outputDir}`); + console.log(''); + console.log('🎬 Next steps:'); + console.log(' 1. Review the generated frames for consistency'); + console.log(' 2. Regenerate any problematic frames individually'); + console.log(' 3. Use frames as input for Veo image-to-video generation'); +} + +// Run the script +generateStoryboardFrames() + .then(() => { + console.log('🏁 Script completed successfully'); + process.exit(0); + }) + .catch((error) => { + console.error(''); + console.error('💥 Script failed:', error); + process.exit(1); + }); diff --git a/scripts/generate-veo-video.ts b/scripts/generate-veo-video.ts new file mode 100644 index 0000000..1167b69 --- /dev/null +++ b/scripts/generate-veo-video.ts @@ -0,0 +1,156 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +import { GoogleGenAI } from '@google/genai'; +import type { GenerateVideosParameters } from '@google/genai'; + +async function generateVideo() { + const apiKey = process.env.GOOGLE_AI_API_KEY; + if (!apiKey) { + throw new Error('GOOGLE_AI_API_KEY environment variable is required'); + } + + const client = new GoogleGenAI({ apiKey }); + + // The prompt from scene_01_ladder_delivery_prompt.md + const prompt = `Generate a 40-second stop-motion style cinematic video of miniature suburban neighborhoods, shot as if with a real camera over a physical diorama. Use warm Kodachrome-style color grading and soft naturalistic lighting. The video begins with an aerial view of six adjacent ranch-style houses, where six white delivery vans arrive simultaneously and deliver identical wooden ladders. Tiny figures place the ladders in their garages, which close in sequence. The camera then performs a smooth crane-up and lateral dolly pan across a miniature town landscape, before descending into a second neighborhood arranged in a cul-de-sac. Here, a single ladder is passed hand-to-hand between six neighbors, each using it to string glowing Christmas lights on their house in turn. The houses progressively light up with warm holiday lights as the ladder circulates. The aesthetic should feel handcrafted and textured—avoid glossy CGI—with slight camera sway and vintage 1960s educational film warmth. Audio: ambient suburban sounds, garage doors, wind, and soft holiday ambiance.`; + + const negativePrompt = `Hyperrealistic CGI human faces, glossy over-polished surfaces, pure photorealism, cartoonish bounce or exaggerated physics, modern flat vector animation style, corporate stock footage look, overly saturated colors`; + + console.log('🎬 Starting Veo 3 video generation...'); + console.log('📝 Prompt:', prompt.substring(0, 100) + '...'); + console.log(''); + + try { + // Start video generation + const videoRequest: GenerateVideosParameters = { + model: 'veo-3.0-generate-001', + prompt, + config: { + negativePrompt, + aspectRatio: '16:9', + personGeneration: 'allow_all', + }, + }; + const operation = await client.models.generateVideos(videoRequest); + + console.log('✅ Generation request submitted'); + console.log(`📋 Operation ID: ${operation.name}`); + console.log('⏳ Waiting for video generation to complete...'); + console.log(' (This can take 11 seconds to 6 minutes)'); + console.log(''); + + // Poll for completion + let pollOperation = operation; + let pollCount = 0; + const startTime = Date.now(); + + while (!pollOperation.done) { + await new Promise((resolve) => setTimeout(resolve, 10000)); // Wait 10 seconds + pollCount++; + const elapsed = Math.round((Date.now() - startTime) / 1000); + console.log( + `⏳ Still generating... (check ${pollCount}, ${elapsed}s elapsed)` + ); + + // Get the latest operation status + pollOperation = await client.operations.get({ operation: pollOperation }); + } + + const totalTime = Math.round((Date.now() - startTime) / 1000); + console.log(''); + console.log(`🎉 Video generation complete! (${totalTime}s total)`); + + // Debug: log the response structure + console.log(''); + console.log('📊 Response structure:'); + console.log( + ' Response:', + JSON.stringify(pollOperation.response, null, 2).substring(0, 500) + ); + console.log(''); + + // Check if response and generatedVideos exist (camelCase in JS SDK) + const generatedVideos = + ( + pollOperation.response as + | { generatedVideos?: Array<{ video?: { uri?: string } }> } + | undefined + )?.generatedVideos ?? []; + if (generatedVideos.length === 0) { + console.error('❌ No generated videos found in response'); + console.error('Full response:', JSON.stringify(pollOperation, null, 2)); + throw new Error('No generated videos in response'); + } + + // Download the generated video (camelCase in JS SDK) + const [generatedVideo] = generatedVideos; + const videoUri = generatedVideo?.video?.uri; + if (!videoUri) { + throw new Error('Generated video URI missing'); + } + + // Create output directory if it doesn't exist + const outputDir = path.join(process.cwd(), 'docs', 'video', 'outputs'); + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const outputPath = path.join( + outputDir, + `scene_01_ladder_paradox_${timestamp}.mp4` + ); + + console.log(`💾 Downloading video from: ${videoUri}`); + console.log(`💾 Saving to: ${outputPath}`); + + // Download the video file from the URI with API key authentication + const videoResponse = await fetch(videoUri, { + headers: { + 'x-goog-api-key': apiKey, + }, + }); + if (!videoResponse.ok) { + throw new Error( + `Failed to download video: ${videoResponse.status} ${videoResponse.statusText}` + ); + } + + const videoBuffer = await videoResponse.arrayBuffer(); + fs.writeFileSync(outputPath, Buffer.from(videoBuffer)); + + console.log(''); + console.log('✅ SUCCESS!'); + console.log(`📹 Video saved to: ${outputPath}`); + console.log(''); + console.log('Video details:'); + console.log(` - Duration: 8 seconds`); + console.log(` - Resolution: 1080p`); + console.log(` - Aspect Ratio: 16:9`); + console.log(` - Model: veo-3.0-generate-001`); + console.log(` - SynthID watermark: Yes`); + console.log(''); + console.log('🎬 Ready to review! Open the file to see the results.'); + } catch (error) { + console.error(''); + console.error('❌ Error generating video:', error); + if (error instanceof Error) { + console.error(' Message:', error.message); + } + throw error; + } +} + +// Run the script +generateVideo() + .then(() => { + console.log(''); + console.log('🏁 Script completed successfully'); + process.exit(0); + }) + .catch((error) => { + console.error(''); + console.error('💥 Script failed:', error); + process.exit(1); + }); diff --git a/src/app/api/items/[id]/route.ts b/src/app/api/items/[id]/route.ts index 567f8d8..1143d2a 100644 --- a/src/app/api/items/[id]/route.ts +++ b/src/app/api/items/[id]/route.ts @@ -468,14 +468,15 @@ export async function PATCH( if (libraryIds && Array.isArray(libraryIds)) { // Replacement mode await db.itemCollection.deleteMany({ where: { itemId } }); - if (libraryIds.length > 0) { - await db.$transaction( - libraryIds.map((libraryId: string) => - db.itemCollection.create({ - data: { itemId, collectionId: libraryId }, - }) - ) - ); + const uniqueLibraryIds = Array.from(new Set(libraryIds as string[])); + if (uniqueLibraryIds.length > 0) { + await db.itemCollection.createMany({ + data: uniqueLibraryIds.map((libraryId) => ({ + itemId, + collectionId: libraryId, + })), + skipDuplicates: true, + }); } } else { // Incremental mode diff --git a/src/app/api/items/__tests__/suggest-visuals-route.test.ts b/src/app/api/items/__tests__/suggest-visuals-route.test.ts new file mode 100644 index 0000000..685ffb4 --- /dev/null +++ b/src/app/api/items/__tests__/suggest-visuals-route.test.ts @@ -0,0 +1,142 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; + +const mockGetServerSession = vi.hoisted(() => vi.fn()); +const mockGenerateConcepts = vi.hoisted(() => vi.fn()); +const mockDiscardBatch = vi.hoisted(() => vi.fn()); + +vi.mock('next-auth', () => ({ + getServerSession: mockGetServerSession, +})); + +vi.mock('@/lib/item-concept-service', () => ({ + ItemConceptService: vi.fn().mockImplementation(() => ({ + generateConcepts: mockGenerateConcepts, + discardBatch: mockDiscardBatch, + })), +})); + +describe('POST /api/items/suggest-visuals', () => { + beforeEach(() => { + mockGetServerSession.mockReset(); + mockGenerateConcepts.mockReset(); + mockDiscardBatch.mockReset(); + }); + + it('returns 401 when user is unauthenticated', async () => { + mockGetServerSession.mockResolvedValue(null); + const { POST } = await import('../suggest-visuals/route'); + + const request = { + json: vi.fn(), + headers: new Headers(), + } as any; + + const response = await POST(request); + expect(response.status).toBe(401); + }); + + it('returns generated concepts when request is valid', async () => { + mockGetServerSession.mockResolvedValue({ user: { id: 'user_123' } }); + mockGenerateConcepts.mockResolvedValue({ + batchId: 'batch_1', + options: [ + { + id: 'concept_1', + watercolorUrl: 'https://example.com/image.webp', + watercolorThumbUrl: 'https://example.com/thumb.webp', + sourceType: 'OPENVERSE', + generatedName: 'Cordless Drill', + sourceAttribution: null, + }, + ], + }); + + const { POST } = await import('../suggest-visuals/route'); + + const requestBody = { name: 'Cordless drill', count: 10 }; + const request = { + json: vi.fn().mockResolvedValue(requestBody), + headers: new Headers(), + } as any; + + const response = await POST(request); + + expect(mockGenerateConcepts).toHaveBeenCalledWith({ + userId: 'user_123', + name: 'Cordless drill', + description: undefined, + brand: undefined, + count: 5, + }); + + const payload = await response.json(); + expect(payload).toEqual({ + batchId: 'batch_1', + options: [ + { + conceptId: 'concept_1', + watercolorUrl: 'https://example.com/image.webp', + watercolorThumbUrl: 'https://example.com/thumb.webp', + sourceType: 'OPENVERSE', + generatedName: 'Cordless Drill', + sourceAttribution: null, + }, + ], + }); + }); + + it('discards previous batches when discardBatchId provided', async () => { + mockGetServerSession.mockResolvedValue({ user: { id: 'user_123' } }); + mockGenerateConcepts.mockResolvedValue({ batchId: 'batch_2', options: [] }); + + const { POST } = await import('../suggest-visuals/route'); + + const request = { + json: vi.fn().mockResolvedValue({ + name: 'Camping tent', + discardBatchId: 'old_batch', + }), + headers: new Headers(), + } as any; + + await POST(request); + + expect(mockDiscardBatch).toHaveBeenCalledWith('old_batch', 'user_123'); + }); + + it('returns 503 when concept storage is missing', async () => { + mockGetServerSession.mockResolvedValue({ user: { id: 'user_123' } }); + mockGenerateConcepts.mockRejectedValue( + new Error('ITEM_CONCEPTS_TABLE_MISSING') + ); + + const { POST } = await import('../suggest-visuals/route'); + + const request = { + json: vi.fn().mockResolvedValue({ name: 'Cordless drill' }), + headers: new Headers(), + } as any; + + const response = await POST(request); + expect(response.status).toBe(503); + }); + + it('returns 404 when no visuals can be produced', async () => { + mockGetServerSession.mockResolvedValue({ user: { id: 'user_123' } }); + mockGenerateConcepts.mockRejectedValue( + new Error( + 'No matching library photos found. Configure GOOGLE_AI_API_KEY to enable watercolor suggestions.' + ) + ); + + const { POST } = await import('../suggest-visuals/route'); + + const request = { + json: vi.fn().mockResolvedValue({ name: 'Cordless drill' }), + headers: new Headers(), + } as any; + + const response = await POST(request); + expect(response.status).toBe(404); + }); +}); diff --git a/src/app/api/items/create-from-concept/route.ts b/src/app/api/items/create-from-concept/route.ts new file mode 100644 index 0000000..c34789e --- /dev/null +++ b/src/app/api/items/create-from-concept/route.ts @@ -0,0 +1,240 @@ +import crypto from 'crypto'; + +import { ItemConceptStatus } from '@prisma/client'; +import { NextRequest, NextResponse } from 'next/server'; +import { getServerSession } from 'next-auth'; + +import { authOptions } from '@/lib/auth'; +import { db } from '@/lib/db'; +import { ItemConceptService } from '@/lib/item-concept-service'; + +function slugify(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 60); +} + +export async function POST(request: NextRequest) { + let concept: Awaited< + ReturnType + > | null = null; + + try { + const session = await getServerSession(authOptions); + + if (!session?.user) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + } + + const userId = + (session.user as any).id || + (session as any).user?.id || + (session as any).userId; + + if (!userId) { + return NextResponse.json({ error: 'User ID not found' }, { status: 400 }); + } + + const body = await request.json().catch(() => null); + if (!body) { + return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }); + } + + const { + conceptId, + name: overrideName, + description: overrideDescription, + libraryIds, + category = 'other', + } = body; + + if (!conceptId || typeof conceptId !== 'string') { + return NextResponse.json( + { error: 'conceptId is required' }, + { status: 400 } + ); + } + + const conceptService = new ItemConceptService(); + concept = await conceptService.consumeConcept(conceptId, userId); + + if (!concept) { + return NextResponse.json( + { error: 'Concept not found or already used' }, + { status: 404 } + ); + } + + const itemName = ( + overrideName || + concept.inputName || + concept.generatedName || + '' + ).trim(); + if (!itemName) { + return NextResponse.json( + { error: 'Item name is required' }, + { status: 400 } + ); + } + + const itemDescription = ( + overrideDescription || + concept.inputDescription || + 'Added via description flow' + ).trim(); + + const imageUrl = concept.originalImageUrl || concept.watercolorUrl; + const watercolorUrl = concept.watercolorUrl; + const watercolorThumbUrl = concept.watercolorThumbUrl; + + if (!watercolorUrl) { + return NextResponse.json( + { error: 'Concept is missing visual assets' }, + { status: 422 } + ); + } + + const slug = slugify(itemName); + const uniqueName = `${slug || crypto.randomUUID()}-${category}`; + + const libraryIdArray: string[] = Array.isArray(libraryIds) + ? libraryIds.filter((id: unknown): id is string => typeof id === 'string') + : []; + const itemId = await db.$transaction(async (tx) => { + let stuffType = await tx.stuffType.findFirst({ + where: { name: uniqueName }, + }); + + if (!stuffType) { + stuffType = await tx.stuffType.create({ + data: { + name: uniqueName, + displayName: itemName, + category, + iconPath: watercolorUrl, + }, + }); + } + + const createdItem = await tx.item.create({ + data: { + name: itemName, + description: itemDescription, + category, + condition: 'good', + imageUrl, + watercolorUrl, + watercolorThumbUrl, + styleVersion: watercolorUrl ? 'wc_v1' : null, + ownerId: userId, + stuffTypeId: stuffType.id, + active: true, + }, + }); + + const uniqueLibraryIds = Array.from(new Set(libraryIdArray)); + if (uniqueLibraryIds.length > 0) { + await tx.itemCollection.createMany({ + data: uniqueLibraryIds.map((libraryId) => ({ + itemId: createdItem.id, + collectionId: libraryId, + })), + skipDuplicates: true, + }); + } + + await tx.itemConcept.update({ + where: { id: conceptId }, + data: { + itemId: createdItem.id, + }, + }); + + return createdItem.id; + }); + + const item = await db.item.findUnique({ + where: { id: itemId }, + include: { + owner: { + select: { + id: true, + name: true, + image: true, + }, + }, + stuffType: { + select: { + displayName: true, + category: true, + iconPath: true, + }, + }, + collections: { + include: { + collection: { + select: { + id: true, + name: true, + }, + }, + }, + }, + }, + }); + + if (!item) { + throw new Error('Created item could not be fetched'); + } + + return NextResponse.json({ + itemId: item.id, + conceptId: concept.id, + item: { + id: item.id, + name: item.name, + description: item.description, + category: item.category, + condition: item.condition, + imageUrl: item.imageUrl, + watercolorUrl: item.watercolorUrl, + watercolorThumbUrl: item.watercolorThumbUrl, + isAvailable: !item.currentBorrowRequestId, + createdAt: item.createdAt, + owner: item.owner, + stuffType: item.stuffType, + libraries: item.collections.map((ic) => ic.collection), + }, + }); + } catch (error) { + console.error('Error creating item from concept:', error); + if (concept) { + try { + await db.itemConcept.update({ + where: { id: concept.id }, + data: { + status: ItemConceptStatus.READY, + consumedAt: null, + itemId: null, + }, + }); + } catch (restoreError) { + console.error( + 'Failed to restore concept status after error:', + restoreError + ); + } + } + return NextResponse.json( + { + error: 'Failed to create item from concept', + message: error instanceof Error ? error.message : 'Unknown error', + }, + { status: 500 } + ); + } +} diff --git a/src/app/api/items/suggest-visuals/route.ts b/src/app/api/items/suggest-visuals/route.ts new file mode 100644 index 0000000..ea2f068 --- /dev/null +++ b/src/app/api/items/suggest-visuals/route.ts @@ -0,0 +1,114 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getServerSession } from 'next-auth'; + +import { authOptions } from '@/lib/auth'; +import { ItemConceptService } from '@/lib/item-concept-service'; + +export async function POST(request: NextRequest) { + try { + const session = await getServerSession(authOptions); + + if (!session?.user) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + } + + const userId = + (session.user as any).id || + (session as any).user?.id || + (session as any).userId; + + if (!userId) { + return NextResponse.json({ error: 'User ID not found' }, { status: 400 }); + } + + const body = await request.json().catch(() => null); + if (!body) { + return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }); + } + + const { name, description, brand, count, discardBatchId } = body; + + if (!name || typeof name !== 'string') { + return NextResponse.json({ error: 'Name is required' }, { status: 400 }); + } + + const safeCount = Math.min(Math.max(parseInt(count ?? 3, 10) || 3, 1), 5); + + const service = new ItemConceptService(); + + if (discardBatchId && typeof discardBatchId === 'string') { + await service.discardBatch(discardBatchId, userId); + } + + const result = await service.generateConcepts({ + userId, + name, + description, + brand, + count: safeCount, + }); + + return NextResponse.json({ + batchId: result.batchId, + options: result.options.map((option) => ({ + conceptId: option.id, + watercolorUrl: option.watercolorUrl, + watercolorThumbUrl: option.watercolorThumbUrl, + sourceType: option.sourceType, + generatedName: option.generatedName, + sourceAttribution: option.sourceAttribution ?? null, + })), + }); + } catch (error) { + console.error('Error generating item concepts:', error); + if ( + error instanceof Error && + error.message?.includes( + 'GOOGLE_AI_API_KEY environment variable is required' + ) + ) { + return NextResponse.json( + { + error: 'Concept generation is not configured', + details: 'Google AI credentials are required', + }, + { status: 503 } + ); + } + + if ( + error instanceof Error && + error.message === 'ITEM_CONCEPTS_TABLE_MISSING' + ) { + return NextResponse.json( + { + error: 'Concept storage unavailable', + details: + 'Run `npx prisma migrate dev` to create the item_concepts table.', + }, + { status: 503 } + ); + } + + if ( + error instanceof Error && + error.message?.includes('No matching library photos found') + ) { + return NextResponse.json( + { + error: 'No visuals found', + message: error.message, + }, + { status: 404 } + ); + } + + return NextResponse.json( + { + error: 'Failed to generate concepts', + message: error instanceof Error ? error.message : 'Unknown error', + }, + { status: 500 } + ); + } +} diff --git a/src/components/AddItemClient.tsx b/src/components/AddItemClient.tsx index ef489f9..7e11548 100644 --- a/src/components/AddItemClient.tsx +++ b/src/components/AddItemClient.tsx @@ -17,9 +17,13 @@ import { Alert, IconButton, TextField, + Tabs, + Tab, + Stack, } from '@mui/material'; import { useRouter } from 'next/navigation'; import { useRef, useState, useCallback, useEffect } from 'react'; +import type { SyntheticEvent } from 'react'; import { brandColors } from '@/theme/brandTokens'; @@ -85,6 +89,17 @@ type CaptureState = | 'uploaded' | 'error'; +type AddMode = 'camera' | 'describe'; + +interface ConceptOption { + conceptId: string; + watercolorUrl: string | null; + watercolorThumbUrl: string | null; + sourceType: 'OPENVERSE' | 'GENERATIVE'; + generatedName?: string | null; + sourceAttribution?: Record | null; +} + interface RecognitionResult { name: string; description: string; @@ -107,6 +122,7 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { const canvasRef = useRef(null); const streamRef = useRef(null); + const [mode, setMode] = useState('camera'); const [state, setState] = useState('permission'); const [error, setError] = useState(null); const [recognitionResult, setRecognitionResult] = @@ -122,6 +138,17 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { const [showWatercolor, setShowWatercolor] = useState(false); const [showHintInput, setShowHintInput] = useState(false); const [nameHint, setNameHint] = useState(''); + const [conceptBatchId, setConceptBatchId] = useState(null); + const [conceptOptions, setConceptOptions] = useState([]); + const [isGeneratingConcepts, setIsGeneratingConcepts] = useState(false); + const [conceptError, setConceptError] = useState(null); + const [describeName, setDescribeName] = useState(''); + const [describeDescription, setDescribeDescription] = useState(''); + const [describeBrand, setDescribeBrand] = useState(''); + const [selectedConceptId, setSelectedConceptId] = useState( + null + ); + const [isCreatingFromConcept, setIsCreatingFromConcept] = useState(false); // Request camera permission and start stream const startCamera = useCallback(async () => { @@ -606,6 +633,120 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { } }, [capturedImageUrl, nameHint, draftItemId]); + const handleModeChange = useCallback( + (_event: SyntheticEvent, newMode: AddMode) => { + setMode(newMode); + if (newMode === 'camera') { + setError(null); + setState('permission'); + } + }, + [] + ); + + const handleGenerateConcepts = useCallback(async () => { + if (!describeName.trim()) { + setConceptError('Please provide a name for your item.'); + return; + } + + setIsGeneratingConcepts(true); + setConceptError(null); + setSelectedConceptId(null); + setConceptOptions([]); + + try { + const response = await fetch('/api/items/suggest-visuals', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + name: describeName.trim(), + description: describeDescription.trim() || undefined, + brand: describeBrand.trim() || undefined, + discardBatchId: conceptBatchId || undefined, + }), + }); + + if (!response.ok) { + const data = await response.json().catch(() => ({})); + const message = + data.message || data.error || 'Failed to generate visuals'; + throw new Error(message); + } + + const data = await response.json(); + setConceptBatchId(data.batchId || null); + const options = (data.options as ConceptOption[] | undefined) ?? []; + setConceptOptions(options); + + if (options.length === 0) { + setConceptError( + 'No visuals found. Try adding more detail or a brand name.' + ); + } + } catch (conceptErr) { + console.error('Concept generation failed:', conceptErr); + setConceptError( + conceptErr instanceof Error + ? conceptErr.message + : 'Failed to generate visuals' + ); + } finally { + setIsGeneratingConcepts(false); + } + }, [describeName, describeDescription, describeBrand, conceptBatchId]); + + const handleCreateFromConcept = useCallback(async () => { + if (!selectedConceptId) { + setConceptError('Please choose one of the illustrations first.'); + return; + } + + setIsCreatingFromConcept(true); + setConceptError(null); + + try { + const response = await fetch('/api/items/create-from-concept', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + conceptId: selectedConceptId, + name: describeName.trim(), + description: describeDescription.trim() || undefined, + libraryIds: libraryId ? [libraryId] : undefined, + }), + }); + + if (!response.ok) { + const data = await response.json().catch(() => ({})); + throw new Error(data.error || 'Failed to add item'); + } + + const data = await response.json(); + const newItem = data.item as { id: string; name: string } | undefined; + if (newItem) { + setUploadedItem({ id: newItem.id, name: newItem.name }); + } + + setConceptBatchId(null); + setConceptOptions([]); + setSelectedConceptId(null); + setDescribeName(''); + setDescribeDescription(''); + setDescribeBrand(''); + setShowLibraryModal(true); + } catch (conceptErr) { + console.error('Failed to create item from concept:', conceptErr); + setConceptError( + conceptErr instanceof Error + ? conceptErr.message + : 'Failed to add item from concept' + ); + } finally { + setIsCreatingFromConcept(false); + } + }, [selectedConceptId, describeName, describeDescription, libraryId]); + // Handle library selection completion const handleLibrarySelectionComplete = useCallback( (_selectedLibraryIds: string[]) => { @@ -658,7 +799,241 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { }; }, [startCamera, stopCamera]); + useEffect(() => { + if (mode === 'describe') { + stopCamera(); + } else { + setConceptError(null); + setConceptOptions([]); + setSelectedConceptId(null); + } + }, [mode, stopCamera]); + + const renderDescribeContent = () => { + const helperText = + 'Share a clear name and a few details. We will keep your library safe and suggest watercolor illustrations for you to choose from.'; + + return ( + + + Describe Your Item + + + {helperText} + + + + setDescribeName(event.target.value)} + required + placeholder="e.g. DeWalt cordless drill" + fullWidth + /> + setDescribeBrand(event.target.value)} + placeholder="e.g. DeWalt, KitchenAid, Trek" + fullWidth + /> + setDescribeDescription(event.target.value)} + placeholder="Share notable features, color, size, or accessories" + fullWidth + multiline + minRows={3} + /> + + {conceptError && ( + setConceptError(null)}> + {conceptError} + + )} + + + + {conceptOptions.length > 0 && ( + + )} + + + + {isGeneratingConcepts && ( + + + + Creating watercolor options… + + + )} + + {conceptOptions.length > 0 && ( + + + Choose your favorite illustration + + + {conceptOptions.map((option, index) => { + const imageSrc = + option.watercolorThumbUrl || option.watercolorUrl; + const isSelected = option.conceptId === selectedConceptId; + const attributionRecord = option.sourceAttribution as { + attribution?: unknown; + } | null; + const attributionText = + attributionRecord && + typeof attributionRecord.attribution === 'string' + ? attributionRecord.attribution + : null; + const optionNames = ['one', 'two', 'three', 'four', 'five']; + const optionLabel = + index < optionNames.length + ? `Option ${optionNames[index]}` + : `Option ${index + 1}`; + const optionDescription = + option.generatedName || describeName || 'Illustration'; + return ( + setSelectedConceptId(option.conceptId)} + > + + + {optionLabel} + + + {optionDescription} + + + + + {attributionText && ( + + {attributionText} + + )} + + + ); + })} + + + {conceptOptions.length > 0 && conceptOptions.length < 3 && ( + + Only {conceptOptions.length} illustration + {conceptOptions.length === 1 ? '' : 's'} found so far. Try + adding more detail or regenerate for new options. + + )} + + + + + We will store your choice, clean up unused drafts, and route you + to library selection. + + + + )} + + ); + }; + const renderContent = () => { + if (mode === 'describe') { + return renderDescribeContent(); + } + console.log('🎯 Current state:', state); switch (state) { case 'permission': @@ -1159,8 +1534,8 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { sx={{ px: 1, pt: 1, - pb: 2, - height: '100vh', + pb: 4, + minHeight: '100vh', display: 'flex', flexDirection: 'column', bgcolor: 'background.default', @@ -1179,14 +1554,31 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { + + + + + {/* Content */} {renderContent()} diff --git a/src/lib/__tests__/prompt-safety-service.test.ts b/src/lib/__tests__/prompt-safety-service.test.ts new file mode 100644 index 0000000..aa9491b --- /dev/null +++ b/src/lib/__tests__/prompt-safety-service.test.ts @@ -0,0 +1,28 @@ +import { describe, it, expect, beforeEach } from 'vitest'; + +import { PromptSafetyService } from '../prompt-safety-service'; + +describe('PromptSafetyService', () => { + beforeEach(() => { + delete process.env.GOOGLE_AI_API_KEY; + }); + + it('rejects descriptions with prohibited content', async () => { + const result = await PromptSafetyService.validatePrompt({ + name: 'Handgun', + description: 'Compact handgun with magazine', + }); + + expect(result.allowed).toBe(false); + expect(result.reason).toContain('prohibited'); + }); + + it('allows safe descriptions when no issues found', async () => { + const result = await PromptSafetyService.validatePrompt({ + name: 'Cordless drill', + description: '18V drill with two batteries', + }); + + expect(result.allowed).toBe(true); + }); +}); diff --git a/src/lib/item-concept-service.ts b/src/lib/item-concept-service.ts new file mode 100644 index 0000000..15adca3 --- /dev/null +++ b/src/lib/item-concept-service.ts @@ -0,0 +1,494 @@ +import crypto from 'crypto'; + +import { GoogleGenAI } from '@google/genai'; +import { + ConceptSourceType, + ItemConcept, + ItemConceptStatus, + Prisma, +} from '@prisma/client'; + +import { db } from './db'; +import { PromptSafetyService } from './prompt-safety-service'; +import { WatercolorService } from './watercolor-service'; + +interface ConceptInput { + userId: string; + name: string; + description?: string | null; + brand?: string | null; + count?: number; +} + +interface OpenverseImage { + url: string; + thumbnail: string; + title?: string; + creator?: string; + license?: string; + provider?: string; + attribution?: string; +} + +interface ConceptOptionSummary { + id: string; + watercolorUrl: string | null; + watercolorThumbUrl: string | null; + sourceType: ConceptSourceType; + generatedName?: string | null; + sourceAttribution?: Record | null; +} + +const DEFAULT_CONCEPT_COUNT = 3; +const CONCEPT_TTL_MS = 1000 * 60 * 60 * 24 * 2; // 48 hours + +function isPrismaMissingTableError(error: unknown): boolean { + return ( + typeof error === 'object' && + !!error && + 'code' in error && + (error as { code?: string }).code === 'P2021' + ); +} + +export class ItemConceptService { + private watercolorService?: WatercolorService; + private genAI?: GoogleGenAI; + + constructor(private prisma = db) { + const apiKey = process.env.GOOGLE_AI_API_KEY; + if (apiKey) { + this.genAI = new GoogleGenAI({ apiKey }); + } + } + + private getWatercolorService(): WatercolorService { + if (!this.watercolorService) { + this.watercolorService = new WatercolorService(); + } + return this.watercolorService; + } + + async generateConcepts( + input: ConceptInput + ): Promise<{ batchId: string; options: ConceptOptionSummary[] }> { + const count = input.count ?? DEFAULT_CONCEPT_COUNT; + + // Safety check + const safety = await PromptSafetyService.validatePrompt({ + name: input.name, + description: input.description ?? null, + brand: input.brand ?? null, + }); + if (!safety.allowed) { + throw new Error(safety.reason || 'Description failed safety checks'); + } + + const batchId = crypto.randomUUID(); + const expiresAt = new Date(Date.now() + CONCEPT_TTL_MS); + const summaries: ConceptOptionSummary[] = []; + + // Step 1: Try to source from Openverse (public domain/licensed) + try { + const openverseImages = await this.fetchOpenverseImages(input, count); + for (const image of openverseImages) { + if (summaries.length >= count) break; + const concept = await this.createConceptFromImage({ + batchId, + expiresAt, + input, + sourceType: ConceptSourceType.OPENVERSE, + image, + }); + if (concept) { + summaries.push(concept); + } + } + } catch (error) { + console.warn( + 'Openverse fetch failed, falling back to generative only:', + error + ); + } + + // Step 2: Top up with generative imagery if needed + while (summaries.length < count) { + const concept = await this.createConceptFromGeneration({ + batchId, + expiresAt, + input, + }); + if (!concept) { + break; + } + summaries.push(concept); + } + + if (summaries.length === 0) { + if (!this.genAI) { + throw new Error( + 'No matching library photos found. Configure GOOGLE_AI_API_KEY to enable watercolor suggestions.' + ); + } + throw new Error('Unable to generate concepts at this time'); + } + + return { batchId, options: summaries }; + } + + async consumeConcept( + conceptId: string, + userId: string + ): Promise { + const concept = await this.prisma.itemConcept.findFirst({ + where: { + id: conceptId, + ownerId: userId, + status: ItemConceptStatus.READY, + expiresAt: { gt: new Date() }, + }, + }); + + if (!concept) { + return null; + } + + return this.prisma.itemConcept.update({ + where: { id: conceptId }, + data: { + status: ItemConceptStatus.CONSUMED, + consumedAt: new Date(), + }, + }); + } + + async discardBatch(batchId: string, userId: string): Promise { + await this.prisma.itemConcept.updateMany({ + where: { + batchId, + ownerId: userId, + status: { + in: [ItemConceptStatus.PENDING, ItemConceptStatus.READY], + }, + }, + data: { + status: ItemConceptStatus.DISCARDED, + }, + }); + } + + private async createConceptFromImage({ + batchId, + expiresAt, + input, + sourceType, + image, + }: { + batchId: string; + expiresAt: Date; + input: ConceptInput; + sourceType: ConceptSourceType; + image: OpenverseImage; + }): Promise { + let conceptRecord: ItemConcept | null = null; + try { + const response = await fetch(image.url, { + headers: { + 'User-Agent': 'StuffLibrary/1.0 (concept-generation)', + }, + }); + if (!response.ok) { + throw new Error('Failed to fetch source image'); + } + const arrayBuffer = await response.arrayBuffer(); + const imageBuffer = Buffer.from(arrayBuffer); + const contentType = response.headers.get('content-type') || 'image/jpeg'; + + conceptRecord = await this.prisma.itemConcept.create({ + data: { + batchId, + ownerId: input.userId, + inputName: input.name, + inputDescription: input.description ?? null, + inputBrand: input.brand ?? null, + generatedName: image.title ?? null, + sourceType, + sourceAttribution: image.attribution + ? ({ + title: image.title, + creator: image.creator, + provider: image.provider, + license: image.license, + attribution: image.attribution, + sourceUrl: image.url, + } as Prisma.InputJsonValue) + : Prisma.JsonNull, + status: ItemConceptStatus.PENDING, + expiresAt, + }, + }); + + const watercolor = await this.getWatercolorService().renderWatercolor({ + itemId: `concept-${conceptRecord.id}`, + originalImageBuffer: imageBuffer, + originalImageName: `concept-${conceptRecord.id}.jpg`, + mimeType: contentType, + }); + + const updated = await this.prisma.itemConcept.update({ + where: { id: conceptRecord.id }, + data: { + watercolorUrl: watercolor.watercolorUrl, + watercolorThumbUrl: watercolor.watercolorThumbUrl, + originalImageUrl: watercolor.originalUrl, + generatedDetails: image.title + ? ({ title: image.title } as Prisma.InputJsonValue) + : Prisma.JsonNull, + status: ItemConceptStatus.READY, + }, + }); + + return { + id: updated.id, + watercolorUrl: updated.watercolorUrl, + watercolorThumbUrl: updated.watercolorThumbUrl, + sourceType: updated.sourceType, + generatedName: updated.generatedName, + sourceAttribution: updated.sourceAttribution + ? (updated.sourceAttribution as Record) + : null, + }; + } catch (error) { + console.error('Failed to create concept from sourced image:', error); + if (isPrismaMissingTableError(error)) { + throw new Error('ITEM_CONCEPTS_TABLE_MISSING'); + } + if (conceptRecord) { + try { + await this.prisma.itemConcept.update({ + where: { id: conceptRecord.id }, + data: { + status: ItemConceptStatus.DISCARDED, + }, + }); + } catch (updateError) { + console.error('Failed to mark concept as discarded:', updateError); + } + } + return null; + } + } + + private async createConceptFromGeneration({ + batchId, + expiresAt, + input, + }: { + batchId: string; + expiresAt: Date; + input: ConceptInput; + }): Promise { + if (!this.genAI) { + console.warn('Generative AI unavailable; skipping concept generation'); + return null; + } + + const prompt = this.buildGenerativePrompt(input); + + let conceptRecord: ItemConcept | null = null; + try { + conceptRecord = await this.prisma.itemConcept.create({ + data: { + batchId, + ownerId: input.userId, + inputName: input.name, + inputDescription: input.description ?? null, + inputBrand: input.brand ?? null, + sourceType: ConceptSourceType.GENERATIVE, + status: ItemConceptStatus.PENDING, + expiresAt, + }, + }); + + const response = await this.genAI.models.generateContent({ + model: 'gemini-2.5-flash-image-preview', + contents: [ + { + role: 'user', + parts: [ + { + text: prompt, + }, + ], + }, + ], + }); + + const imagePart = response.candidates?.[0]?.content?.parts?.find( + (part) => part.inlineData?.data + ); + + if (!imagePart?.inlineData?.data) { + throw new Error('No image returned from generative model'); + } + + const buffer = Buffer.from(imagePart.inlineData.data, 'base64'); + + const watercolor = await this.getWatercolorService().renderWatercolor({ + itemId: `concept-${conceptRecord.id}`, + originalImageBuffer: buffer, + originalImageName: `concept-${conceptRecord.id}.webp`, + mimeType: imagePart.inlineData.mimeType || 'image/webp', + }); + + const updated = await this.prisma.itemConcept.update({ + where: { id: conceptRecord.id }, + data: { + watercolorUrl: watercolor.watercolorUrl, + watercolorThumbUrl: watercolor.watercolorThumbUrl, + originalImageUrl: watercolor.originalUrl, + generatedDetails: { + prompt, + flags: watercolor.flags, + } as Prisma.InputJsonValue, + status: ItemConceptStatus.READY, + }, + }); + + return { + id: updated.id, + watercolorUrl: updated.watercolorUrl, + watercolorThumbUrl: updated.watercolorThumbUrl, + sourceType: updated.sourceType, + generatedName: updated.generatedName, + sourceAttribution: null, + }; + } catch (error) { + console.error('Failed to generate concept from prompt:', error); + if (isPrismaMissingTableError(error)) { + throw new Error('ITEM_CONCEPTS_TABLE_MISSING'); + } + if (conceptRecord) { + try { + await this.prisma.itemConcept.update({ + where: { id: conceptRecord.id }, + data: { + status: ItemConceptStatus.DISCARDED, + }, + }); + } catch (updateError) { + console.error('Failed to mark concept as discarded:', updateError); + } + } + return null; + } + } + + private buildGenerativePrompt(input: ConceptInput): string { + const parts = [ + 'Create a single product watercolor illustration with the signature StuffLibrary style.', + 'Style: warm cream background (#F9F5EB), subtle ink outlines, soft watercolor washes, no text anywhere.', + 'Composition: center the item with generous margin, no shadows on the ground, no background clutter.', + ]; + + if (input.brand) { + parts.push(`Brand context: ${input.brand}.`); + } + + parts.push(`Item description: ${input.name}.`); + + if (input.description) { + parts.push(`Additional details: ${input.description}.`); + } + + parts.push( + 'Constraints: absolutely no people, faces, hands, body parts, weapons, or unsafe objects. Maintain realistic proportions.' + ); + + return parts.join('\n'); + } + + private async fetchOpenverseImages( + input: ConceptInput, + limit: number + ): Promise { + if (limit <= 0) { + return []; + } + + const terms = [input.brand ?? '', input.name, input.description ?? ''] + .filter(Boolean) + .join(' ') + .trim(); + + if (!terms) { + return []; + } + + const params = new URLSearchParams({ + q: terms, + license_type: 'commercial', + content_filter: 'high', + page_size: limit.toString(), + }); + params.set( + 'fields', + [ + 'url', + 'thumbnail', + 'title', + 'creator', + 'license', + 'provider', + 'attribution', + ].join(',') + ); + params.set('aspect_ratio', 'square'); + + const url = `https://api.openverse.engineering/v1/images/?${params.toString()}`; + + const response = await fetch(url, { + headers: { + 'User-Agent': 'StuffLibrary/1.0 (+https://stufflibrary.com)', + Accept: 'application/json', + }, + }); + + if (!response.ok) { + throw new Error(`Openverse API error: ${response.status}`); + } + + const data = await response.json(); + if (!data?.results || !Array.isArray(data.results)) { + return []; + } + + type OpenverseApiResult = { + url?: string; + thumbnail?: string; + title?: string; + creator?: string; + license?: string; + provider?: string; + attribution?: string; + }; + + return (data.results as OpenverseApiResult[]) + .map((result) => { + if (!result?.url) { + return null; + } + return { + url: result.url as string, + thumbnail: (result.thumbnail ?? result.url) as string, + title: result.title as string | undefined, + creator: result.creator as string | undefined, + license: result.license as string | undefined, + provider: result.provider as string | undefined, + attribution: result.attribution as string | undefined, + }; + }) + .filter(Boolean) + .slice(0, limit) as OpenverseImage[]; + } +} diff --git a/src/lib/prompt-safety-service.ts b/src/lib/prompt-safety-service.ts new file mode 100644 index 0000000..060b186 --- /dev/null +++ b/src/lib/prompt-safety-service.ts @@ -0,0 +1,113 @@ +import { GoogleGenAI } from '@google/genai'; + +const DISALLOWED_KEYWORDS = [ + /bomb/i, + /weapon/i, + /gun/i, + /explosive/i, + /drug/i, + /narcotic/i, + /meth/i, + /cocaine/i, + /heroin/i, + /fentanyl/i, + /grenade/i, + /machete/i, + /dead body/i, + /corpse/i, + /blood/i, + /gore/i, +]; + +const PROHIBITED_RESPONSE = 'BLOCKED'; + +export interface PromptSafetyInput { + name?: string | null; + description?: string | null; + brand?: string | null; +} + +export interface PromptSafetyResult { + allowed: boolean; + reason?: string; +} + +export class PromptSafetyService { + static async validatePrompt( + input: PromptSafetyInput + ): Promise { + const combined = [input.name, input.brand, input.description] + .filter(Boolean) + .join(' ') + .trim(); + + if (!combined) { + return { allowed: false, reason: 'Description is required' }; + } + + for (const keyword of DISALLOWED_KEYWORDS) { + if (keyword.test(combined)) { + return { + allowed: false, + reason: 'Description contains prohibited content', + }; + } + } + + const apiKey = process.env.GOOGLE_AI_API_KEY; + if (!apiKey) { + // If we cannot call advanced safety checks, fall back to keyword filter + return { allowed: true }; + } + + try { + const client = new GoogleGenAI({ apiKey }); + const response = await client.models.generateContent({ + model: 'gemini-2.0-flash', + contents: [ + { + role: 'user', + parts: [ + { + text: `You are a trust and safety classifier for a community item-sharing app. +Classify the following description. Respond with a single word: "SAFE" if the text is acceptable, or "${PROHIBITED_RESPONSE}" if it references anything illegal, violent, hateful, adult, or otherwise inappropriate for a community lending library. + +Text: +""" +${combined} +"""`, + }, + ], + }, + ], + }); + + const raw = + response.candidates?.[0]?.content?.parts?.[0]?.text + ?.trim() + .toUpperCase() || ''; + + if (!raw) { + return { + allowed: false, + reason: 'Safety classification failed', + }; + } + + if (raw.includes(PROHIBITED_RESPONSE) || raw.includes('UNSAFE')) { + return { + allowed: false, + reason: 'Description flagged by safety classifier', + }; + } + + return { allowed: true }; + } catch (error) { + console.error('Prompt safety classification failed:', error); + return { + allowed: false, + reason: 'Safety check error, please refine description', + }; + } + } +}