From 9e5532b797123c0b790fb3fd7d02e22c7eccd7e3 Mon Sep 17 00:00:00 2001 From: mcull Date: Mon, 29 Sep 2025 19:42:40 -0700 Subject: [PATCH 1/2] feat(camera): add name hint for failed item recognition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #310 - "when the camera can't recognize an item, offer the ability to give it a hint" ## Problem Camera item recognition was failing on certain items (gas cans, radial saws) with no way for users to provide additional context to help the AI identify the item. ## Solution ### 1. Enhanced Error UI with Hint Option When recognition fails, users now see: - The captured photo (so they know what image is being analyzed) - Two clear options: - **"Take New Photo"** - Start over with a fresh capture - **"Try Again with Hint"** - Provide a name hint for the same photo ### 2. Name Hint Input Flow Clicking "Try Again with Hint" shows: - Text input: "What is this item?" - Placeholder examples: "e.g., gas can, radial saw, ladder..." - Clear explanation: "Give us a hint about what this item is, and we'll try again with this photo" - Cancel and "Try with Hint" buttons ### 3. Backend: analyze-item API Enhancement Updated `/api/analyze-item` to accept optional `nameHint` parameter: - Passes hint to OpenAI with: "USER HINT: The user indicated this might be a '{hint}'. Use this hint to help identify the object in the image, but still verify it matches what you see." - AI uses hint as context while still applying safety restrictions ### 4. Smart Retry Logic `retryWithHint()` function: - Reuses the already-captured photo (no new photo needed) - Sends photo + hint to analysis API - Creates draft item if not already created - Updates item with recognized name/description - Triggers watercolor generation on success - Shows improved error if hint still doesn't work ## Distinguishing Recognition Failures The API already distinguishes between: - **Prohibited items**: Returns `prohibited: true` with `prohibitionReason` - **Recognition failures**: Returns `recognized: false` (genuine can't identify) Hint feature only shows for genuine recognition failures, not prohibited items. ## User Flow 1. User takes photo of gas can 2. Recognition fails: "Could not identify the item" 3. User sees photo + two buttons 4. User clicks "Try Again with Hint" 5. User types "gas can" 6. System re-analyzes with hint 7. Success! Item recognized as "Gas Can" ## Testing Needed - Gas can recognition with hint "gas can" - Radial saw recognition with hint "radial saw" - Verify hint doesn't bypass safety restrictions - Test "Take New Photo" still works correctly - Test canceling hint input returns to error state 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/app/api/analyze-item/route.ts | 3 +- src/components/AddItemClient.tsx | 248 ++++++++++++++++++++++++++++-- 2 files changed, 241 insertions(+), 10 deletions(-) diff --git a/src/app/api/analyze-item/route.ts b/src/app/api/analyze-item/route.ts index 7936a46..1a410cf 100644 --- a/src/app/api/analyze-item/route.ts +++ b/src/app/api/analyze-item/route.ts @@ -44,6 +44,7 @@ export async function POST(request: NextRequest) { const formData = await request.formData(); const image = formData.get('image') as File; + const nameHint = formData.get('nameHint') as string | null; if (!image) { return NextResponse.json({ error: 'No image provided' }, { status: 400 }); @@ -65,7 +66,7 @@ export async function POST(request: NextRequest) { { type: 'text', text: `Analyze this image and identify the main object. This is for a community sharing library that only accepts normal household goods. - +${nameHint ? `\n USER HINT: The user indicated this might be a "${nameHint}". Use this hint to help identify the object in the image, but still verify it matches what you see.\n` : ''} IMPORTANT CONTENT RESTRICTIONS: - REJECT items that are: illegal, unsafe, inappropriate, nudity, weapons, firearms, alcohol, tobacco, drugs, age-restricted items, or anything requiring ID verification - REJECT items that appear dangerous, hazardous, or could cause harm diff --git a/src/components/AddItemClient.tsx b/src/components/AddItemClient.tsx index 0b72d09..3e06356 100644 --- a/src/components/AddItemClient.tsx +++ b/src/components/AddItemClient.tsx @@ -16,6 +16,7 @@ import { CircularProgress, Alert, IconButton, + TextField, } from '@mui/material'; import { useRouter } from 'next/navigation'; import { useRef, useState, useCallback, useEffect } from 'react'; @@ -119,6 +120,8 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { const [isGeneratingWatercolor, setIsGeneratingWatercolor] = useState(false); const [watercolorUrl, setWatercolorUrl] = useState(null); const [showWatercolor, setShowWatercolor] = useState(false); + const [showHintInput, setShowHintInput] = useState(false); + const [nameHint, setNameHint] = useState(''); // Request camera permission and start stream const startCamera = useCallback(async () => { @@ -304,6 +307,10 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { // Step 2: Get AI analysis const analysisFormData = new FormData(); analysisFormData.append('image', blob, 'capture.jpg'); + // Include hint if available (empty string means no hint) + if (nameHint) { + analysisFormData.append('nameHint', nameHint); + } const analysisResponse = await fetch('/api/analyze-item', { method: 'POST', @@ -399,7 +406,7 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { 'image/jpeg', 0.8 ); - }, [shouldMirrorCamera]); + }, [shouldMirrorCamera, nameHint]); // Add item and navigate to metadata page const addItem = useCallback(async () => { @@ -459,9 +466,146 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { setIsGeneratingWatercolor(false); setWatercolorUrl(null); setShowWatercolor(false); + setShowHintInput(false); + setNameHint(''); setState('streaming'); }, []); + // Retry analysis with hint on the existing captured image + const retryWithHint = useCallback(async () => { + if (!capturedImageUrl || !nameHint.trim()) { + return; + } + + console.log('🔄 Retrying analysis with hint:', nameHint); + setIsAnalyzing(true); + setState('capturing'); + setError(null); + + try { + // Convert data URL back to blob + const response = await fetch(capturedImageUrl); + const blob = await response.blob(); + + // If we don't have a draft item yet, create one + let itemId = draftItemId; + if (!itemId) { + const draftFormData = new FormData(); + draftFormData.append('image', blob, 'capture.jpg'); + + const draftResponse = await fetch('/api/items/create-draft', { + method: 'POST', + body: draftFormData, + }); + + if (!draftResponse.ok) { + throw new Error('Failed to create draft item'); + } + + const draftResult = await draftResponse.json(); + itemId = draftResult.itemId; + setDraftItemId(itemId); + console.log('✅ Draft item created:', itemId); + } + + // Analyze with hint + const analysisFormData = new FormData(); + analysisFormData.append('image', blob, 'capture.jpg'); + analysisFormData.append('nameHint', nameHint.trim()); + + const analysisResponse = await fetch('/api/analyze-item', { + method: 'POST', + body: analysisFormData, + }); + + if (!analysisResponse.ok) { + throw new Error('Analysis failed'); + } + + const analysisResult = await analysisResponse.json(); + console.log('🔍 Analysis result with hint:', analysisResult); + + if (analysisResult.recognized && analysisResult.name) { + // Update item with analysis results + const updateAnalysisResponse = await fetch( + '/api/items/update-analysis', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + itemId: itemId, + name: analysisResult.name, + description: analysisResult.description, + category: analysisResult.category, + }), + } + ); + + if (updateAnalysisResponse.ok) { + console.log('✅ Item updated with analysis'); + } + + // Set recognition results for UI + setRecognitionResult({ + name: analysisResult.name, + description: analysisResult.description || '', + confidence: analysisResult.confidence || 0, + category: analysisResult.category || 'other', + }); + + setUploadedItem({ + id: itemId, + name: analysisResult.name, + }); + + setIsAnalyzing(false); + setState('recognized'); + setShowHintInput(false); + + // Start watercolor generation + setIsGeneratingWatercolor(true); + + fetch('/api/items/render-watercolor', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ itemId: itemId }), + }) + .then(async (watercolorResponse) => { + if (watercolorResponse.ok) { + const watercolorResult = await watercolorResponse.json(); + console.log( + '🎨 Watercolor generated:', + watercolorResult.watercolorUrl + ); + setWatercolorUrl(watercolorResult.watercolorUrl); + setTimeout(() => { + setShowWatercolor(true); + setIsGeneratingWatercolor(false); + }, 100); + } else { + console.error('❌ Watercolor generation failed'); + setIsGeneratingWatercolor(false); + } + }) + .catch((error) => { + console.error('❌ Watercolor generation failed:', error); + setIsGeneratingWatercolor(false); + }); + } else { + setError( + 'Still could not identify the item with that hint. Please try a new photo or different hint.' + ); + setState('error'); + setIsAnalyzing(false); + } + } catch (err) { + console.error('❌ Error retrying with hint:', err); + setError('Failed to analyze with hint. Please try again.'); + setState('error'); + setIsAnalyzing(false); + } + }, [capturedImageUrl, nameHint, draftItemId]); + // Handle library selection completion const handleLibrarySelectionComplete = useCallback( (_selectedLibraryIds: string[]) => { @@ -900,7 +1044,7 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { case 'error': return ( - + Oops! @@ -908,13 +1052,99 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { {error} - + + {/* Show captured image if available */} + {capturedImageUrl && ( + + Captured item + + )} + + {/* Show hint input or hint input button */} + {showHintInput ? ( + + + Give us a hint about what this item is, and we'll try + again with this photo. + + setNameHint(e.target.value)} + onKeyPress={(e) => { + if (e.key === 'Enter' && nameHint.trim()) { + retryWithHint(); + } + }} + autoFocus + sx={{ mb: 2 }} + /> + + + + + + ) : ( + + + {capturedImageUrl && ( + + )} + + )} ); From eb93cd5be47f1e369e199b191dae511c8d027552 Mon Sep 17 00:00:00 2001 From: mcull Date: Mon, 29 Sep 2025 20:02:17 -0700 Subject: [PATCH 2/2] fix(camera): resolve TypeScript error in retryWithHint function - Add explicit type annotation for itemId variable - Cast draftResult.itemId to string type - Fixes TS2322: Type 'string | null' is not assignable to type 'string' --- src/components/AddItemClient.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/AddItemClient.tsx b/src/components/AddItemClient.tsx index 3e06356..ef489f9 100644 --- a/src/components/AddItemClient.tsx +++ b/src/components/AddItemClient.tsx @@ -488,7 +488,7 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { const blob = await response.blob(); // If we don't have a draft item yet, create one - let itemId = draftItemId; + let itemId: string = draftItemId || ''; if (!itemId) { const draftFormData = new FormData(); draftFormData.append('image', blob, 'capture.jpg'); @@ -503,7 +503,7 @@ export function AddItemClient({ libraryId }: AddItemClientProps) { } const draftResult = await draftResponse.json(); - itemId = draftResult.itemId; + itemId = draftResult.itemId as string; setDraftItemId(itemId); console.log('✅ Draft item created:', itemId); }