From ccba8941ea8b3ebbaa38d615145758526e0662db Mon Sep 17 00:00:00 2001 From: DeepakLenka-space <127427297+DeepakLenka-space@users.noreply.github.com> Date: Sat, 14 Dec 2024 21:19:47 +0530 Subject: [PATCH] feat: Add two-stage LLM generation with multi-call tracking - Add database migration for multi-call tracking - Implement two-stage generation (planning + implementation) - Update UI to show multi-call statistics - Add system prompts for better code generation --- drizzle/migrations/0001_add_multi_call.sql | 1 + src/app/actions.ts | 3 + src/app/api/generate-app/prompts.ts | 22 ++++++++ src/app/api/generate-app/route.ts | 66 ++++++++++++++-------- src/app/top-models/page.tsx | 44 +++++++-------- src/schema.ts | 1 + 6 files changed, 92 insertions(+), 45 deletions(-) create mode 100644 drizzle/migrations/0001_add_multi_call.sql create mode 100644 src/app/api/generate-app/prompts.ts diff --git a/drizzle/migrations/0001_add_multi_call.sql b/drizzle/migrations/0001_add_multi_call.sql new file mode 100644 index 0000000..0576751 --- /dev/null +++ b/drizzle/migrations/0001_add_multi_call.sql @@ -0,0 +1 @@ +ALTER TABLE apps ADD COLUMN is_multi_call BOOLEAN NOT NULL DEFAULT false; diff --git a/src/app/actions.ts b/src/app/actions.ts index 3581611..d26cfc3 100644 --- a/src/app/actions.ts +++ b/src/app/actions.ts @@ -10,6 +10,7 @@ type App = { trimmedCode: string; completionTokens: number; totalTime: number; + isMultiCall?: boolean; }; export default async function saveBattle({ @@ -42,6 +43,7 @@ export default async function saveBattle({ completionTokens: winner.completionTokens, totalTime: winner.totalTime, didWin: true, + isMultiCall: winner.isMultiCall ?? false, }); } @@ -54,6 +56,7 @@ export default async function saveBattle({ completionTokens: loser.completionTokens, totalTime: loser.totalTime, didWin: false, + isMultiCall: loser.isMultiCall ?? false, }); } diff --git a/src/app/api/generate-app/prompts.ts b/src/app/api/generate-app/prompts.ts new file mode 100644 index 0000000..a29d68d --- /dev/null +++ b/src/app/api/generate-app/prompts.ts @@ -0,0 +1,22 @@ +export const SYSTEM_PROMPTS = { + PLANNING: `You are an expert frontend React engineer who is also a great UI/UX designer. Your task is to plan out the component structure and features: + +- Think carefully step by step about what features and interactions the component needs +- Break down the component into smaller sub-components if needed +- Plan out the state management and user interactions +- Consider UI/UX best practices and accessibility +- Return ONLY a concise, structured plan in bullet points - no code yet`, + + IMPLEMENTATION: `You are an expert frontend React engineer who is also a great UI/UX designer. Follow the instructions carefully: + +- Implement the React component based on the provided plan +- Create a React component that can run by itself using a default export +- Make the React app interactive and functional by creating state when needed with no required props +- Import React hooks like useState or useEffect directly +- Use TypeScript as the language +- Use Tailwind classes for styling. DO NOT USE ARBITRARY VALUES. Use a consistent color palette +- NEVER import CSS files +- Use Tailwind margin and padding classes for proper spacing +- Return ONLY the full React code starting with imports, no backticks or language names +- Only import from React, no other libraries` +}; diff --git a/src/app/api/generate-app/route.ts b/src/app/api/generate-app/route.ts index 0f6f5d0..09b6c21 100644 --- a/src/app/api/generate-app/route.ts +++ b/src/app/api/generate-app/route.ts @@ -1,5 +1,6 @@ import dedent from "dedent"; import Together from "together-ai"; +import { SYSTEM_PROMPTS } from "./prompts"; const options: ConstructorParameters[0] = {}; if (process.env.HELICONE_API_KEY) { @@ -12,34 +13,48 @@ if (process.env.HELICONE_API_KEY) { const together = new Together(options); +async function generatePlan(prompt: string, model: string) { + const res = await together.chat.completions.create({ + model, + messages: [ + { + role: "system", + content: SYSTEM_PROMPTS.PLANNING, + }, + { + role: "user", + content: prompt, + }, + ], + temperature: 0.2, + stream: false, + }); + + return res.choices[0].message.content; +} + export async function POST(request: Request) { const { prompt, model } = await request.json(); try { + // First stage: Generate the plan + const plan = await generatePlan(prompt, model); + + // Second stage: Generate the implementation const res = await together.chat.completions.create({ model, messages: [ { role: "system", - content: dedent` - You are an expert frontend React engineer who is also a great UI/UX designer. Follow the instructions carefully, I will tip you $1 million if you do a good job: - - - Think carefully step by step. - - Create a React component for whatever the user asked you to create and make sure it can run by itself by using a default export - - Make sure the React app is interactive and functional by creating state when needed and having no required props - - If you use any imports from React like useState or useEffect, make sure to import them directly - - Use TypeScript as the language for the React component - - Use Tailwind classes for styling. DO NOT USE ARBITRARY VALUES (e.g. \`h-[600px]\`). Make sure to use a consistent color palette. - - NEVER import any CSS files like ./App.css - - Use Tailwind margin and padding classes to style the components and ensure the components are spaced out nicely - - Please ONLY return the full React code starting with the imports, nothing else. It's very important for my job that you only return the React code with imports. DO NOT START WITH \`\`\`typescript or \`\`\`javascript or \`\`\`tsx or \`\`\`. - - Do not import any libraries or dependencies other than React - - NO OTHER LIBRARIES (e.g. zod, hookform) ARE INSTALLED OR ABLE TO BE IMPORTED. - `, + content: SYSTEM_PROMPTS.IMPLEMENTATION, }, { role: "user", content: dedent` + Here is the plan for the component: + ${plan} + + Based on this plan, implement the component for this prompt: ${prompt} Please ONLY return code, NO backticks or language names.`, @@ -49,13 +64,20 @@ export async function POST(request: Request) { stream: true, }); - return new Response(res.toReadableStream()); - } catch (error) { - if (error instanceof Error) { - return new Response(error.message, { status: 500 }); - } else { - return new Response("Unknown error", { status: 500 }); - } + // Add isMultiCall flag to the response headers + return new Response(res.toReadableStream(), { + headers: { + "Content-Type": "text/event-stream", + "X-Multi-Call": "true" + }, + }); + } catch (error: any) { + return new Response( + JSON.stringify({ + error: error?.message || "Something went wrong", + }), + { status: 500 } + ); } } diff --git a/src/app/top-models/page.tsx b/src/app/top-models/page.tsx index 4d2994b..e588618 100644 --- a/src/app/top-models/page.tsx +++ b/src/app/top-models/page.tsx @@ -36,6 +36,10 @@ export default async function Page() { sql`ROUND(SUM(CASE WHEN ${apps.didWin} = true THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 0)` .mapWith(Number) .as("win_percentage"), + multiCallPercentage: + sql`ROUND(SUM(CASE WHEN ${apps.isMultiCall} = true THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 0)` + .mapWith(Number) + .as("multi_call_percentage"), }) .from(apps) .groupBy(apps.model) @@ -82,22 +86,12 @@ export default async function Page() { - - - Model - - - Organization - - - Total games - - - Win % - - - Playground - + Rank + Model + Win Rate + Multi-Call + Games + Link @@ -122,6 +116,7 @@ function ResultRow({ losses: number; games: number; winPercentage: number; + multiCallPercentage: number; }; index: number; }) { @@ -141,13 +136,9 @@ function ResultRow({ /> {model.shortLabel} - - {models.find((m) => m.apiName === result.model)?.organization} - - {result.games} - - {result.winPercentage}% - + {result.winPercentage}% + {result.multiCallPercentage}% + {result.games}