From 650869b8582d0f9e3bcb6af6cab3d91d4c2a5800 Mon Sep 17 00:00:00 2001 From: naaa760 Date: Fri, 14 Nov 2025 16:08:32 +0530 Subject: [PATCH] feat: add Vercel AI Gateway provider with pricing support --- packages/app/server/src/env.ts | 1 + .../server/src/providers/ProviderFactory.ts | 34 ++++ .../app/server/src/providers/ProviderType.ts | 1 + .../src/providers/VercelAIGatewayProvider.ts | 187 ++++++++++++++++++ .../react/src/hooks/useEchoModelProviders.ts | 6 + packages/sdk/ts/src/providers/index.ts | 1 + .../sdk/ts/src/providers/vercel-ai-gateway.ts | 27 +++ 7 files changed, 257 insertions(+) create mode 100644 packages/app/server/src/providers/VercelAIGatewayProvider.ts create mode 100644 packages/sdk/ts/src/providers/vercel-ai-gateway.ts diff --git a/packages/app/server/src/env.ts b/packages/app/server/src/env.ts index cc993ae07..81845e56e 100644 --- a/packages/app/server/src/env.ts +++ b/packages/app/server/src/env.ts @@ -45,6 +45,7 @@ export const env = createEnv({ GROQ_API_KEY: z.string().optional(), XAI_API_KEY: z.string().optional(), OPENROUTER_API_KEY: z.string().optional(), + VERCEL_AI_GATEWAY_API_KEY: z.string().optional(), TAVILY_API_KEY: z.string().optional(), E2B_API_KEY: z.string().optional(), GOOGLE_SERVICE_ACCOUNT_KEY_ENCODED: z.string().optional(), diff --git a/packages/app/server/src/providers/ProviderFactory.ts b/packages/app/server/src/providers/ProviderFactory.ts index 8bc01e719..4730c892a 100644 --- a/packages/app/server/src/providers/ProviderFactory.ts +++ b/packages/app/server/src/providers/ProviderFactory.ts @@ -26,6 +26,7 @@ import { VertexAIProvider, PROXY_PASSTHROUGH_ONLY_MODEL as VertexAIProxyPassthroughOnlyModel, } from './VertexAIProvider'; +import { VercelAIGatewayProvider } from './VercelAIGatewayProvider'; /** * Creates model-to-provider mapping from the model_prices_and_context_window.json file. @@ -58,6 +59,11 @@ const createChatModelToProviderMapping = (): Record => { case 'Xai': mapping[modelConfig.model_id] = ProviderType.XAI; break; + case 'VercelAIGateway': + case 'Vercel AI Gateway': + case 'Vercel': + mapping[modelConfig.model_id] = ProviderType.VERCEL_AI_GATEWAY; + break; // Add other providers as needed default: // Skip models with unsupported providers @@ -165,6 +171,32 @@ export const getProvider = ( type = ProviderType.GEMINI_GPT; } + if ( + (completionPath.includes('audio/transcriptions') || + completionPath.includes('audio/speech')) && + model.includes('/') + ) { + type = ProviderType.VERCEL_AI_GATEWAY; + } + + if (type === undefined && model.includes('/')) { + const [providerPrefix] = model.split('/'); + const supportedPrefixes = [ + 'openai', + 'anthropic', + 'google', + 'gemini', + 'groq', + 'xai', + 'cohere', + 'mistral', + 'perplexity', + ]; + if (supportedPrefixes.includes(providerPrefix.toLowerCase())) { + type = ProviderType.VERCEL_AI_GATEWAY; + } + } + switch (type) { case ProviderType.GPT: return new GPTProvider(stream, model); @@ -192,6 +224,8 @@ export const getProvider = ( return new GroqProvider(stream, model); case ProviderType.XAI: return new XAIProvider(stream, model); + case ProviderType.VERCEL_AI_GATEWAY: + return new VercelAIGatewayProvider(stream, model); default: throw new Error(`Unknown provider type: ${type}`); } diff --git a/packages/app/server/src/providers/ProviderType.ts b/packages/app/server/src/providers/ProviderType.ts index b2514ac80..41ded7703 100644 --- a/packages/app/server/src/providers/ProviderType.ts +++ b/packages/app/server/src/providers/ProviderType.ts @@ -12,4 +12,5 @@ export enum ProviderType { OPENAI_VIDEOS = 'OPENAI_VIDEOS', GROQ = 'GROQ', XAI = 'XAI', + VERCEL_AI_GATEWAY = 'VERCEL_AI_GATEWAY', } diff --git a/packages/app/server/src/providers/VercelAIGatewayProvider.ts b/packages/app/server/src/providers/VercelAIGatewayProvider.ts new file mode 100644 index 000000000..aefc3062d --- /dev/null +++ b/packages/app/server/src/providers/VercelAIGatewayProvider.ts @@ -0,0 +1,187 @@ +import { LlmTransactionMetadata, Transaction } from '../types'; +import { getCostPerToken, getModelPrice, isValidModel } from '../services/AccountingService'; +import { BaseProvider } from './BaseProvider'; +import { ProviderType } from './ProviderType'; +import logger from '../logger'; +import { env } from '../env'; +import { parseSSEGPTFormat, type CompletionStateBody } from './GPTProvider'; +import { Decimal } from '@prisma/client/runtime/library'; + +export class VercelAIGatewayProvider extends BaseProvider { + private readonly VERCEL_AI_GATEWAY_BASE_URL = 'https://ai-gateway.vercel.sh/v1'; + + getType(): ProviderType { + return ProviderType.VERCEL_AI_GATEWAY; + } + + getBaseUrl(): string { + return this.VERCEL_AI_GATEWAY_BASE_URL; + } + + getApiKey(): string | undefined { + return env.VERCEL_AI_GATEWAY_API_KEY; + } + + async handleBody( + data: string, + requestBody?: Record + ): Promise { + try { + const model = this.getModel().toLowerCase(); + const isTranscriptionModel = model.includes('whisper') || model.includes('transcription'); + const isSpeechModel = model.includes('tts') || model.includes('speech'); + + let isTranscriptionResponse = false; + try { + const parsed = JSON.parse(data); + if (parsed.text !== undefined && typeof parsed.text === 'string') { + isTranscriptionResponse = true; + } + } catch { + } + + if (isTranscriptionModel || isTranscriptionResponse) { + return this.handleAudioResponse(data, requestBody, 'transcription'); + } + + if (isSpeechModel) { + return this.handleAudioResponse(data, requestBody, 'speech'); + } + + return this.handleChatCompletionResponse(data); + } catch (error) { + logger.error(`Error processing Vercel AI Gateway response: ${error}`); + throw error; + } + } + + private handleChatCompletionResponse(data: string): Transaction { + let prompt_tokens = 0; + let completion_tokens = 0; + let total_tokens = 0; + let providerId = 'null'; + + if (this.getIsStream()) { + const chunks = parseSSEGPTFormat(data); + + for (const chunk of chunks) { + if (chunk.usage && chunk.usage !== null) { + prompt_tokens += chunk.usage.prompt_tokens; + completion_tokens += chunk.usage.completion_tokens; + total_tokens += chunk.usage.total_tokens; + } + providerId = chunk.id || 'null'; + } + } else { + const parsed = JSON.parse(data) as CompletionStateBody; + prompt_tokens += parsed.usage.prompt_tokens; + completion_tokens += parsed.usage.completion_tokens; + total_tokens += parsed.usage.total_tokens; + providerId = parsed.id || 'null'; + } + + const cost = getCostPerToken( + this.getModel(), + prompt_tokens, + completion_tokens + ); + + const metadata: LlmTransactionMetadata = { + providerId: providerId, + provider: this.getType(), + model: this.getModel(), + inputTokens: prompt_tokens, + outputTokens: completion_tokens, + totalTokens: total_tokens, + }; + + return { + rawTransactionCost: cost, + metadata: metadata, + status: 'success', + }; + } + + private handleAudioResponse( + data: string, + requestBody: Record | undefined, + endpointType: 'transcription' | 'speech' + ): Transaction { + let cost = new Decimal(0); + let metadata: LlmTransactionMetadata; + const model = this.getModel(); + + const modelPrice = getModelPrice(model); + + if (endpointType === 'transcription') { + try { + const transcriptionData = JSON.parse(data); + const text = transcriptionData.text || ''; + + if (modelPrice && isValidModel(model)) { + const textTokens = Math.ceil(text.length / 4); + cost = getCostPerToken(model, 0, textTokens); + } else { + cost = new Decimal(0.01); + } + + metadata = { + providerId: 'transcription', + provider: this.getType(), + model: model, + inputTokens: 0, + outputTokens: text.length, + totalTokens: text.length, + }; + } catch (error) { + logger.error(`Error parsing transcription response: ${error}`); + cost = modelPrice && isValidModel(model) ? new Decimal(0) : new Decimal(0.01); + metadata = { + providerId: 'transcription', + provider: this.getType(), + model: model, + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + }; + } + } else if (endpointType === 'speech') { + const inputText = (requestBody?.input as string) || ''; + const characterCount = inputText.length; + + if (modelPrice && isValidModel(model)) { + const inputTokens = Math.ceil(characterCount / 4); + cost = getCostPerToken(model, inputTokens, 0); + } else { + const costPerCharacter = new Decimal(0.000015); + cost = costPerCharacter.mul(characterCount); + } + + metadata = { + providerId: 'speech', + provider: this.getType(), + model: model, + inputTokens: characterCount, + outputTokens: 0, + totalTokens: characterCount, + }; + } else { + cost = modelPrice && isValidModel(model) ? new Decimal(0) : new Decimal(0.01); + metadata = { + providerId: 'audio', + provider: this.getType(), + model: model, + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + }; + } + + return { + rawTransactionCost: cost, + metadata: metadata, + status: 'success', + }; + } +} + diff --git a/packages/sdk/react/src/hooks/useEchoModelProviders.ts b/packages/sdk/react/src/hooks/useEchoModelProviders.ts index c11631ce5..5fbc19bb0 100644 --- a/packages/sdk/react/src/hooks/useEchoModelProviders.ts +++ b/packages/sdk/react/src/hooks/useEchoModelProviders.ts @@ -4,6 +4,7 @@ import { createEchoGroq, createEchoOpenAI, createEchoOpenRouter, + createEchoVercelAIGateway, createEchoXAI, } from '@merit-systems/echo-typescript-sdk'; import { useMemo } from 'react'; @@ -31,6 +32,11 @@ export const useEchoModelProviders = () => { ), groq: createEchoGroq(baseConfig, getToken, onInsufficientFunds), xai: createEchoXAI(baseConfig, getToken, onInsufficientFunds), + vercelAIGateway: createEchoVercelAIGateway( + baseConfig, + getToken, + onInsufficientFunds + ), }; }, [getToken, config.appId, config.baseRouterUrl, setIsInsufficientFunds]); }; diff --git a/packages/sdk/ts/src/providers/index.ts b/packages/sdk/ts/src/providers/index.ts index 62f54fac8..a9024c7f0 100644 --- a/packages/sdk/ts/src/providers/index.ts +++ b/packages/sdk/ts/src/providers/index.ts @@ -4,6 +4,7 @@ export * from './groq'; export * from './xai'; export * from './openai'; export * from './openrouter'; +export * from './vercel-ai-gateway'; export function echoFetch( originalFetch: typeof fetch, diff --git a/packages/sdk/ts/src/providers/vercel-ai-gateway.ts b/packages/sdk/ts/src/providers/vercel-ai-gateway.ts new file mode 100644 index 000000000..447742f23 --- /dev/null +++ b/packages/sdk/ts/src/providers/vercel-ai-gateway.ts @@ -0,0 +1,27 @@ +import { + createOpenAI as createOpenAIBase, + OpenAIProvider, +} from '@ai-sdk/openai'; +import { ROUTER_BASE_URL } from 'config'; +import { EchoConfig } from '../types'; +import { validateAppId } from '../utils/validation'; +import { echoFetch } from './index'; + +export function createEchoVercelAIGateway( + { appId, baseRouterUrl = ROUTER_BASE_URL }: EchoConfig, + getTokenFn: (appId: string) => Promise, + onInsufficientFunds?: () => void +): OpenAIProvider { + validateAppId(appId, 'createEchoVercelAIGateway'); + + return createOpenAIBase({ + baseURL: baseRouterUrl, + apiKey: 'placeholder_replaced_by_echoFetch', + fetch: echoFetch( + fetch, + async () => await getTokenFn(appId), + onInsufficientFunds + ), + }); +} +