diff --git a/src/globals.ts b/src/globals.ts index db4549941..2ceaadfa5 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -95,6 +95,7 @@ export const LEPTON: string = 'lepton'; export const KLUSTER_AI: string = 'kluster-ai'; export const NSCALE: string = 'nscale'; export const HYPERBOLIC: string = 'hyperbolic'; +export const BYTEZ: string = 'bytez'; export const FEATHERLESS_AI: string = 'featherless-ai'; export const KRUTRIM: string = 'krutrim'; export const QDRANT: string = 'qdrant'; @@ -158,6 +159,7 @@ export const VALID_PROVIDERS = [ KLUSTER_AI, NSCALE, HYPERBOLIC, + BYTEZ, FEATHERLESS_AI, KRUTRIM, QDRANT, diff --git a/src/providers/bytez/api.ts b/src/providers/bytez/api.ts new file mode 100644 index 000000000..c54ab546e --- /dev/null +++ b/src/providers/bytez/api.ts @@ -0,0 +1,20 @@ +import { ProviderAPIConfig } from '../types'; +import { version } from '../../../package.json'; + +const BytezInferenceAPI: ProviderAPIConfig = { + getBaseURL: () => 'https://api.bytez.com', + headers: async ({ providerOptions }) => { + const { apiKey } = providerOptions; + + const headers: Record = {}; + + headers['Authorization'] = `Key ${apiKey}`; + headers['user-agent'] = `portkey/${version}`; + + return headers; + }, + getEndpoint: ({ gatewayRequestBodyJSON: { version = 2, model } }) => + `/models/v${version}/${model}`, +}; + +export default BytezInferenceAPI; diff --git a/src/providers/bytez/chatComplete.ts b/src/providers/bytez/chatComplete.ts new file mode 100644 index 000000000..5c2c1756f --- /dev/null +++ b/src/providers/bytez/chatComplete.ts @@ -0,0 +1,77 @@ +import { BYTEZ } from '../../globals'; +import { ProviderConfig } from '../types'; +import { BytezResponse } from './types'; +import { generateErrorResponse } from '../utils'; + +const BytezInferenceChatCompleteConfig: ProviderConfig = { + messages: { + param: 'messages', + required: true, + }, + max_tokens: { + param: 'params.max_new_tokens', + default: 100, + min: 0, + }, + temperature: { + param: 'params.temperature', + default: 1, + min: 0, + max: 2, + }, + top_p: { + param: 'params.top_p', + default: 1, + min: 0, + max: 1, + }, + stream: { + param: 'stream', + default: false, + }, +}; + +function chatComplete( + response: BytezResponse, + responseStatus: number, + responseHeaders: any, + strictOpenAiCompliance: boolean, + endpoint: string, + requestBody: any +) { + const { error, output } = response; + + if (error) { + return generateErrorResponse( + { + message: error, + type: String(responseStatus), + param: null, + code: null, + }, + BYTEZ + ); + } + + return { + id: crypto.randomUUID(), + object: 'chat.completion', + created: Date.now(), + model: requestBody.model, + choices: [ + { + index: 0, + message: output, + logprobs: null, + finish_reason: 'stop', + }, + ], + usage: { + completion_tokens: -1, + prompt_tokens: -1, + total_tokens: -1, + }, + }; +} + +export { BytezInferenceChatCompleteConfig, chatComplete }; diff --git a/src/providers/bytez/index.ts b/src/providers/bytez/index.ts new file mode 100644 index 000000000..2b1782bec --- /dev/null +++ b/src/providers/bytez/index.ts @@ -0,0 +1,13 @@ +import { ProviderConfigs } from '../types'; +import BytezInferenceAPI from './api'; +import { BytezInferenceChatCompleteConfig, chatComplete } from './chatComplete'; + +const BytezInferenceAPIConfig: ProviderConfigs = { + api: BytezInferenceAPI, + chatComplete: BytezInferenceChatCompleteConfig, + responseTransforms: { + chatComplete, + }, +}; + +export default BytezInferenceAPIConfig; diff --git a/src/providers/bytez/types.ts b/src/providers/bytez/types.ts new file mode 100644 index 000000000..1d640ea8b --- /dev/null +++ b/src/providers/bytez/types.ts @@ -0,0 +1,10 @@ +interface Model { + task: string; +} + +interface BytezResponse { + error: string; + output: Model[]; +} + +export { Model, BytezResponse }; diff --git a/src/providers/index.ts b/src/providers/index.ts index d5450e696..f95e9b949 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -1,3 +1,4 @@ +import BytezConfig from './bytez'; import AI21Config from './ai21'; import AnthropicConfig from './anthropic'; import AnyscaleConfig from './anyscale'; @@ -120,6 +121,7 @@ const Providers: { [key: string]: ProviderConfigs } = { 'kluster-ai': KlusterAIConfig, nscale: NscaleConfig, hyperbolic: HyperbolicConfig, + bytez: BytezConfig, 'featherless-ai': FeatherlessAIConfig, krutrim: KrutrimConfig, }; diff --git a/src/public/index.html b/src/public/index.html index 45faccae0..9bd7e77e2 100644 --- a/src/public/index.html +++ b/src/public/index.html @@ -1115,6 +1115,7 @@

Select Provider

+ @@ -1465,6 +1466,7 @@

Enter API Key

"together-ai": "llama-3.1-8b-instruct", "perplexity-ai": "pplx-7b-online", "mistral-ai": "mistral-small-latest", + "bytez": "google/gemma-3-1b-it", "others": "gpt-4o-mini" } diff --git a/src/utils.ts b/src/utils.ts index ba42f7545..d32896f62 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -7,6 +7,7 @@ import { DEEPINFRA, SAMBANOVA, BEDROCK, + BYTEZ, } from './globals'; import { Params } from './types/requestBody'; @@ -48,9 +49,13 @@ export const getStreamModeSplitPattern = ( splitPattern = '\n'; } + if (proxyProvider === BYTEZ) { + splitPattern = ' '; + } + return splitPattern; }; -export type SplitPatternType = '\n\n' | '\r\n\r\n' | '\n' | '\r\n'; +export type SplitPatternType = '\n\n' | '\r\n\r\n' | '\n' | '\r\n' | ' '; export const getStreamingMode = ( reqBody: Params,