From e40cd7f976c6da231b5041f403d1f50737f09760 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 16 Dec 2025 16:37:43 +0100 Subject: [PATCH 1/4] Add SGLang to local apps --- packages/tasks/src/local-apps.ts | 54 ++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index e723266cdc..4b19efd391 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -251,6 +251,46 @@ curl -X POST "http://localhost:8000/v1/completions" \\ }, ]; }; +const snippetSglang = (model: ModelData): LocalAppSnippet[] => { + const messages = getModelInputSnippet(model) as ChatCompletionInputMessage[]; + + let setup; + let serverCommand; + let dockerCommand; + let runCommand; + + setup = ["# Install SGLang from pip:", "pip install sglang"].join("\n"); + serverCommand = `# Start the SGLang serve:\npython3 -m sglang.launch_server --model-path ${model.id} \\ + --host 0.0.0.0 --log-level warning"`; + dockerCommand = `docker run --gpus all \\ + --shm-size 32g \\ + -p 30000:30000 \\ + -v ~/.cache/huggingface:/root/.cache/huggingface \\ + --env "HF_TOKEN=" \\ + --ipc=host \\ + lmsysorg/sglang:latest \\ + python3 -m sglang.launch_server --model-path ${model.id} --host 0.0.0.0 --port 30000` + runCommand = `curl -s http://localhost:{port}/v1/chat/completions \\ + -H "Content-Type: application/json" \\ + -d '{{"model": "${model.id}", "messages": ${stringifyMessages(messages, { + indent: "\t\t", + attributeKeyQuotes: true, + customContentEscaper: (str) => str.replace(/'/g, "'\\''"), + })}'` + + return [ + { + title: "Install from pip and serve model", + setup: setup, + content: [serverCommand, runCommand] + }, + { + title: "Use Docker images", + setup: dockerCommand, + content: [runCommand] + }, + ]; +}; const snippetTgi = (model: ModelData): LocalAppSnippet[] => { const runCommand = [ "# Call the server using curl:", @@ -401,6 +441,20 @@ export const LOCAL_APPS = { (model.pipeline_tag === "text-generation" || model.pipeline_tag === "image-text-to-text"), snippet: snippetVllm, }, + sglang: { + prettyLabel: "SGLang", + docsUrl: "https://docs.sglang.io", + mainTask: "text-generation", + displayOnModelPage: (model: ModelData) => + (isAwqModel(model) || + isGptqModel(model) || + isAqlmModel(model) || + isMarlinModel(model) || + isLlamaCppGgufModel(model) || + isTransformersModel(model)) && + (model.pipeline_tag === "text-generation" || model.pipeline_tag === "image-text-to-text"), + snippet: snippetSglang, + }, "mlx-lm": { prettyLabel: "MLX LM", docsUrl: "https://github.com/ml-explore/mlx-lm", From 6d0d3e79a2448d1ae3a1324e3cd844489640f34a Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 16 Dec 2025 16:43:23 +0100 Subject: [PATCH 2/4] let -> const --- packages/tasks/src/local-apps.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index 4b19efd391..33c870e011 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -254,15 +254,10 @@ curl -X POST "http://localhost:8000/v1/completions" \\ const snippetSglang = (model: ModelData): LocalAppSnippet[] => { const messages = getModelInputSnippet(model) as ChatCompletionInputMessage[]; - let setup; - let serverCommand; - let dockerCommand; - let runCommand; - - setup = ["# Install SGLang from pip:", "pip install sglang"].join("\n"); - serverCommand = `# Start the SGLang serve:\npython3 -m sglang.launch_server --model-path ${model.id} \\ + const setup = ["# Install SGLang from pip:", "pip install sglang"].join("\n"); + const serverCommand = `# Start the SGLang serve:\npython3 -m sglang.launch_server --model-path ${model.id} \\ --host 0.0.0.0 --log-level warning"`; - dockerCommand = `docker run --gpus all \\ + const dockerCommand = `docker run --gpus all \\ --shm-size 32g \\ -p 30000:30000 \\ -v ~/.cache/huggingface:/root/.cache/huggingface \\ @@ -270,7 +265,7 @@ const snippetSglang = (model: ModelData): LocalAppSnippet[] => { --ipc=host \\ lmsysorg/sglang:latest \\ python3 -m sglang.launch_server --model-path ${model.id} --host 0.0.0.0 --port 30000` - runCommand = `curl -s http://localhost:{port}/v1/chat/completions \\ + const runCommand = `curl -s http://localhost:{port}/v1/chat/completions \\ -H "Content-Type: application/json" \\ -d '{{"model": "${model.id}", "messages": ${stringifyMessages(messages, { indent: "\t\t", From 7136c760693238ffcf04650692c3e044c894dc57 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 16 Dec 2025 16:48:06 +0100 Subject: [PATCH 3/4] lint --- packages/tasks/src/local-apps.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index 33c870e011..626dbb764d 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -264,25 +264,25 @@ const snippetSglang = (model: ModelData): LocalAppSnippet[] => { --env "HF_TOKEN=" \\ --ipc=host \\ lmsysorg/sglang:latest \\ - python3 -m sglang.launch_server --model-path ${model.id} --host 0.0.0.0 --port 30000` + python3 -m sglang.launch_server --model-path ${model.id} --host 0.0.0.0 --port 30000`; const runCommand = `curl -s http://localhost:{port}/v1/chat/completions \\ -H "Content-Type: application/json" \\ -d '{{"model": "${model.id}", "messages": ${stringifyMessages(messages, { indent: "\t\t", attributeKeyQuotes: true, customContentEscaper: (str) => str.replace(/'/g, "'\\''"), - })}'` + })}'`; return [ { title: "Install from pip and serve model", setup: setup, - content: [serverCommand, runCommand] + content: [serverCommand, runCommand], }, { title: "Use Docker images", setup: dockerCommand, - content: [runCommand] + content: [runCommand], }, ]; }; From c9703ebc61f94c4b84482640d190acc400f5be09 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 17 Dec 2025 19:40:38 +0100 Subject: [PATCH 4/4] focus on mostly transformers compatibility --- packages/tasks/src/local-apps.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index 626dbb764d..0d0b67793e 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -82,6 +82,7 @@ function isMarlinModel(model: ModelData): boolean { function isTransformersModel(model: ModelData): boolean { return model.tags.includes("transformers"); } + function isTgiModel(model: ModelData): boolean { return model.tags.includes("text-generation-inference"); } @@ -441,12 +442,7 @@ export const LOCAL_APPS = { docsUrl: "https://docs.sglang.io", mainTask: "text-generation", displayOnModelPage: (model: ModelData) => - (isAwqModel(model) || - isGptqModel(model) || - isAqlmModel(model) || - isMarlinModel(model) || - isLlamaCppGgufModel(model) || - isTransformersModel(model)) && + isTransformersModel(model) && (model.pipeline_tag === "text-generation" || model.pipeline_tag === "image-text-to-text"), snippet: snippetSglang, },