From 6a7e2bd83cee7bff6dc4994211f583cdf1b32437 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 18 Dec 2025 18:56:21 -0800 Subject: [PATCH 01/11] Update upload bm results GHA Signed-off-by: Huy Do --- .../actions/gather-runners-info/action.yml | 20 ++++++- .../upload-benchmark-results/action.yml | 55 ++++++++++++++----- .github/scripts/upload_benchmark_results.py | 8 +++ 3 files changed, 67 insertions(+), 16 deletions(-) diff --git a/.github/actions/gather-runners-info/action.yml b/.github/actions/gather-runners-info/action.yml index 3c02bf6730..8f61ed4947 100644 --- a/.github/actions/gather-runners-info/action.yml +++ b/.github/actions/gather-runners-info/action.yml @@ -27,8 +27,20 @@ runs: elif command -v rocm-smi; then DEVICE_NAME=rocm rocm-smi + elif command -v hl-smi; then + DEVICE_NAME=hpu + hl-smi else - DEVICE_NAME=cpu + arch=$(uname -m) + + case "$arch" in + aarch64|arm64) + DEVICE_NAME=arm64-cpu + ;; + *) + DEVICE_NAME=cpu + ;; + esac lscpu fi echo "DEVICE_NAME=$DEVICE_NAME" >> $GITHUB_ENV @@ -42,8 +54,12 @@ runs: DEVICE_TYPE=$(nvidia-smi -i 0 --query-gpu=name --format=csv,noheader | awk '{print $2}') elif [[ "${DEVICE_NAME}" == "rocm" ]]; then DEVICE_TYPE=$(rocminfo | grep "Marketing Name" | tail -n1 | awk -F':' '{print $2}' | xargs) + elif [[ "${DEVICE_NAME}" == "hpu" ]]; then + DEVICE_TYPE="Intel Gaudi3 "$(hl-smi -q | grep "Product Name" | head -n 1 | awk -F ':' '{print $2}' | sed 's/^ *//') elif [[ "${DEVICE_NAME}" == "cpu" ]]; then - DEVICE_TYPE=$(lscpu | grep 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1' | cut -f 2 -d " ") + DEVICE_TYPE="$(lscpu | grep "Model name" | sed -E 's/.*Model name:[[:space:]]*//; s/Intel\(R\)//g; s/\(R\)//g; s/\(TM\)//g; s/CPU//g; s/Processor//g; s/[[:space:]]+/ /g; s/^ //; s/ $//; s/ /_/g')_$(awk -F: '/Core\(s\) per socket/ {c=$2} /Socket\(s\)/ {s=$2} END {gsub(/ /,"",c); gsub(/ /,"",s); printf "%sc", c*s}' < <(lscpu))" + elif [[ "${DEVICE_NAME}" == "arm64-cpu" ]]; then + DEVICE_TYPE=$(lscpu | grep 'Vendor ID' | cut -f 2 -d ":" | awk '{$1=$1}1' | cut -f 2 -d " ") fi echo "DEVICE_TYPE=$DEVICE_TYPE" >> $GITHUB_ENV diff --git a/.github/actions/upload-benchmark-results/action.yml b/.github/actions/upload-benchmark-results/action.yml index 521f16e776..e209b88d70 100644 --- a/.github/actions/upload-benchmark-results/action.yml +++ b/.github/actions/upload-benchmark-results/action.yml @@ -26,26 +26,53 @@ runs: if [[ -n "${{ inputs.venv }}" ]]; then source "${{ inputs.venv }}" fi - python3 -mpip install boto3==1.35.33 psutil==7.0.0 pynvml==12.0.0 + python3 -mpip install boto3==1.35.33 psutil==7.0.0 nvidia-ml-py==13.580.82 - DEVICE_NAME="" - DEVICE_TYPE="" + - name: Get device name + shell: bash + run: | + set -eux if command -v nvidia-smi; then - # NB: I'm using PyTorch here to get the device name, however, it needs to - # install the correct version of PyTorch manually for now. Any PyTorch - # version is fine, I just use 2.7.1 to satify PYPIDEP linter - python3 -mpip install torch==2.7.1 - elif command -v rocminfo; then - # NB: Installing torch on ROCm runner with pip here causes CI to fail - # with a memoryview is too large error only on MI300 runners. Is pip - # version on ROCm runner there too old? As a workaround, let's use the - # GPU device name coming from rocminfo instead + DEVICE_NAME=cuda + nvidia-smi + elif command -v rocm-smi; then DEVICE_NAME=rocm - DEVICE_TYPE=$(rocminfo | grep "Marketing Name" | tail -n1 | awk -F':' '{print $2}' | xargs) + rocm-smi + elif command -v hl-smi; then + DEVICE_NAME=hpu + hl-smi + else + arch=$(uname -m) + + case "$arch" in + aarch64|arm64) + DEVICE_NAME=arm64-cpu + ;; + *) + DEVICE_NAME=cpu + ;; + esac + lscpu fi - echo "DEVICE_NAME=$DEVICE_NAME" >> $GITHUB_ENV + + - name: Get device type + shell: bash + run: | + set -eux + + if [[ "${DEVICE_NAME}" == "cuda" ]]; then + DEVICE_TYPE=$(nvidia-smi -i 0 --query-gpu=name --format=csv,noheader | awk '{print $2}') + elif [[ "${DEVICE_NAME}" == "rocm" ]]; then + DEVICE_TYPE=$(rocminfo | grep "Marketing Name" | tail -n1 | awk -F':' '{print $2}' | xargs) + elif [[ "${DEVICE_NAME}" == "hpu" ]]; then + DEVICE_TYPE="Intel Gaudi3 "$(hl-smi -q | grep "Product Name" | head -n 1 | awk -F ':' '{print $2}' | sed 's/^ *//') + elif [[ "${DEVICE_NAME}" == "cpu" ]]; then + DEVICE_TYPE="$(lscpu | grep "Model name" | sed -E 's/.*Model name:[[:space:]]*//; s/Intel\(R\)//g; s/\(R\)//g; s/\(TM\)//g; s/CPU//g; s/Processor//g; s/[[:space:]]+/ /g; s/^ //; s/ $//; s/ /_/g')_$(awk -F: '/Core\(s\) per socket/ {c=$2} /Socket\(s\)/ {s=$2} END {gsub(/ /,"",c); gsub(/ /,"",s); printf "%sc", c*s}' < <(lscpu))" + elif [[ "${DEVICE_NAME}" == "arm64-cpu" ]]; then + DEVICE_TYPE=$(lscpu | grep 'Vendor ID' | cut -f 2 -d ":" | awk '{$1=$1}1' | cut -f 2 -d " ") + fi echo "DEVICE_TYPE=$DEVICE_TYPE" >> $GITHUB_ENV - name: Check that GITHUB_TOKEN is defined diff --git a/.github/scripts/upload_benchmark_results.py b/.github/scripts/upload_benchmark_results.py index 5513030003..6cbcff3b27 100755 --- a/.github/scripts/upload_benchmark_results.py +++ b/.github/scripts/upload_benchmark_results.py @@ -319,6 +319,7 @@ def upload_to_s3( def main() -> None: args = parse_args() + has_results_uploaded = False for file in os.listdir(args.benchmark_results_dir): if not file.endswith(".json"): continue @@ -349,6 +350,7 @@ def main() -> None: if not benchmark_results: continue + has_results_uploaded = True upload_to_s3( s3_bucket=OSSCI_BENCHMARKS_BUCKET, filepath=filepath, @@ -357,6 +359,12 @@ def main() -> None: dry_run=args.dry_run, ) + # When there is no benchmark results, treat it as a failure. This is better + # than failing silently. + if not has_results_uploaded: + warn(f"Find no benchmark results in {args.benchmark_results}") + sys.exit(1) + if __name__ == "__main__": main() From cbb2fdf954e8ce66d83c0c52ebcdffa361658a87 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 18 Dec 2025 19:06:58 -0800 Subject: [PATCH 02/11] Set benchmark name manually Signed-off-by: Huy Do --- .github/actions/upload-benchmark-results/action.yml | 7 +++++-- .github/scripts/benchmarks/gather_metadata.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/actions/upload-benchmark-results/action.yml b/.github/actions/upload-benchmark-results/action.yml index e209b88d70..1a0b3aa952 100644 --- a/.github/actions/upload-benchmark-results/action.yml +++ b/.github/actions/upload-benchmark-results/action.yml @@ -3,7 +3,10 @@ name: Upload benchmark results inputs: benchmark-results-dir: description: 'The path to the directory with all the results in JSON format' - required: True + required: true + benchmark-name: + description: 'Manually set the name of the benchmark' + default: '' dry-run: default: 'true' schema-version: @@ -12,7 +15,6 @@ inputs: default: '' venv: description: 'Path to virtual environment to activate' - required: false default: '' runs: @@ -106,6 +108,7 @@ runs: RUN_ATTEMPT: ${{ github.run_attempt }} JOB_ID: ${{ inputs.github-token != '' && steps.get-job-id.outputs.job-id || '0' }} JOB_NAME: ${{ inputs.github-token != '' && steps.get-job-id.outputs.job-name || '' }} + BENCHMARK_NAME: ${{ inputs.benchmark-name || '' }} run: | set -eux diff --git a/.github/scripts/benchmarks/gather_metadata.py b/.github/scripts/benchmarks/gather_metadata.py index e38c8b5bdf..8bb9969fa4 100755 --- a/.github/scripts/benchmarks/gather_metadata.py +++ b/.github/scripts/benchmarks/gather_metadata.py @@ -83,7 +83,7 @@ def main() -> None: metadata = { "timestamp": int(time.time()), "schema_version": args.schema_version, - "name": args.job_name, + "name": os.getenv("BENCHMARK_NAME", args.job_name), "repo": args.repo, "head_branch": args.head_branch, "head_sha": args.head_sha, From e2037cdaac6281ee5baa46205e762b5a0f62c414 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 18 Dec 2025 19:20:05 -0800 Subject: [PATCH 03/11] Typo Signed-off-by: Huy Do --- .github/scripts/upload_benchmark_results.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/upload_benchmark_results.py b/.github/scripts/upload_benchmark_results.py index 6cbcff3b27..4f7588ff16 100755 --- a/.github/scripts/upload_benchmark_results.py +++ b/.github/scripts/upload_benchmark_results.py @@ -17,6 +17,7 @@ from logging import info from typing import Any, Callable, Dict, List, Optional from warnings import warn +import sys import boto3 # type: ignore[import-not-found] From 59ae17283c0247d14aa90e767d7c827dbc115d3d Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 18 Dec 2025 20:56:39 -0800 Subject: [PATCH 04/11] Fix lint Signed-off-by: Huy Do --- .github/scripts/upload_benchmark_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/upload_benchmark_results.py b/.github/scripts/upload_benchmark_results.py index 4f7588ff16..e3021a0a7e 100755 --- a/.github/scripts/upload_benchmark_results.py +++ b/.github/scripts/upload_benchmark_results.py @@ -10,6 +10,7 @@ import json import logging import os +import sys import time from argparse import Action, ArgumentParser, Namespace from decimal import Decimal @@ -17,7 +18,6 @@ from logging import info from typing import Any, Callable, Dict, List, Optional from warnings import warn -import sys import boto3 # type: ignore[import-not-found] From 4b225163ab6a71a6541fade2ce2eba6c205a4b17 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 19 Dec 2025 12:13:35 -0800 Subject: [PATCH 05/11] Remove redundant change Signed-off-by: Huy Do --- .../actions/gather-runners-info/action.yml | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/.github/actions/gather-runners-info/action.yml b/.github/actions/gather-runners-info/action.yml index 8f61ed4947..3c02bf6730 100644 --- a/.github/actions/gather-runners-info/action.yml +++ b/.github/actions/gather-runners-info/action.yml @@ -27,20 +27,8 @@ runs: elif command -v rocm-smi; then DEVICE_NAME=rocm rocm-smi - elif command -v hl-smi; then - DEVICE_NAME=hpu - hl-smi else - arch=$(uname -m) - - case "$arch" in - aarch64|arm64) - DEVICE_NAME=arm64-cpu - ;; - *) - DEVICE_NAME=cpu - ;; - esac + DEVICE_NAME=cpu lscpu fi echo "DEVICE_NAME=$DEVICE_NAME" >> $GITHUB_ENV @@ -54,12 +42,8 @@ runs: DEVICE_TYPE=$(nvidia-smi -i 0 --query-gpu=name --format=csv,noheader | awk '{print $2}') elif [[ "${DEVICE_NAME}" == "rocm" ]]; then DEVICE_TYPE=$(rocminfo | grep "Marketing Name" | tail -n1 | awk -F':' '{print $2}' | xargs) - elif [[ "${DEVICE_NAME}" == "hpu" ]]; then - DEVICE_TYPE="Intel Gaudi3 "$(hl-smi -q | grep "Product Name" | head -n 1 | awk -F ':' '{print $2}' | sed 's/^ *//') elif [[ "${DEVICE_NAME}" == "cpu" ]]; then - DEVICE_TYPE="$(lscpu | grep "Model name" | sed -E 's/.*Model name:[[:space:]]*//; s/Intel\(R\)//g; s/\(R\)//g; s/\(TM\)//g; s/CPU//g; s/Processor//g; s/[[:space:]]+/ /g; s/^ //; s/ $//; s/ /_/g')_$(awk -F: '/Core\(s\) per socket/ {c=$2} /Socket\(s\)/ {s=$2} END {gsub(/ /,"",c); gsub(/ /,"",s); printf "%sc", c*s}' < <(lscpu))" - elif [[ "${DEVICE_NAME}" == "arm64-cpu" ]]; then - DEVICE_TYPE=$(lscpu | grep 'Vendor ID' | cut -f 2 -d ":" | awk '{$1=$1}1' | cut -f 2 -d " ") + DEVICE_TYPE=$(lscpu | grep 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1' | cut -f 2 -d " ") fi echo "DEVICE_TYPE=$DEVICE_TYPE" >> $GITHUB_ENV From faabc80e937b027d3f2675a1d50bb9f51cedb613 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 19 Dec 2025 13:46:10 -0800 Subject: [PATCH 06/11] Configure PyTorch x vLLM nightly dashboard Signed-off-by: Huy Do --- .../benchmark_v3/configs/configurations.tsx | 18 +++ .../teams/vllm/pytorch_x_vllm_config.ts | 146 ++++++++++++++++++ .../backend/dataFetchers/fetchers.ts | 1 + 3 files changed, 165 insertions(+) create mode 100644 torchci/components/benchmark_v3/configs/teams/vllm/pytorch_x_vllm_config.ts diff --git a/torchci/components/benchmark_v3/configs/configurations.tsx b/torchci/components/benchmark_v3/configs/configurations.tsx index b90aa428df..5d852b56f6 100644 --- a/torchci/components/benchmark_v3/configs/configurations.tsx +++ b/torchci/components/benchmark_v3/configs/configurations.tsx @@ -31,6 +31,10 @@ import { PYTORCH_VLLM_BENCHMARK_ID, PytorchVllmBenchmarkDashoboardConfig, } from "./teams/vllm/config"; +import { + PYTORCH_X_VLLM_BENCHMARK_ID, + PytorchXVllmBenchmarkDashboardConfig, +} from "./teams/vllm/pytorch_x_vllm_config"; export const REPORT_ID_TO_BENCHMARK_ID_MAPPING: Record = { compiler_regression: "compiler_inductor", @@ -40,6 +44,9 @@ export const PREDEFINED_BENCHMARK_CONFIG: BenchmarkConfigMap = { [COMPILTER_BENCHMARK_NAME]: { [BenchmarkPageType.DashboardPage]: CompilerDashboardBenchmarkUIConfig, }, + [PYTORCH_X_VLLM_BENCHMARK_ID]: { + [BenchmarkPageType.DashboardPage]: PytorchXVllmBenchmarkDashboardConfig, + }, [COMPILTER_PRECOMPUTE_BENCHMARK_ID]: { [BenchmarkPageType.AggregatePage]: CompilerPrecomputeBenchmarkUIConfig, }, @@ -69,6 +76,11 @@ export const BENCHMARK_ID_MAPPING: Record = { repoName: "pytorch/pytorch", benchmarkName: "compiler_inductor", }, + [PYTORCH_X_VLLM_BENCHMARK_ID]: { + id: PYTORCH_X_VLLM_BENCHMARK_ID, + repoName: "pytorch/pytorch", + benchmarkName: "PyTorch x vLLM benchmark", + }, [COMPILTER_PRECOMPUTE_BENCHMARK_ID]: { id: COMPILTER_PRECOMPUTE_BENCHMARK_ID, repoName: "pytorch/pytorch", @@ -181,6 +193,12 @@ export const BENCHMARK_CATEGORIES: BenchmarkCategoryGroup[] = [ name: "Triton Benchmark", route: "/tritonbench/commit_view", }, + { + name: "PyTorch x vLLM Benchmark", + route: `/benchmark/v3/dashboard/${PYTORCH_X_VLLM_BENCHMARK_ID}`, + info: "PyTorch integration benchmark for vLLM", + description: "Benchmark comparing PyTorch with vLLM performance", + }, ], }, { diff --git a/torchci/components/benchmark_v3/configs/teams/vllm/pytorch_x_vllm_config.ts b/torchci/components/benchmark_v3/configs/teams/vllm/pytorch_x_vllm_config.ts new file mode 100644 index 0000000000..69f99f33da --- /dev/null +++ b/torchci/components/benchmark_v3/configs/teams/vllm/pytorch_x_vllm_config.ts @@ -0,0 +1,146 @@ +import { BenchmarkUIConfig } from "../../config_book_types"; +import { + BRANCH_METADATA_COLUMN, + DEFAULT_DASHBOARD_BENCHMARK_INITIAL, +} from "../defaults/default_dashboard_config"; + +export const PYTORCH_X_VLLM_BENCHMARK_ID = "pytorch_x_vllm_benchmark"; + +const COMPARISON_TABLE_METADATA_COLUMNS = [ + { + field: "device", + displayName: "Hardware type", + }, + { + field: "arch", + displayName: "Hardware model", + }, + { + field: "extra_key.use_compile", + displayName: "Use Compile", + }, + { + field: "extra_key.request_rate", + displayName: "Request Rate", + }, + { + field: "extra_key.tensor_parallel_size", + displayName: "Tensor Parallel", + }, + { + field: "extra_key.input_len", + displayName: "Input Len", + }, + { + field: "extra_key.output_len", + displayName: "Max Output Len", + }, +] as const; + +export const PytorchXVllmBenchmarkDashboardConfig: BenchmarkUIConfig = { + benchmarkId: PYTORCH_X_VLLM_BENCHMARK_ID, + apiId: PYTORCH_X_VLLM_BENCHMARK_ID, + title: "PyTorch x vLLM Benchmark", + type: "dashboard", + dataBinding: { + initial: { + ...DEFAULT_DASHBOARD_BENCHMARK_INITIAL, + benchmarkId: PYTORCH_X_VLLM_BENCHMARK_ID, + }, + required_filter_fields: [], + }, + dataRender: { + type: "auto", + subSectionRenders: { + detail_view: { + filterConstraint: { + model: { + disabled: true, + }, + deviceName: { + disableOptions: [""], + }, + mode: { + disableOptions: [""], + }, + }, + renders: [ + { + type: "AutoBenchmarkTimeSeriesChartGroup", + title: "Metrics Time Series Chart Detail View", + config: { + type: "line", + groupByFields: ["metric"], + lineKey: [ + "model", + "extra_key.use_compile", + "extra_key.request_rate", + "extra_key.input_len", + "extra_key.output_len", + "metric", + "branch", + ], + chart: { + renderOptions: { + showLegendDetails: true, + }, + }, + }, + }, + { + type: "AutoBenchmarkTimeSeriesTable", + title: "Comparison Table Detail View", + config: { + primary: { + fields: ["model"], + displayName: "Model", + }, + extraMetadata: COMPARISON_TABLE_METADATA_COLUMNS, + renderOptions: { + missingText: "", + flex: { + primary: 2, + }, + }, + }, + }, + { + type: "AutoBenchmarkRawDataTable", + title: "Raw Data Table", + config: { + extraMetadata: [ + BRANCH_METADATA_COLUMN, + ...COMPARISON_TABLE_METADATA_COLUMNS, + ], + }, + }, + ], + }, + }, + renders: [ + { + type: "AutoBenchmarkPairwiseTable", + title: "Comparison Table", + config: { + primary: { + fields: ["model"], + displayName: "Model", + navigation: { + type: "subSectionRender", + value: "detail_view", + applyFilterFields: ["model", "device", "arch"], + }, + }, + extraMetadata: COMPARISON_TABLE_METADATA_COLUMNS, + renderOptions: { + missingText: "none", + bothMissingText: "", + flex: { + primary: 2, + }, + }, + }, + }, + ], + }, +}; diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts index 8b8622f7cb..01ed7aab9b 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts @@ -26,6 +26,7 @@ const dataCtors: Record BenchmarkDataFetcher> = { pytorch_helion: PytorchHelionDataFetcher, torchao_micro_api_benchmark: PytorchAoMicroApiBenchmarkDataFetcher, vllm_benchmark: VllmBenchmarkDataFetcher, + pytorch_x_vllm_benchmark: VllmBenchmarkDataFetcher, default: BenchmarkDataQuery, }; From 8188d6165bbc61a8ed148e68d8f356cf786a7a09 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 19 Dec 2025 13:46:44 -0800 Subject: [PATCH 07/11] Revert "Configure PyTorch x vLLM nightly dashboard" This reverts commit faabc80e937b027d3f2675a1d50bb9f51cedb613. --- .../benchmark_v3/configs/configurations.tsx | 18 --- .../teams/vllm/pytorch_x_vllm_config.ts | 146 ------------------ .../backend/dataFetchers/fetchers.ts | 1 - 3 files changed, 165 deletions(-) delete mode 100644 torchci/components/benchmark_v3/configs/teams/vllm/pytorch_x_vllm_config.ts diff --git a/torchci/components/benchmark_v3/configs/configurations.tsx b/torchci/components/benchmark_v3/configs/configurations.tsx index 5d852b56f6..b90aa428df 100644 --- a/torchci/components/benchmark_v3/configs/configurations.tsx +++ b/torchci/components/benchmark_v3/configs/configurations.tsx @@ -31,10 +31,6 @@ import { PYTORCH_VLLM_BENCHMARK_ID, PytorchVllmBenchmarkDashoboardConfig, } from "./teams/vllm/config"; -import { - PYTORCH_X_VLLM_BENCHMARK_ID, - PytorchXVllmBenchmarkDashboardConfig, -} from "./teams/vllm/pytorch_x_vllm_config"; export const REPORT_ID_TO_BENCHMARK_ID_MAPPING: Record = { compiler_regression: "compiler_inductor", @@ -44,9 +40,6 @@ export const PREDEFINED_BENCHMARK_CONFIG: BenchmarkConfigMap = { [COMPILTER_BENCHMARK_NAME]: { [BenchmarkPageType.DashboardPage]: CompilerDashboardBenchmarkUIConfig, }, - [PYTORCH_X_VLLM_BENCHMARK_ID]: { - [BenchmarkPageType.DashboardPage]: PytorchXVllmBenchmarkDashboardConfig, - }, [COMPILTER_PRECOMPUTE_BENCHMARK_ID]: { [BenchmarkPageType.AggregatePage]: CompilerPrecomputeBenchmarkUIConfig, }, @@ -76,11 +69,6 @@ export const BENCHMARK_ID_MAPPING: Record = { repoName: "pytorch/pytorch", benchmarkName: "compiler_inductor", }, - [PYTORCH_X_VLLM_BENCHMARK_ID]: { - id: PYTORCH_X_VLLM_BENCHMARK_ID, - repoName: "pytorch/pytorch", - benchmarkName: "PyTorch x vLLM benchmark", - }, [COMPILTER_PRECOMPUTE_BENCHMARK_ID]: { id: COMPILTER_PRECOMPUTE_BENCHMARK_ID, repoName: "pytorch/pytorch", @@ -193,12 +181,6 @@ export const BENCHMARK_CATEGORIES: BenchmarkCategoryGroup[] = [ name: "Triton Benchmark", route: "/tritonbench/commit_view", }, - { - name: "PyTorch x vLLM Benchmark", - route: `/benchmark/v3/dashboard/${PYTORCH_X_VLLM_BENCHMARK_ID}`, - info: "PyTorch integration benchmark for vLLM", - description: "Benchmark comparing PyTorch with vLLM performance", - }, ], }, { diff --git a/torchci/components/benchmark_v3/configs/teams/vllm/pytorch_x_vllm_config.ts b/torchci/components/benchmark_v3/configs/teams/vllm/pytorch_x_vllm_config.ts deleted file mode 100644 index 69f99f33da..0000000000 --- a/torchci/components/benchmark_v3/configs/teams/vllm/pytorch_x_vllm_config.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { BenchmarkUIConfig } from "../../config_book_types"; -import { - BRANCH_METADATA_COLUMN, - DEFAULT_DASHBOARD_BENCHMARK_INITIAL, -} from "../defaults/default_dashboard_config"; - -export const PYTORCH_X_VLLM_BENCHMARK_ID = "pytorch_x_vllm_benchmark"; - -const COMPARISON_TABLE_METADATA_COLUMNS = [ - { - field: "device", - displayName: "Hardware type", - }, - { - field: "arch", - displayName: "Hardware model", - }, - { - field: "extra_key.use_compile", - displayName: "Use Compile", - }, - { - field: "extra_key.request_rate", - displayName: "Request Rate", - }, - { - field: "extra_key.tensor_parallel_size", - displayName: "Tensor Parallel", - }, - { - field: "extra_key.input_len", - displayName: "Input Len", - }, - { - field: "extra_key.output_len", - displayName: "Max Output Len", - }, -] as const; - -export const PytorchXVllmBenchmarkDashboardConfig: BenchmarkUIConfig = { - benchmarkId: PYTORCH_X_VLLM_BENCHMARK_ID, - apiId: PYTORCH_X_VLLM_BENCHMARK_ID, - title: "PyTorch x vLLM Benchmark", - type: "dashboard", - dataBinding: { - initial: { - ...DEFAULT_DASHBOARD_BENCHMARK_INITIAL, - benchmarkId: PYTORCH_X_VLLM_BENCHMARK_ID, - }, - required_filter_fields: [], - }, - dataRender: { - type: "auto", - subSectionRenders: { - detail_view: { - filterConstraint: { - model: { - disabled: true, - }, - deviceName: { - disableOptions: [""], - }, - mode: { - disableOptions: [""], - }, - }, - renders: [ - { - type: "AutoBenchmarkTimeSeriesChartGroup", - title: "Metrics Time Series Chart Detail View", - config: { - type: "line", - groupByFields: ["metric"], - lineKey: [ - "model", - "extra_key.use_compile", - "extra_key.request_rate", - "extra_key.input_len", - "extra_key.output_len", - "metric", - "branch", - ], - chart: { - renderOptions: { - showLegendDetails: true, - }, - }, - }, - }, - { - type: "AutoBenchmarkTimeSeriesTable", - title: "Comparison Table Detail View", - config: { - primary: { - fields: ["model"], - displayName: "Model", - }, - extraMetadata: COMPARISON_TABLE_METADATA_COLUMNS, - renderOptions: { - missingText: "", - flex: { - primary: 2, - }, - }, - }, - }, - { - type: "AutoBenchmarkRawDataTable", - title: "Raw Data Table", - config: { - extraMetadata: [ - BRANCH_METADATA_COLUMN, - ...COMPARISON_TABLE_METADATA_COLUMNS, - ], - }, - }, - ], - }, - }, - renders: [ - { - type: "AutoBenchmarkPairwiseTable", - title: "Comparison Table", - config: { - primary: { - fields: ["model"], - displayName: "Model", - navigation: { - type: "subSectionRender", - value: "detail_view", - applyFilterFields: ["model", "device", "arch"], - }, - }, - extraMetadata: COMPARISON_TABLE_METADATA_COLUMNS, - renderOptions: { - missingText: "none", - bothMissingText: "", - flex: { - primary: 2, - }, - }, - }, - }, - ], - }, -}; diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts index 01ed7aab9b..8b8622f7cb 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts @@ -26,7 +26,6 @@ const dataCtors: Record BenchmarkDataFetcher> = { pytorch_helion: PytorchHelionDataFetcher, torchao_micro_api_benchmark: PytorchAoMicroApiBenchmarkDataFetcher, vllm_benchmark: VllmBenchmarkDataFetcher, - pytorch_x_vllm_benchmark: VllmBenchmarkDataFetcher, default: BenchmarkDataQuery, }; From 75bb8bf20941f7193d37952cd8897b07eb16bff8 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 19 Dec 2025 14:06:30 -0800 Subject: [PATCH 08/11] Overwrite the bm name Signed-off-by: Huy Do --- .github/scripts/upload_benchmark_results.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/scripts/upload_benchmark_results.py b/.github/scripts/upload_benchmark_results.py index e3021a0a7e..7f38d92100 100755 --- a/.github/scripts/upload_benchmark_results.py +++ b/.github/scripts/upload_benchmark_results.py @@ -217,6 +217,13 @@ def read_benchmark_results(filepath: str) -> List[Dict[str, Any]]: except JSONDecodeError: warn(f"Invalid JSON {line}, skipping") + # Overwrite the benchmark name if needed + if os.getenv("BENCHMARK_NAME"): + benchmark_name = os.getenv("BENCHMARK_NAME") + for bresult in benchmark_results: + if bresult.get("benchmark") and bresult.get("benchmark").get("name"): + bresult["benchmark"]["name"] = benchmark_name + return benchmark_results From 4e9bd68321855603859ccd3a80b20e8776a01dcc Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 19 Dec 2025 14:10:25 -0800 Subject: [PATCH 09/11] Fix lint Signed-off-by: Huy Do --- .github/scripts/upload_benchmark_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/upload_benchmark_results.py b/.github/scripts/upload_benchmark_results.py index 7f38d92100..d71902a48f 100755 --- a/.github/scripts/upload_benchmark_results.py +++ b/.github/scripts/upload_benchmark_results.py @@ -186,7 +186,7 @@ def upload_to_dynamodb( def read_benchmark_results(filepath: str) -> List[Dict[str, Any]]: - benchmark_results = [] + benchmark_results: List[Dict[str, Any]] = [] with open(filepath) as f: try: r = json.load(f) From 2770a0e7b21def27909ce8b699e9aede63b8fb01 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 19 Dec 2025 14:19:18 -0800 Subject: [PATCH 10/11] Fix lint Signed-off-by: Huy Do --- .github/scripts/upload_benchmark_results.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/upload_benchmark_results.py b/.github/scripts/upload_benchmark_results.py index d71902a48f..93fe9f0e42 100755 --- a/.github/scripts/upload_benchmark_results.py +++ b/.github/scripts/upload_benchmark_results.py @@ -221,7 +221,9 @@ def read_benchmark_results(filepath: str) -> List[Dict[str, Any]]: if os.getenv("BENCHMARK_NAME"): benchmark_name = os.getenv("BENCHMARK_NAME") for bresult in benchmark_results: - if bresult.get("benchmark") and bresult.get("benchmark").get("name"): + if bresult.get("benchmark", {}) and bresult.get("benchmark", {}).get( + "name" + ): bresult["benchmark"]["name"] = benchmark_name return benchmark_results From f6b7fd68bbd3b79a4da892a0b1d4ef014b57fcec Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 19 Dec 2025 15:12:11 -0800 Subject: [PATCH 11/11] Expose the env variable Signed-off-by: Huy Do --- .github/actions/upload-benchmark-results/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/upload-benchmark-results/action.yml b/.github/actions/upload-benchmark-results/action.yml index 1a0b3aa952..1a33fcd378 100644 --- a/.github/actions/upload-benchmark-results/action.yml +++ b/.github/actions/upload-benchmark-results/action.yml @@ -151,6 +151,7 @@ runs: shell: bash env: BENCHMARK_RESULTS_DIR: ${{ inputs.benchmark-results-dir }} + BENCHMARK_NAME: ${{ inputs.benchmark-name || '' }} DRY_RUN: ${{ inputs.dry-run }} # Additional information about the benchmarks BENCHMARK_METADATA: ${{ steps.gather-metadata.outputs.metadata }}