diff --git a/.bazelrc b/.bazelrc index 9ffd30da6..6881c576e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -12,6 +12,16 @@ build:clang-common --action_env=CXX=clang++-18 --host_action_env=CXX=clang++-18 build:rbe-toolchain-clang --action_env=CC=clang-18 --action_env=CXX=clang++-18 build:rbe-toolchain-arm64-clang --action_env=CC=clang-18 --action_env=CXX=clang++-18 +coverage --config=coverage + +build:coverage --combined_report=lcov +build:coverage --action_env=CC=clang-18 --host_action_env=CC=clang-18 +build:coverage --action_env=CXX=clang++-18 --host_action_env=CXX=clang++-18 +#build:coverage --test_arg="--log-path /dev/null" +build:coverage --test_tag_filters=-nocoverage,-fuzz_target +build:coverage --remote_download_minimal +build:coverage --coverage_report_generator=//tools/coverage:cilium_report_generator + # Use platforms based toolchain resolution build --incompatible_enable_cc_toolchain_resolution build --platform_mappings=bazel/platform_mappings diff --git a/.github/workflows/ci-check-coverage.yaml b/.github/workflows/ci-check-coverage.yaml new file mode 100644 index 000000000..923d47593 --- /dev/null +++ b/.github/workflows/ci-check-coverage.yaml @@ -0,0 +1,100 @@ +name: CI Check coverage +on: + pull_request: {} + +permissions: + # To be able to access the repository with actions/checkout + contents: read + pull-requests: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.after }} + cancel-in-progress: true + +jobs: + coverage: + timeout-minutes: 460 + name: Check coverage for pull request #${{ github.event.pull_request.number }} + runs-on: ubuntu-22.04 + steps: + - name: Clear Workspace + shell: bash + run: | + sudo rm -r /usr/local/.ghcup + sudo rm -r /usr/local/lib/android + sudo rm -r /opt/hostedtoolcache + - name: Check disk space + shell: bash + run: | + df . -h + - name: Check out the repository to the runner + uses: actions/checkout@v4 + - name: Setup gcloud credentials + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.CI_CILIUM_PROXY_SA_KEY }} + - name: set up google cloud sdk + uses: 'google-github-actions/setup-gcloud@v2' + with: + version: '>= 507.0.0' + - name: Install deps (for C++) + shell: bash + run: | + sudo apt-get update && \ + sudo apt-get upgrade -y --no-install-recommends && \ + sudo apt-get install -y --no-install-recommends \ + ca-certificates libc6-dev autoconf automake cmake coreutils curl git libtool make ninja-build patch patchelf \ + python3 python-is-python3 unzip virtualenv wget zip software-properties-common && \ + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc && \ + sudo apt-add-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" && \ + sudo apt-get update && \ + sudo apt-get install -y --no-install-recommends \ + clang-18 clangd-18 clang-tidy-18 clang-tools-18 llvm-18-dev lldb-18 lld-18 clang-format-18 libc++-18-dev libc++abi-18-dev libclang-rt-18-dev lcov && \ + sudo apt-get purge --auto-remove && \ + sudo apt-get clean && \ + sudo rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + cp /usr/lib/llvm-18/bin/llvm-cov /usr/local/bin + cp /usr/lib/llvm-18/bin/llvm-profdata /usr/local/bin + - name: Install Go + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + # renovate: datasource=golang-version depName=go + go-version: 1.24.6 + - name: Sync crate lockfile with bazel + shell: bash + run: | + CARGO_BAZEL_ISOLATED=0 CARGO_BAZEL_REPIN=1 bazel sync --only=crate_index + - name: Build proxylib + shell: bash + run: | + go version + make -C proxylib + - name: Generate coverage data + shell: bash + run: | + echo "Generating bazel coverage: " + ./tests/run_bazel_coverage.sh + ls -la /home/runner/work/cilium-proxy/cilium-proxy/generated/coverage/ + - name: Upload (sync) to GCS bucket + if: '!cancelled()' + shell: bash + run: | + UPLOAD_DIR="/home/runner/work/cilium-proxy/cilium-proxy/generated/coverage/" + SHA=${{ github.sha }} + BUCKET_PATH="${{ vars.GCS_ARTIFACT_BUCKET_COVERAGE }}/${SHA:0:7}/coverage" + echo "Uploading to gs://$BUCKET_PATH ..." + gsutil \ + -mq rsync \ + -dr "$UPLOAD_DIR" \ + "gs://$BUCKET_PATH" + echo "Artifacts uploaded to: https://storage.googleapis.com/$BUCKET_PATH/html/index.html" >&2 + strategy: + fail-fast: false + matrix: + include: + - target: coverage + name: Coverage + diskspace-hack: true + diskspace-hack-paths: | + /opt/hostedtoolcache + /usr/local/lib/android diff --git a/WORKSPACE b/WORKSPACE index b456c2609..eb5fdbfed 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -49,6 +49,10 @@ git_repository( # // clang-format on ) +load("//bazel:repo.bzl", "cilium_proxy_repo") + +cilium_proxy_repo() + # # Bazel does not do transitive dependencies, so we must basically # include all of Envoy's WORKSPACE file below, with the following diff --git a/bazel/repo.bzl b/bazel/repo.bzl new file mode 100644 index 000000000..424aa5a1d --- /dev/null +++ b/bazel/repo.bzl @@ -0,0 +1,75 @@ +# `@cilium_proxy_repo` repository rule for managing the repo and querying its metadata. + +def _cilium_proxy_repo_impl(repository_ctx): + """This provides information about the Envoy repository + + You can access the current project and api versions and the path to the repository in + .bzl/BUILD files as follows: + + ```starlark + load("@cilium_proxy_repo//:version.bzl", "VERSION", "API_VERSION") + ``` + + `*VERSION` can be used to derive version-specific rules and can be passed + to the rules. + + The `VERSION`s and also the local `PATH` to the repo can be accessed in + python libraries/binaries. By adding `@cilium_proxy_repo` to `deps` they become + importable through the `cilium_proxy_repo` namespace. + + As the `PATH` is local to the machine, it is generally only useful for + jobs that will run locally. + + This can be useful, for example, for bazel run jobs to run bazel queries that cannot be run + within the constraints of a `genquery`, or that otherwise need access to the repository + files. + + Project and repo data can be accessed in JSON format using `@cilium_proxy_repo//:project`, eg: + + ```starlark + load("@aspect_bazel_lib//lib:jq.bzl", "jq") + + jq( + name = "project_version", + srcs = ["@cilium_proxy_repo//:data"], + out = "version.txt", + args = ["-r"], + filter = ".version", + ) + + ``` + + """ + repo_version_path = repository_ctx.path(repository_ctx.attr.envoy_version) + api_version_path = repository_ctx.path(repository_ctx.attr.envoy_api_version) + version = repository_ctx.read(repo_version_path).strip() + api_version = repository_ctx.read(api_version_path).strip() + repository_ctx.file("version.bzl", "VERSION = '%s'\nAPI_VERSION = '%s'" % (version, api_version)) + repository_ctx.file("path.bzl", "PATH = '%s'" % repo_version_path.dirname) + repository_ctx.file("__init__.py", "PATH = '%s'\nVERSION = '%s'\nAPI_VERSION = '%s'" % (repo_version_path.dirname, version, api_version)) + repository_ctx.file("WORKSPACE", "") + repository_ctx.file("BUILD", ''' +load("@rules_python//python:defs.bzl", "py_library") +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") +load("//:path.bzl", "PATH") + +py_library( + name = "cilium_proxy_repo", + srcs = ["__init__.py"], + visibility = ["//visibility:public"], +) + +''') + +_cilium_proxy_repo = repository_rule( + implementation = _cilium_proxy_repo_impl, + attrs = { + #todo(nezdolik) add cilium version + "envoy_version": attr.label(default = "@envoy//:VERSION.txt"), + "envoy_api_version": attr.label(default = "@envoy//:API_VERSION.txt"), + }, +) + +def cilium_proxy_repo(): + if "cilium_proxy_repo" not in native.existing_rules().keys(): + _cilium_proxy_repo(name = "cilium_proxy_repo") diff --git a/tests/BUILD b/tests/BUILD index 594300e99..bf1d3c1e6 100644 --- a/tests/BUILD +++ b/tests/BUILD @@ -1,3 +1,4 @@ +load("@aspect_bazel_lib//lib:yq.bzl", "yq") load( "@envoy//bazel:envoy_build_system.bzl", "envoy_cc_test", @@ -18,6 +19,14 @@ api_cc_py_proto_library( srcs = ["bpf_metadata.proto"], ) +yq( + name = "coverage_config", + srcs = [":coverage.yaml"], + outs = ["cilium_coverage_config.json"], + args = ["-o=json"], + visibility = ["//visibility:public"], +) + envoy_cc_test_library( name = "accesslog_server_lib", srcs = ["accesslog_server.cc"], diff --git a/tests/coverage.yaml b/tests/coverage.yaml new file mode 100644 index 000000000..6c433c29c --- /dev/null +++ b/tests/coverage.yaml @@ -0,0 +1,7 @@ +thresholds: + total: 95.0 + per_directory: 95.0 + +directories: + cilium: 95.0 + linux: 95.0 diff --git a/tests/run_bazel_coverage.sh b/tests/run_bazel_coverage.sh new file mode 100755 index 000000000..b75c3fa49 --- /dev/null +++ b/tests/run_bazel_coverage.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +set -e -o pipefail +set +x +set -u + +LLVM_VERSION=${LLVM_VERSION:-"18.1.8"} +CLANG_VERSION=$(clang-18 --version | grep version | sed -e 's/\ *Ubuntu clang version \([0-9.]*\).*/\1/') +LLVM_COV_VERSION=$(llvm-cov --version | grep version | sed -e 's/\ *Ubuntu LLVM version \([0-9.]*\).*/\1/') +LLVM_PROFDATA_VERSION=$(llvm-profdata show --version | grep version | sed -e 's/\ *Ubuntu LLVM version \(.*\)/\1/') +SRCDIR=${SRCDIR:-"${PWD}"} + +#ERROR: clang version Ubuntu18.1.3 does not match expected 18.1.3 +if [[ "${CLANG_VERSION}" != "${LLVM_VERSION}" ]]; then + echo "ERROR: clang version ${CLANG_VERSION} does not match expected ${LLVM_VERSION}" >&2 + exit 1 +fi + +if [[ "${LLVM_COV_VERSION}" != "${LLVM_VERSION}" ]]; then + echo "ERROR: llvm-cov version ${LLVM_COV_VERSION} does not match expected ${LLVM_VERSION}" >&2 + exit 1 +fi + +if [[ "${LLVM_PROFDATA_VERSION}" != "${LLVM_VERSION}" ]]; then + echo "ERROR: llvm-profdata version ${LLVM_PROFDATA_VERSION} does not match expected ${LLVM_VERSION}" >&2 + exit 1 +fi + +COVERAGE_TARGET="${COVERAGE_TARGET:-}" +#TBD propogate any important global build options +read -ra BAZEL_BUILD_OPTIONS <<< "${BAZEL_BUILD_OPTION_LIST:-}" +read -ra BAZEL_GLOBAL_OPTIONS <<< "${BAZEL_GLOBAL_OPTION_LIST:-}" + +# This is the target that will be run to generate coverage data. It can be overridden by consumer +# projects that want to run coverage on a different/combined target. +# Command-line arguments take precedence over ${COVERAGE_TARGET}. +if [[ $# -gt 0 ]]; then + COVERAGE_TARGETS=("$@") +elif [[ -n "${COVERAGE_TARGET}" ]]; then + COVERAGE_TARGETS=("${COVERAGE_TARGET}") +else + COVERAGE_TARGETS=(//tests/...) +fi + +BAZEL_COVERAGE_OPTIONS=() +BAZEL_COVERAGE_OPTIONS+=(--heap_dump_on_oom) +BAZEL_COVERAGE_OPTIONS+=(--action_env=BAZEL_USE_LLVM_NATIVE_COVERAGE=1) +BAZEL_COVERAGE_OPTIONS+=(--combined_report=lcov) +BAZEL_COVERAGE_OPTIONS+=(--coverage_report_generator=//tools/coverage:cilium_report_generator) +BAZEL_COVERAGE_OPTIONS+=(--experimental_use_llvm_covmap) +BAZEL_COVERAGE_OPTIONS+=(--experimental_generate_llvm_lcov) +BAZEL_COVERAGE_OPTIONS+=(--experimental_split_coverage_postprocessing) +BAZEL_COVERAGE_OPTIONS+=(--experimental_fetch_all_coverage_outputs) +BAZEL_COVERAGE_OPTIONS+=(--collect_code_coverage) +BAZEL_COVERAGE_OPTIONS+=(--remote_download_minimal) +BAZEL_COVERAGE_OPTIONS+=(--copt=-DNDEBUG) +BAZEL_COVERAGE_OPTIONS+=(--build_tests_only) +#from envoy +BAZEL_COVERAGE_OPTIONS+=(--experimental_repository_cache_hardlinks) +BAZEL_COVERAGE_OPTIONS+=(--verbose_failures) +BAZEL_COVERAGE_OPTIONS+=(--experimental_generate_json_trace_profile) +BAZEL_COVERAGE_OPTIONS+=(--action_env=GCOV=llvm-profdata) +BAZEL_VALIDATE_OPTIONS=() + + +# Output unusually long logs due to trace logging. +BAZEL_COVERAGE_OPTIONS+=("--experimental_ui_max_stdouterr_bytes=80000000") +BAZEL_BUILD_OPTIONS+=("--remote_cache=https://storage.googleapis.com/cilium-proxy-bazel-remote-cache") +BAZEL_BUILD_OPTIONS+=("--google_default_credentials") + +COVERAGE_DIR="${SRCDIR}/generated/coverage" + +COVERAGE_DATA="${COVERAGE_DIR}/coverage.dat" + + +run_coverage() { + echo "Running bazel coverage with:" + echo " Options: ${BAZEL_BUILD_OPTIONS[*]} ${BAZEL_COVERAGE_OPTIONS[*]}" + echo " Targets: ${COVERAGE_TARGETS[*]}" + bazel coverage "${COVERAGE_TARGETS[@]}" "${BAZEL_BUILD_OPTIONS[@]}" "${BAZEL_COVERAGE_OPTIONS[@]}" --compiler=clang-18 --verbose_failures --sandbox_writable_path=$(bazel info output_path) --test_timeout=300 --local_test_jobs=1 --flaky_test_attempts=3 --instrument_test_targets --instrumentation_filter='^//' + + if [[ ! -e bazel-out/_coverage/_coverage_report.dat ]]; then + echo "ERROR: No coverage report found (bazel-out/_coverage/_coverage_report.dat)" >&2 + exit 1 + elif [[ ! -s bazel-out/_coverage/_coverage_report.dat ]]; then + echo "ERROR: Coverage report is empty (bazel-out/_coverage/_coverage_report.dat)" >&2 + exit 1 + fi +} + +unpack_coverage_results() { + rm -rf "${COVERAGE_DIR}" + mkdir -p "${COVERAGE_DIR}" + rm -f bazel-out/_coverage/_coverage_report.tar.zst + mv bazel-out/_coverage/_coverage_report.dat bazel-out/_coverage/_coverage_report.tar.zst + bazel run "${BAZEL_BUILD_OPTIONS[@]}" --nobuild_tests_only @envoy//tools/zstd -- -d -c "${PWD}/bazel-out/_coverage/_coverage_report.tar.zst" \ + | tar -xf - -C "${COVERAGE_DIR}" + COVERAGE_JSON="${COVERAGE_DIR}/coverage.json" +} + +validate_coverage() { + bazel run \ + "${BAZEL_BUILD_OPTIONS[@]}" \ + "${BAZEL_VALIDATE_OPTIONS[@]}" \ + --nobuild_tests_only \ + //tools/coverage:validate \ + "$COVERAGE_JSON" +} + +run_coverage +unpack_coverage_results +validate_coverage diff --git a/tools/coverage/BUILD b/tools/coverage/BUILD new file mode 100644 index 000000000..c6e552e89 --- /dev/null +++ b/tools/coverage/BUILD @@ -0,0 +1,99 @@ +load("@aspect_bazel_lib//lib:jq.bzl", "jq") +load("@cilium_proxy_repo//:path.bzl", "PATH") +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") +load("@envoy//bazel:envoy_build_system.bzl", "envoy_package") + +licenses(["notice"]) # Apache 2 + +envoy_package() + +exports_files(glob(["templates/*.html"])) + +genrule( + name = "grcov", + srcs = ["@grcov"], + outs = ["grcov_bin"], + cmd = "cp $< $@ && chmod +x $@", + executable = True, + visibility = ["//visibility:public"], +) + +jq( + name = "grcov_config", + srcs = [], + out = "grcov_config.json", + args = [ + "-sR", + "--arg", + "macros", + "$(location :templates/macros.html)", + "--arg", + "base", + "$(location :templates/base.html)", + "--arg", + "index", + "$(location :templates/index.html)", + ], + data = [ + ":templates/base.html", + ":templates/index.html", + ":templates/macros.html", + ], + expand_args = True, + filter = """ + {templates: { + "base.html": $base, + "index.html": $index, + "macros.html": $macros}} + """, +) + +# This is a bit of a hack, and totally non-hermetic +# grcov needs access to the source files, so the path +# is injected here. This means CoverageReportGenerator cannot +# run remote. +genrule( + name = "cilium_report_generator_script", + srcs = ["report_generator.sh.template"], + outs = ["report_generator.sh"], + cmd = "sed 's|@@WORKSPACE_PATH@@|$$(realpath %s)|g' $< > $@" % PATH, +) + +label_flag( + name = "config", + build_setting_default = "//tests:coverage_config", + visibility = ["//visibility:public"], +) + +sh_binary( + name = "cilium_report_generator", + srcs = [":cilium_report_generator_script"], + data = [ + ":config", + ":filter.jq", + ":grcov_bin", + ":grcov_config", + ":templates/base.html", + ":templates/index.html", + ":templates/macros.html", + "@envoy//tools/zstd", + "@jq_toolchains//:resolved_toolchain", + ], + toolchains = ["@jq_toolchains//:resolved_toolchain"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "validate", + srcs = [":validate.sh"], + data = [ + ":config", + "@jq_toolchains//:resolved_toolchain", + ], + env = { + "JQ_BIN": "$(JQ_BIN)", + "COVERAGE_CONFIG": "$(location :config)", + }, + toolchains = ["@jq_toolchains//:resolved_toolchain"], + visibility = ["//visibility:public"], +) diff --git a/tools/coverage/filter.jq b/tools/coverage/filter.jq new file mode 100644 index 000000000..6acd6f398 --- /dev/null +++ b/tools/coverage/filter.jq @@ -0,0 +1,235 @@ +# Extract coverage data from grcov's covdir JSON output with validation information + +def round_to_precision(precision): + . * pow(10; precision) + | round / pow(10; precision); + +def calculate_summary: + { + coverage_percent: (.coveragePercent | round_to_precision(1)), + lines_covered: .linesCovered, + lines_total: .linesTotal, + threshold: (if $config[0] then ($config[0].thresholds.total // 0) else 0 end), + threshold_reached: (.coveragePercent >= (if $config[0] then ($config[0].thresholds.total // 0) else 0 end)) + }; + +def get_default_threshold: + if $config[0] then + ($config[0].thresholds.per_directory // 0) + else 0 end; + +def directory_stats(path): + { + expected_percent: ((if $config[0] and $config[0].directories then $config[0].directories[path] else null end) // get_default_threshold), + coverage_percent: (.coveragePercent | round_to_precision(1)), + lines_covered: .linesCovered, + lines_total: .linesTotal + }; + +def extract_dirs(path): + if .children then + (if path != "" and .coveragePercent != null and .linesCovered != null and .linesTotal != null then + { + (path): directory_stats(path) + } + else empty end), + # Recurse into all subdirectories + (.children + | to_entries[] + | .key as $name + | .value + | if .children then + extract_dirs(if path == "" then $name else path + "/\($name)" end) + else empty end) + else empty end; + +def categorize_directories: + . as $coverage_data + | get_default_threshold as $default + | {failed_coverage: [], + low_coverage: [], + excellent_coverage: [], + high_coverage_adjustable: [] + } as $categories + | + # Process each source directory + ($source_directories + | split("\n") + | map(select(. != ""))) as $dirs + | + reduce $dirs[] as $dir ($categories; + ($coverage_data[$dir] // null) as $dir_data + | if $dir_data then + ($dir_data.expected_percent) as $threshold + | ($dir_data.coverage_percent) as $coverage + | + if $coverage < $threshold then + .failed_coverage += [{ + directory: $dir, + coverage_percent: $coverage, + threshold: $threshold + }] + else + if $threshold < $default then + # Directory has a configured exception (lower threshold) + .low_coverage += [{ + directory: $dir, + coverage_percent: $coverage, + threshold: $threshold + }] + | + # Check if significantly higher than configured threshold + if $coverage > $threshold then + .high_coverage_adjustable += [{ + directory: $dir, + coverage_percent: $coverage, + threshold: $threshold + }] + else . end + elif $coverage >= $default then + # Excellent coverage (meets default threshold) + .excellent_coverage += [{ + directory: $dir, + coverage_percent: $coverage + }] + else . end + end + else . end + ); + +def should_process_directories: + ($source_directories != "" and $source_directories != null) + or ($config[0] and $config[0].directories and ($config[0].directories | length) > 0); + +def generate_summary_message: + . as $data + | if $data.summary.threshold_reached then + "Code coverage \(.summary.coverage_percent)% is good and higher than limit of \(.summary.threshold)%" + else + "Code coverage \(.summary.coverage_percent)% is lower than limit of \(.summary.threshold)%" + end; + +def generate_summary_failed: + . as $data + | (if ($data.validation.failed_coverage | length) > 0 then + ["FAILED: Directories not meeting coverage thresholds:"] + + ($data.validation.failed_coverage + | map(" ✗ \(.directory): \(.coverage_percent | tostring)% (threshold: \(.threshold | tostring)%)")) + + [""] + else [] end) + | join("\n"); + +def generate_summary_adjustable: + . as $data + | (if ($data.validation.high_coverage_adjustable | length) > 0 then + ["WARNING: Coverage in the following directories may be adjusted up:"] + + ($data.validation.high_coverage_adjustable + | map(" ⬆ \(.directory): \(.coverage_percent | tostring)% (current threshold: \(.threshold | tostring)%)")) + + [""] + else [] end) + | join("\n"); + +def generate_summary_excellent: + . as $data + | (if ($data.validation.excellent_coverage | length) > 0 then + ["Directories with excellent coverage (>= \($data.default_threshold | tostring)%):"] + + ($data.validation.excellent_coverage + | map(" ✓ \(.directory): \(.coverage_percent | tostring)%")) + + [""] + else [] end) + | join("\n"); + +def generate_summary_low: + . as $data + | (if ($data.validation.low_coverage | length) > 0 then + ["Directories with known low coverage (meeting configured thresholds):"] + + ($data.validation.low_coverage + | map(" ⚠ \(.directory): \(.coverage_percent | tostring)% (configured threshold: \(.threshold | tostring)%)")) + + [""] + else [] end) + | join("\n"); + +def generate_summary_adjust_message: + . as $data + | (if ($data.validation_summary.adjustable_count) > 0 then + "Can be adjusted up: \($data.validation_summary.adjustable_count | tostring)" + else "" end); + +def generate_summary_directory_error: + . as $data + | (if ($data.validation_summary.all_passed | not) then + "ERROR: Coverage check failed. Some directories are below their thresholds." + else "" end); + +def generate_summary_report: + . as $data + | " +================== Per-Directory Coverage Report ================== + +\(generate_summary_excellent) +\(generate_summary_low) +\(generate_summary_adjustable) +\(generate_summary_failed) +================================================================== +Overall Coverage: \($data.summary.coverage_percent)% +Source directories checked: \($data.validation_summary.total_directories), +Failed: \($data.validation_summary.failed_count), +Low coverage (configured): \($data.validation_summary.low_coverage_count), +Excellent coverage: \($data.validation_summary.excellent_coverage_count) +\(generate_summary_adjust_message) +================================================================== + +\(generate_summary_message) +\(generate_summary_directory_error) +"; + +# Main processing +{ + summary: calculate_summary, + source_directories: ( + $source_directories + | split("\n") + | map(select(. != ""))), + # Only calculate per_directory_coverage if we have directories to process + per_directory_coverage: ( + if should_process_directories then + [.children + | to_entries[] + | .key as $root_dir + | .value + | extract_dirs($root_dir)] + | add // {} + else + {} + end + ), + default_threshold: get_default_threshold, + # Include coverage config if available + coverage_config: (if $config[0] then $config[0] else null end) +} +| +# Add validation results only if we have directories to validate +. as $base +| if should_process_directories and $base.source_directories != [] then + .per_directory_coverage as $per_dir + | $base + {validation: ($per_dir | categorize_directories), + validation_summary: ( + $per_dir + | categorize_directories + | {total_directories: ( + (.failed_coverage + .low_coverage + .excellent_coverage) + | map(.directory) + | unique + | length), + failed_count: (.failed_coverage | length), + low_coverage_count: (.low_coverage | length), + excellent_coverage_count: (.excellent_coverage | length), + adjustable_count: (.high_coverage_adjustable | length), + all_passed: ((.failed_coverage | length) == 0)})} + else $base end +| . as $result +| $result + {summary_message: generate_summary_message} +| . as $result +| if $result.validation then + $result + {summary_report: generate_summary_report} + else . end diff --git a/tools/coverage/report_generator.sh.template b/tools/coverage/report_generator.sh.template new file mode 100755 index 000000000..4c5746818 --- /dev/null +++ b/tools/coverage/report_generator.sh.template @@ -0,0 +1,151 @@ +#!/bin/bash +set -eo pipefail + +WORKSPACE_PATH=/home/runner/work/cilium-proxy/cilium-proxy + +while [[ $# -gt 0 ]]; do + case "$1" in + --reports_file=*) + REPORTS_FILE="${1#*=}" + ;; + --output_file=*) + OUTPUT_FILE="${1#*=}" + ;; + *) + echo "Unknown argument: $1" >&2 + exit 1 + ;; + esac + shift +done + +find_file() { + # kinda wierd to use PYTHON_RUNFILES but that is what is available in the + # bazel generator customization + find "${PYTHON_RUNFILES}" -type f -o -type l -name "${1}" -path "*/tools/coverage/**/*" | head -1 +} + +GRCOV="$(find_file grcov_bin)" +GRCOV_CONFIG="$(find_file grcov_config.json)" +COVERAGE_CONFIG="$(find_file "*cilium_coverage_config.json")" +JQ_BIN="$(find_file jq)" +ZSTD_BIN="$(find_file zstd)" +TEMPLATES_DIR="$(dirname "$(find_file base.html)")" +JQ_FILTER="$(find_file "filter.jq")" +INFO_FILES=() +OUTPUT_DIR="output" + +# This is a little hacky +cwd="$(basename "$PWD")" +if [[ "$cwd" == "envoy_mobile" ]]; then + WORKSPACE_PATH="${WORKSPACE_PATH}/mobile" +fi +if [[ -z "$WORKSPACE_PATH" || ! -e "$WORKSPACE_PATH" ]]; then + echo "WARNING: WORKSPACE not set or not found, unable to process coverage" >&2 + exit 1 +fi +if [[ -z "$JQ_BIN" || ! -x "$JQ_BIN" ]]; then + echo "WARNING: jq not found, unable to process coverage" >&2 + exit 1 +fi +if [[ -z "$JQ_FILTER" || ! -e "$JQ_FILTER" ]]; then + echo "WARNING: JQ filter not set or not found, unable to process coverage" >&2 + exit 1 +fi +if [[ -z "$ZSTD_BIN" || ! -x "$ZSTD_BIN" ]]; then + echo "WARNING: zstd not found, unable to process coverage" >&2 + exit 1 +fi +if [[ -z "$GRCOV" || ! -x "$GRCOV" ]]; then + echo "WARNING: grcov not found, unable to process coverage" >&2 + exit 1 +fi +if [[ -z "$GRCOV_CONFIG" || ! -e "$GRCOV_CONFIG" ]]; then + echo "WARNING: GRCOV_CONFIG not found, unable to process coverage" >&2 + exit 1 +fi +if [[ -z "$COVERAGE_CONFIG" || ! -e "$COVERAGE_CONFIG" ]]; then + echo "WARNING: COVERAGE_CONFIG not found, unable to process coverage" >&2 + exit 1 +fi +if [[ -z "$REPORTS_FILE" || ! -e "$REPORTS_FILE" ]]; then + echo "WARNING: REPORTS_FILE not found, unable to process coverage" >&2 + exit 1 +fi +if [[ -z "$OUTPUT_FILE" ]]; then + echo "WARNING: OUTPUT_FILE not found, unable to process coverage" >&2 + exit 1 +fi + + +create_config() { + # shellcheck disable=SC2016 + $JQ_BIN --arg tpl_dir "$TEMPLATES_DIR" \ + '.templates["base.html"] = ($tpl_dir + "/base.html") + | .templates["index.html"] = ($tpl_dir + "/index.html") + | .templates["macros.html"] = ($tpl_dir + "/macros.html")' \ + "$GRCOV_CONFIG" > grcov_config_updated.json +} + +run_grcov() { + rm -rf "${OUTPUT_DIR}" + mkdir -p "${OUTPUT_DIR}" + $GRCOV \ + "${INFO_FILES[@]}" \ + --precision 1 \ + -s "${WORKSPACE_PATH}" \ + -t html,covdir \ + --output-config-file grcov_config_updated.json \ + -o "${OUTPUT_DIR}" +} + +create_report() { + local source_directories="" + if [[ -n "$COVERAGE_CONFIG" && -f "$COVERAGE_CONFIG" ]]; then + if $JQ_BIN -e '.directories' "$COVERAGE_CONFIG" >/dev/null 2>&1; then + source_directories=$(find "${WORKSPACE_PATH}/cilium" -name "*.cc" -type f -printf '%h\n' | sed "s|^${WORKSPACE_PATH}/||" | sort -u) + fi + fi + echo "Creating report for source directories: $source_directories" + $JQ_BIN \ + --slurpfile config "$COVERAGE_CONFIG" \ + --arg source_directories "$source_directories" \ + -f "$JQ_FILTER" \ + "${OUTPUT_DIR}/covdir" > "${OUTPUT_DIR}/coverage.json" +} + +find_coverage_files() { + while IFS= read -r dat_file; do + if [[ ! -f "$dat_file" ]]; then + continue + fi + dat_file_abs=$(realpath "$dat_file") + dat_dir=$(dirname "$dat_file_abs") + dat_basename=$(basename "$dat_file_abs") + info_basename="${dat_basename%.dat}.info" + pushd "$dat_dir" > /dev/null + ln -sf "$dat_basename" "$info_basename" + popd > /dev/null + info_link="${dat_dir}/${info_basename}" + INFO_FILES+=("$info_link") + done < "$REPORTS_FILE" +} + +create_output() { + rm -rf "${OUTPUT_DIR}/html/badges" + rm -rf "${OUTPUT_DIR}/html/coverage.json" + tar -cf - -C "${OUTPUT_DIR}" . | $ZSTD_BIN -o output.tar.zst + mv output.tar.zst "$OUTPUT_FILE" +} + +generate_report() { + echo "Found coverage config:" + cat $COVERAGE_CONFIG + create_config + find_coverage_files + run_grcov + create_report + create_output +} + +generate_report diff --git a/tools/coverage/templates/base.html b/tools/coverage/templates/base.html new file mode 100644 index 000000000..0e78eb35e --- /dev/null +++ b/tools/coverage/templates/base.html @@ -0,0 +1,23 @@ + + + + {%- block head -%} + + + {% block title %}{% endblock title %} + + {%- endblock head -%} + + +
+ {%- block content -%}{%- endblock content -%} +
+ + + diff --git a/tools/coverage/templates/index.html b/tools/coverage/templates/index.html new file mode 100644 index 000000000..623550072 --- /dev/null +++ b/tools/coverage/templates/index.html @@ -0,0 +1,39 @@ +{% import "macros.html" as macros %} +{% extends "base.html" %} + +{% block title %}Cilium-proxy coverage report - {{ current }} {% endblock title %} + +{%- block content -%} +{{ macros::summary(parents=parents, stats=stats, precision=precision) }} + + + + + + + {% if branch_enabled %} + + {% endif %} + + + + {%- if kind == "Directory" -%} + {%- for item, info in items -%} + {% if info.abs_prefix and info.abs_prefix != "" %} + {{ macros::stats_line(name=item, url=info.abs_prefix~item~"/index.html", stats=info.stats, precision=precision) }} + {% else %} + {{ macros::stats_line(name=item, url=item~"/index.html", stats=info.stats, precision=precision) }} + {% endif %} + {%- endfor -%} + {%- else -%} + {%- for item, info in items -%} + {% if info.abs_prefix and info.abs_prefix != "" %} + {{ macros::stats_line(name=item, url=info.abs_prefix~"/"~item~".html", stats=info.stats, precision=precision) }} + {% else %} + {{ macros::stats_line(name=item, url=item~".html", stats=info.stats, precision=precision) }} + {% endif %} + {%- endfor -%} + {%- endif -%} + +
{{ kind }}Line CoverageFunctionsBranches
+{%- endblock content -%} diff --git a/tools/coverage/templates/macros.html b/tools/coverage/templates/macros.html new file mode 100644 index 000000000..0d8373e86 --- /dev/null +++ b/tools/coverage/templates/macros.html @@ -0,0 +1,65 @@ +{% macro summary_line(kind, covered, total, precision) %} +{%- set per = percent(num=covered, den=total) -%} +
+
+

{{ kind | capitalize }}

+

+ {{ per | round(precision=precision) }} %

+
+
+{% endmacro -%} + +{% macro summary(parents, stats, precision) %} + + +{% endmacro %} + +{% macro stats_line(name, url, stats, precision) %} +{%- set lines_per = percent(num=stats.covered_lines, den=stats.total_lines) -%} +{%- set lines_sev = lines_per | severity(kind="lines") -%} +{%- set functions_per = percent(num=stats.covered_funs, den=stats.total_funs) -%} +{%- set functions_sev = functions_per | severity(kind="functions") -%} +{% if branch_enabled %} +{%- set branches_per = percent(num=stats.covered_branches, den=stats.total_branches) -%} +{%- set branches_sev = branches_per | severity(kind="branches") -%} +{% endif %} + + {{ name }} + + + + {{ lines_per | round(precision=precision) }}% + + + + {{ lines_per | round(precision=precision) }}% + + + {{ stats.covered_lines }} / {{ stats.total_lines }} + + + {{ functions_per | round(precision=precision) }}% + {{ stats.covered_funs }} / {{ stats.total_funs }} + + {% if branch_enabled %} + {{ branches_per | round(precision=precision) }}% + {{ stats.covered_branches }} / {{ stats.total_branches }} + {% endif %} + +{% endmacro %} diff --git a/tools/coverage/validate.sh b/tools/coverage/validate.sh new file mode 100755 index 000000000..42ec48f36 --- /dev/null +++ b/tools/coverage/validate.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -eo pipefail + +COVERAGE_JSON="${1}" + +if [[ -z "$COVERAGE_JSON" || ! -f "$COVERAGE_JSON" ]]; then + echo "ERROR: Coverage JSON file not provided or not found: $COVERAGE_JSON" >&2 + exit 1 +fi + +if [[ -z "$COVERAGE_CONFIG" || ! -f "$COVERAGE_CONFIG" ]]; then + echo "ERROR: Coverage config file not found: $COVERAGE_CONFIG" >&2 + exit 1 +fi + +THRESHOLD_REACHED="$($JQ_BIN '.summary.threshold_reached' "$COVERAGE_JSON")" +SUMMARY_MESSAGE="$($JQ_BIN -r '.summary_message' "$COVERAGE_JSON")" + + +FAILED_COUNT=$($JQ_BIN '.validation_summary.failed_count' "$COVERAGE_JSON") +ADJUSTABLE_COUNT=$($JQ_BIN '.validation_summary.adjustable_count' "$COVERAGE_JSON") +if [[ "$FAILED_COUNT" -gt 0 || "$ADJUSTABLE_COUNT" -gt 0 ]]; then + $JQ_BIN -r '.summary_report' "$COVERAGE_JSON" +else + echo "$SUMMARY_MESSAGE" +fi +if [[ "$FAILED_COUNT" -gt 0 || "${THRESHOLD_REACHED}" == "false" ]]; then + exit 1 +fi +exit 0