From b788ac14613a2f437c752a3f81107ca298f022b0 Mon Sep 17 00:00:00 2001 From: KPD Date: Mon, 2 Jun 2025 09:03:01 +0000 Subject: [PATCH 1/2] adding ci files --- .github/actions/archive/action.yml | 24 ++++ .github/actions/cleanup/action.yml | 24 ++++ .github/actions/setup/action.yml | 181 +++++++++++++++++++++++++++++ .github/actions/test/action.yml | 44 +++++++ .github/workflows/main.yml | 107 +++++++++++++++++ .github/workflows/manual.yml | 58 +++++++++ .github/workflows/schedule.yml | 50 ++++++++ scripts/github_output.sh | 9 ++ scripts/korg-releases.py | 92 +++++++++++++++ 9 files changed, 589 insertions(+) create mode 100644 .github/actions/archive/action.yml create mode 100644 .github/actions/cleanup/action.yml create mode 100644 .github/actions/setup/action.yml create mode 100644 .github/actions/test/action.yml create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/manual.yml create mode 100644 .github/workflows/schedule.yml create mode 100755 scripts/github_output.sh create mode 100755 scripts/korg-releases.py diff --git a/.github/actions/archive/action.yml b/.github/actions/archive/action.yml new file mode 100644 index 00000000000000..40a84b864e1ae6 --- /dev/null +++ b/.github/actions/archive/action.yml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Archive results +description: Archive kdevops results in https://github.com/linux-kdevops/kdevops-results-archive.git +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' +runs: + using: "composite" + steps: + - name: Get systemd journal files + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make journal-dump + + - name: Build our kdevops archive results + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make ci-archive CI_WORKFLOW="${{ inputs.ci_workflow }}" + diff --git a/.github/actions/cleanup/action.yml b/.github/actions/cleanup/action.yml new file mode 100644 index 00000000000000..18a8656aa165b6 --- /dev/null +++ b/.github/actions/cleanup/action.yml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Cleanup kdevops VMs +description: Destroy VMs and cleanup workspace + +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' + +runs: + using: "composite" + steps: + - name: Run kdevops make destroy + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make destroy + + - name: Remove working-directory + shell: bash + run: | + rm --recursive --force --verbose ${{ inputs.dir }} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000000000..37d06d9e0c8d78 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,181 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Setup kdevops +description: Setup kdevops workspace + +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' + kernel_tree: + required: false + type: string + default: 'linux' + kernel_ref: + required: false + type: string + default: 'master' + ci_workflow: + required: false + type: string + default: 'demo' + +runs: + using: "composite" + steps: + - name: Create workspace directory + shell: bash + run: | + pwd + rm --recursive --force --verbose ${{ inputs.dir }} + mkdir --parent --verbose ${{ inputs.dir }} + find . + + - name: Configure git + shell: bash + run: | + git config --global --add safe.directory '*' + git config --global user.name "kdevops" + git config --global user.email "kdevops@lists.linux.dev" + + - name: Checkout kdevops + working-directory: ${{ inputs.dir }} + shell: bash + run: | + rm --recursive --force --verbose kdevops/ + git clone https://github.com/dkruces/kdevops.git --branch ci-workflow kdevops + + - name: Checkout custom branch with delta on kdevops/linux + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + set -euxo pipefail + LINUX_TREE="/mirror/${{ inputs.kernel_tree }}.git" + LINUX_TREE_REF="${{ inputs.kernel_ref }}" + git clone $LINUX_TREE linux + cd linux + git checkout $LINUX_TREE_REF + git log -1 + + - name: Make sure our repo kdevops defconfig exists + id: defconfig + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + set -euxo pipefail + if [[ -z "${{ inputs.kdevops_defconfig }}" ]]; then + KDEVOPS_DEFCONFIG=${{ inputs.ci_workflow }} + else + KDEVOPS_DEFCONFIG="${{ inputs.kdevops_defconfig }}" + fi + + if [[ ! -f defconfigs/$KDEVOPS_DEFCONFIG ]]; then + echo "kdevops lacks a defconfig for this repository, expected to find: defconfigs/$KDEVOPS_DEFCONFIG" + exit 1 + fi + + "${{ github.workspace }}/scripts/github_output.sh" KDEVOPS_DEFCONFIG "$KDEVOPS_DEFCONFIG" + + - name: Initialize CI metadata for kdevops-results-archive for linux + id: metadata + working-directory: ${{ inputs.dir }}/kdevops/linux + shell: bash + run: | + set -euxo pipefail + echo "${{ inputs.kernel_tree }}" > ../ci.trigger + echo "testing" > ../ci.subject + echo "${{ inputs.kernel_ref }}" > ../ci.ref + + RELEVANT_GIT_TAG=$(cat ../ci.ref) + RELEVANT_GIT_REF=$(git rev-parse --short=12 $RELEVANT_GIT_TAG) + + "${{ github.workspace }}/scripts/github_output.sh" LINUX_GIT_REF "$RELEVANT_GIT_REF" + "${{ github.workspace }}/scripts/github_output.sh" LINUX_GIT_TAG "$RELEVANT_GIT_TAG" + + # Start out pessimistic + echo "unknown" > ../ci.result + echo "Nothing to write home about." > ../ci.commit_extra + + - name: Run a quick Linux kernel defconfig build test + working-directory: ${{ inputs.dir }}/kdevops/linux + env: + LINUX_GIT_TAG: ${{ steps.metadata.outputs.LINUX_GIT_TAG }} + shell: bash + run: | + set -euxo pipefail + git reset --hard "$LINUX_GIT_TAG" + make defconfig + make -j$(nproc) + + - name: Run kdevops make defconfig-repo + working-directory: ${{ inputs.dir }}/kdevops + env: + LINUX_GIT_TAG: ${{ steps.metadata.outputs.LINUX_GIT_TAG }} + LINUX_GIT_REF: ${{ steps.metadata.outputs.LINUX_GIT_REF }} + KDEVOPS_DEFCONFIG: ${{ steps.defconfig.outputs.KDEVOPS_DEFCONFIG }} + shell: bash + run: | + LINUX_TREE="/mirror/${{ inputs.kernel_tree }}.git" + LINUX_TREE_REF="$LINUX_GIT_TAG" + + # We make the compromise here to use a relevant git tag for the + # host prefix so that folks can easily tell what exact kernel tree + # is being tested by using the relevant git ref. That is, if you + # pushed a tree with the .github/ directory as the top of the tree, + # that commit will not be used, we'll use the last one as that is + # the relevant git ref we want to annotate a test for. + # + # The compromise here is that we expect no two same identical tests + # on the same self-hosted server. We could extend this with something + # like github.run_id but its not yet clear if we can have kdevops + # hosts with a bundled prefix ID like that ref-runid-testname. Tests + # have been done with the full lenght sha1sum though and we know that + # does work. + KDEVOPS_HOSTS_PREFIX="$LINUX_GIT_REF" + + echo "Going to use defconfig-$KDEVOPS_DEFCONFIG" + + echo "Linux tree: $LINUX_TREE" + echo "Linux trigger ref: $LINUX_TREE_REF" + echo "Linux tag: $LINUX_GIT_TAG" + echo "Runner ID: ${{ github.run_id }}" + echo "kdevops host prefix: $KDEVOPS_HOSTS_PREFIX" + echo "kdevops defconfig: defconfig-$KDEVOPS_DEFCONFIG" + + KDEVOPS_ARGS="\ + KDEVOPS_HOSTS_PREFIX=$KDEVOPS_HOSTS_PREFIX \ + LINUX_TREE=$LINUX_TREE \ + LINUX_TREE_REF=$LINUX_TREE_REF \ + ANSIBLE_CFG_CALLBACK_PLUGIN="debug" \ + defconfig-$KDEVOPS_DEFCONFIG" + echo "Going to run:" + echo "make $KDEVOPS_ARGS" + + make $KDEVOPS_ARGS + + - name: Run kdevops make + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make -j$(nproc) + + - name: Run kdevops make bringup + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + ls -ld linux + make destroy + make bringup + + - name: Build linux and boot test nodes on test kernel + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make linux + + - name: Build required ci tests + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make ci-build-test CI_WORKFLOW=${{ inputs.ci_workflow }} diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml new file mode 100644 index 00000000000000..75b95c3721c2a5 --- /dev/null +++ b/.github/actions/test/action.yml @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Setup kdevops +description: Setup kdevops workspace + +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' + ci_workflow: + required: false + type: string + default: 'demo' + +runs: + using: "composite" + steps: + - name: Run CI tests + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make ci-test CI_WORKFLOW="${{ inputs.ci_workflow }}" + + echo -e "Kernel tests results:\n" > ci.commit_extra + + - name: Generate CI commit info + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + find workflows/blktests/results/last-run/ -name '*.dmesg.log' \ + -exec tail -n 1 {} + >> ci.commit_extra + + echo -e "\n\n" >> ci.commit_extra + echo -e "Userspace test results:\n" >> ci.commit_extra + find workflows/blktests/results/last-run/ -name '*.userspace.log' \ + -exec tail -n 1 {} + >> ci.commit_extra + echo -e "\n\n" >> ci.commit_extra + + if grep -i -q "fail" ci.commit_extra ; then + echo "fail" > ci.result + else + echo "ok" > ci.result + fi diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000000000..13dba4eb512f0f --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Run kdevops CI Workflow - Reusable + +on: + workflow_call: + inputs: + ci_workflow: + description: "CI Workflow" + required: true + default: 'blktests_nvme' + type: string + kernel_tree: + description: "Linux kernel tree to use" + required: true + default: 'linux' + type: string + kernel_ref: + description: "Linux tree git reference (branch/tag/commit-id)" + required: true + default: 'master' + type: string + +jobs: + check_ref: + name: Check Linux kernel Git Reference + runs-on: [self-hosted] + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Check kernel_ref exists + id: check_kernel_ref + run: | + set -euxo pipefail + + ref="${{ github.event.inputs.kernel_ref }}" + tree="${{ github.event.inputs.kernel_tree }}" + mirror="/mirror/${tree}.git" + ls_remote="$(git ls-remote "$mirror" "refs/*/${ref}" || true)" + contains_branch="$(git -C "$mirror" branch --contains "${ref}" || true)" + contains_tag="$(git -C "$mirror" branch --contains "${ref}" || true)" + + if [ -z "$ls_remote" ] && [ -z "$contains_branch" ] && [ -z "$contains_tag" ]; then + echo "Linux kernel ${ref} does not exist." + exit 1 + fi + + setup: + name: Setup kdevops workspace + runs-on: [self-hosted] + needs: [check_ref] + steps: + - name: Checkout kdevops-ci + uses: actions/checkout@v4 + + - name: kdevops setup + uses: ./.github/actions/setup + with: + dir: ${{ inputs.ci_workflow }} + kernel_tree: ${{ inputs.kernel_tree }} + kernel_ref: ${{ inputs.kernel_ref }} + ci_workflow: ${{ inputs.ci_workflow }} + + test: + name: Run kdevops ci-test + runs-on: [self-hosted] + needs: [setup] + steps: + - name: kdevops ci-test + uses: ./.github/actions/test + with: + dir: ${{ inputs.ci_workflow }} + ci_workflow: ${{ inputs.ci_workflow }} + + archive: + name: Archive kdevops + runs-on: [self-hosted] + needs: [setup, test] + steps: + - name: Start SSH Agent + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Archive ci-test results + uses: ./.github/actions/archive + with: + dir: ${{ inputs.ci_workflow }} + + - name: Upload our kdevops results archive + uses: actions/upload-artifact@v4 + with: + name: kdevops-ci-results + path: ${{ inputs.ci_workflow }}/kdevops/archive/*.zip + + cleanup: + name: Cleanup kdevops workspace + runs-on: [self-hosted] + needs: [setup, test, archive] + if: always() + steps: + - name: kdevops cleanup + uses: ./.github/actions/cleanup + with: + dir: ${{ inputs.ci_workflow }} + ci_workflow: ${{ inputs.ci_workflow }} diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml new file mode 100644 index 00000000000000..7b570265c45685 --- /dev/null +++ b/.github/workflows/manual.yml @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Run kdevops CI Workflow - Manual + +on: + workflow_dispatch: + inputs: + ci_workflow: + description: "CI Workflow" + required: true + default: 'blktests_nvme' + type: choice + options: + - blktests + - blktests_block + - blktests_loop + - blktests_meta + - blktests_nbd + - blktests_nvme + - blktests_nvmemp + - blktests_scsi + - blktests_srp + - blktests_zbd + - tmpfs + - tmpfs_default + - tmpfs_huge + - tmpfs_noswap + - linux-btrfs-kpd + - linux-ext4-kpd + - linux-firmware-kpd + - linux-mm-kpd + - linux-modules-kpd + - linux-xfs-kpd + - selftests + kernel_tree: + description: "Linux kernel tree to use" + required: true + default: 'linux' + type: choice + options: + - linux + - linux-next + - linux-stable + kernel_ref: + description: "Linux tree git reference (branch/tag/commit-id)" + required: true + default: 'master' + type: string + +jobs: + manual: + name: Manual kdevops CI + uses: ./.github/workflows/main.yml + secrets: inherit + with: + ci_workflow: ${{ inputs.ci_workflow }} + kernel_ref: ${{ inputs.kernel_ref }} + kernel_tree: ${{ inputs.kernel_tree }} diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml new file mode 100644 index 00000000000000..d0221550845fa6 --- /dev/null +++ b/.github/workflows/schedule.yml @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Run kdevops CI Workflow - Schedule + +on: + schedule: + - cron: '0 14 * * *' + + workflow_dispatch: + +jobs: + check_ref: + name: Check Linux kernel Git Reference + outputs: + kernel_ref: ${{ steps.check_kernel_ref.outputs.kernel_ref }} + kernel_tree: ${{ steps.check_kernel_ref.outputs.kernel_tree }} + runs-on: [self-hosted] + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Generate kernel_ref + id: check_kernel_ref + run: | + set -euxo pipefail + day_of_week=$(date +%u) + + kernel_ref=$(./scripts/korg-releases.py --moniker linux-next) + kernel_tree=linux-next + if [ "$day_of_week" -eq 1 ]; then + kernel_ref=$(./scripts/korg-releases.py --moniker mainline) + kernel_tree=linux + fi + + "${{ github.workspace }}/scripts/github_output.sh" kernel_ref "$kernel_ref" + "${{ github.workspace }}/scripts/github_output.sh" kernel_tree "$kernel_tree" + + schedule: + name: Scheduled kdevops CI + needs: [check_ref] + runs-on: [self-hosted] + secrets: inherit + uses: ./.github/workflows/main.yml + strategy: + matrix: + ci_workflow: [blktests-block, blktests-loop, blktests-meta] + with: + ci_workflow: ${{ matrix.ci_workflow }} + kernel_ref: ${{ needs.check_ref.outputs.kernel_ref }} + kernel_tree: ${{ needs.check_ref.outputs.kernel_tree }} diff --git a/scripts/github_output.sh b/scripts/github_output.sh new file mode 100755 index 00000000000000..711306c38cb5ff --- /dev/null +++ b/scripts/github_output.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# +# Usage: ./github_output.sh key value +set -euxo pipefail + +key="$1" +value="$2" + +echo "$key=$value" >> "$GITHUB_OUTPUT" diff --git a/scripts/korg-releases.py b/scripts/korg-releases.py new file mode 100755 index 00000000000000..bce182e28d7926 --- /dev/null +++ b/scripts/korg-releases.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: copyleft-next-0.3.1 +import sys +import logging +import argparse +import json +import urllib.request +import socket +import re + + +def parser(): + parser = argparse.ArgumentParser(description="kernel.org/releases.json checker") + parser.add_argument("--debug", action="store_true", help="debug") + parser.add_argument( + "--moniker", + help="moniker (mainline, stable, longterm or linux-next)", + required=True, + ) + parser.add_argument( + "--pname", + help="project name for User-Agent request", + default="kdevops", + ) + parser.add_argument( + "--pversion", + help="project version for User-Agent request", + default="5.0.2", + ) + return parser + + +def _check_connection(host, port, timeout=2): + try: + with socket.create_connection((host, port), timeout): + logging.debug(f"Connection to {host} on port {port} succeeded!") + return True + except (socket.timeout, socket.error) as e: + logging.debug(f"Connection to {host} on port {port} failed: {e}") + return False + + +def kreleases(args) -> None: + """Get the latest kernel releases from kernel.org/releases.json""" + + reflist = [] + if _check_connection("kernel.org", 80): + _url = "https://www.kernel.org/releases.json" + req = urllib.request.Request( + _url, + headers={ + "User-Agent": f"{args.pname}/{args.pversion} (kdevops@lists.linux.dev)" + }, + ) + with urllib.request.urlopen(req) as url: + data = json.load(url) + + for release in data["releases"]: + if release["moniker"] == args.moniker: + # Check if release.json is aa.bb.cc type + if re.compile(r'^\d+\.\d+(\.\d+|-rc\d+)?$').match(release["version"]): + reflist.append("v" + release["version"]) + else: + reflist.append(release["version"]) + + logging.debug(f"{reflist}") + for r in reflist: + print(r) + + +def main() -> None: + """Kconfig choice generator for git refereces""" + log = logging.getLogger() + log.setLevel(logging.INFO) + p = parser() + args, _ = p.parse_known_args() + if args.debug: + log.setLevel(logging.DEBUG) + + kreleases(args) + + +if __name__ == "__main__": + ret = 0 + try: + main() + except Exception: + ret = 1 + import traceback + + traceback.print_exc() + sys.exit(ret) From 784f0b9aef011d16b05ea062a52e79db2f1d2d69 Mon Sep 17 00:00:00 2001 From: Petr Pavlu Date: Thu, 6 Mar 2025 11:36:55 +0100 Subject: [PATCH 2/2] module: Taint the kernel when write-protecting ro_after_init fails In the unlikely case that setting ro_after_init data to read-only fails, it is too late to cancel loading of the module. The loader then issues only a warning about the situation. Given that this reduces the kernel's protection, it was suggested to make the failure more visible by tainting the kernel. Allow TAINT_BAD_PAGE to be set per-module and use it in this case. The flag is set in similar situations and has the following description in Documentation/admin-guide/tainted-kernels.rst: "bad page referenced or some unexpected page flags". Adjust the warning that reports the failure to avoid references to internal functions and to add information about the kernel being tainted, both to match the style of other messages in the file. Additionally, merge the message on a single line because checkpatch.pl recommends that for the ability to grep for the string. Suggested-by: Kees Cook Signed-off-by: Petr Pavlu Reviewed-by: Luis Chamberlain --- kernel/module/main.c | 7 ++++--- kernel/panic.c | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/module/main.c b/kernel/module/main.c index 2b6bfaade36a7c..d58377595e6438 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -2968,10 +2968,11 @@ static noinline int do_init_module(struct module *mod) rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms); #endif ret = module_enable_rodata_ro_after_init(mod); - if (ret) - pr_warn("%s: module_enable_rodata_ro_after_init() returned %d, " - "ro_after_init data might still be writable\n", + if (ret) { + pr_warn("%s: write-protecting ro_after_init data failed with %d, the data might still be writable - tainting kernel\n", mod->name, ret); + add_taint_module(mod, TAINT_BAD_PAGE, LOCKDEP_STILL_OK); + } mod_tree_remove_init(mod); module_arch_freeing_init(mod); diff --git a/kernel/panic.c b/kernel/panic.c index a3889f38153d96..ea622bc304e943 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -497,7 +497,7 @@ const struct taint_flag taint_flags[TAINT_FLAGS_COUNT] = { TAINT_FLAG(CPU_OUT_OF_SPEC, 'S', ' ', false), TAINT_FLAG(FORCED_RMMOD, 'R', ' ', false), TAINT_FLAG(MACHINE_CHECK, 'M', ' ', false), - TAINT_FLAG(BAD_PAGE, 'B', ' ', false), + TAINT_FLAG(BAD_PAGE, 'B', ' ', true), TAINT_FLAG(USER, 'U', ' ', false), TAINT_FLAG(DIE, 'D', ' ', false), TAINT_FLAG(OVERRIDDEN_ACPI_TABLE, 'A', ' ', false),