From 96f6e225e3663ec701f4dd6cf6bc5cffc49188df Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Tue, 18 Mar 2025 22:56:12 +0000 Subject: [PATCH 1/2] adding ci files --- .github/workflows/kdevops-cleanup.yml | 60 +++++++++ .github/workflows/kdevops-generic.yml | 37 +++++ .github/workflows/kdevops-init.yml | 187 ++++++++++++++++++++++++++ 3 files changed, 284 insertions(+) create mode 100644 .github/workflows/kdevops-cleanup.yml create mode 100644 .github/workflows/kdevops-generic.yml create mode 100644 .github/workflows/kdevops-init.yml diff --git a/.github/workflows/kdevops-cleanup.yml b/.github/workflows/kdevops-cleanup.yml new file mode 100644 index 00000000000000..d7e69ac48e6c92 --- /dev/null +++ b/.github/workflows/kdevops-cleanup.yml @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# This can be used towards the end of your action. All tasks here run even if +# any of the previous tasks failed. + +name: Kdevops cleanup workflow + +on: + workflow_call: # Makes this workflow reusable + +jobs: + cleanup: + name: Archive results and cleanup + runs-on: [self-hosted, Linux, X64] + steps: + - name: Set Linux kdevops development path + if: ${{ job.status != 'cancelled' }} + run: echo "LINUX_KDEVOPS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV + + - name: Get systemd journal files + if: ${{ job.status != 'cancelled' }} + run: | + if [[ ! -d kdevops ]]; then + exit 0 + fi + cd kdevops + make journal-dump + + - name: Start SSH Agent + if: ${{ job.status != 'cancelled' }} + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Build our kdevops archive results + if: ${{ job.status != 'cancelled' }} + run: | + if [[ ! -d kdevops ]]; then + exit 0 + fi + cd kdevops + make ci-archive + + - name: Upload our kdevops results archive + if: ${{ job.status != 'cancelled' }} + uses: actions/upload-artifact@v4 + with: + name: kdevops-ci-results + path: ${{ env.LINUX_KDEVOPS_PATH }}/kdevops/archive/*.zip + + - name: Run kdevops make destroy + if: always() + run: | + if [[ ! -d kdevops ]]; then + exit 0 + fi + cd kdevops + make destroy + cd .. + rm -rf kdevops diff --git a/.github/workflows/kdevops-generic.yml b/.github/workflows/kdevops-generic.yml new file mode 100644 index 00000000000000..8ca2006c1e1d22 --- /dev/null +++ b/.github/workflows/kdevops-generic.yml @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Most simple Linux kernel subsystems can be tested with this target +# test setup. For more elaborates tests look for a topic branch under the +# kdevops-ci tree. For example to test a filesystem look at the fstests +# branch. + +name: Run generic kdevops CI tests + +on: + push: + branches: ['**'] + pull_request: + branches: ['**'] + workflow_dispatch: # Allow manual triggering + +jobs: + setup: + uses: ./.github/workflows/kdevops-init.yml + secrets: inherit + + run-tests: + needs: setup + name: Run CI tests + runs-on: [self-hosted, Linux, X64] + steps: + - name: Run CI tests + run: | + cd kdevops + make ci-test + echo "ok" > ci.result + + cleanup: + needs: [run-tests, setup] # Add setup as a dependency to ensure proper ordering + if: always() # This ensures cleanup runs even if run-tests fails + uses: ./.github/workflows/kdevops-cleanup.yml + secrets: inherit diff --git a/.github/workflows/kdevops-init.yml b/.github/workflows/kdevops-init.yml new file mode 100644 index 00000000000000..9a265c2dbc1f84 --- /dev/null +++ b/.github/workflows/kdevops-init.yml @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# This can be used as a initialization workflow for most Linux kernel +# development environments. This takes care of: +# +# - Checks out and re-using a local mirror for your kernel tree +# - Looks for a defconfig in kdevops to use for your kernel tree +# - Sets up CI metadata for kdevops-results-archive +# - Ensures your kernel tree at least builds with defconfig +# - Brings up target DUTs nodes +# - Installs your Linux kernel tree on them +# - Builds all of your test requirements for your Linux kernel tree + +name: Base kdevops workflow + +on: + workflow_call: # Makes this workflow reusable + inputs: + kdevops_defconfig: + required: false + type: string + +jobs: + setup: + name: Setup kdevops environment + runs-on: [self-hosted, Linux, X64] + steps: + - name: Verify we won't expect user input interactions on the host key + run: | + mkdir -p ~/.ssh + if ! grep -q "StrictHostKeyChecking no" ~/.ssh/config 2>/dev/null; then + echo "StrictHostKeyChecking no" >> ~/.ssh/config + fi + + - name: Start SSH Agent for initial test + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + # Modify the repo here if you have a custom or private URL for the archive + # This can also just be a repo variable later. + - name: Verify our ssh connection will work + run: | + if ! git ls-remote git@github.com:linux-kdevops/kdevops-results-archive.git HEAD; then + echo "Cannot access kdevops-results-archive repository" + exit 1 + fi + + - name: Configure git + run: | + git config --global --add safe.directory '*' + git config --global user.name "kdevops" + git config --global user.email "kdevops@lists.linux.dev" + + - name: Checkout kdevops + run: | + rm -rf kdevops + git clone /mirror/kdevops.git kdevops + + - name: Make sure our repo kdevops defconfig exists + run: | + cd kdevops + if [[ -z "${{ inputs.kdevops_defconfig }}" ]]; then + KDEVOPS_DEFCONFIG=$(basename ${{ github.repository }}) + else + KDEVOPS_DEFCONFIG="${{ inputs.kdevops_defconfig }}" + fi + + if [[ ! -f defconfigs/$KDEVOPS_DEFCONFIG ]]; then + echo "kdevops lacks a defconfig for this repository, expected to find: defconfigs/$KDEVOPS_DEFCONFIG" + exit 1 + fi + + echo "KDEVOPS_DEFCONFIG=$KDEVOPS_DEFCONFIG" >> $GITHUB_ENV + + - name: Checkout custom branch with delta on kdevops/linux + run: | + LINUX_TREE="https://github.com/${{ github.repository }}" + LINUX_TREE_REF="${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}" + cd kdevops + git clone $LINUX_TREE --reference /mirror/linux.git/ --depth=5 linux + cd linux + git fetch origin $LINUX_TREE_REF + git checkout $LINUX_TREE_REF + git log -1 + + - name: Initialize CI metadata for kdevops-results-archive for linux + run: | + cd kdevops/linux + echo "$(basename ${{ github.repository }})" > ../ci.trigger + + # This supports using kdevops github actions using two different + # approaches: + # + # 1) Commit the .github/ directory onto a Linux tree before your + # kernel changes. This approach is used for example for + # testing patches posted on the mailing list with patchwork, + # this is the strategy kernel-patch-deaemon uses. Since the + # patches are ephemeral there is not important git history to + # maintain. + # + # 2) Merge the .github/ directory at the end of your development + # tree. This is useful for kernel developers wishing to test + # existing trees. + # + # So this checks to see if the last commit (top of the tree) *added* + # the .github directory. If the last commit added it, then we assume + # the commit prior to it was the one we'd like to document as the main + # test point. + if git diff-tree --no-commit-id --name-only --diff-filter=A -r HEAD | grep -q "^\.github/"; then + git log -2 --skip=1 --pretty=format:"%s" -1 > ../ci.subject + git describe --exact-match --tags HEAD^ 2>/dev/null || git rev-parse --short HEAD^ > ../ci.ref + else + git log -1 --pretty=format:"%s" > ../ci.subject + git describe --exact-match --tags HEAD 2>/dev/null || git rev-parse --short HEAD > ../ci.ref + fi + + RELEVANT_GIT_TAG=$(cat ../ci.ref) + RELEVANT_GIT_REF=$(git rev-parse --short=12 $RELEVANT_GIT_TAG) + + echo "LINUX_GIT_REF=$RELEVANT_GIT_REF" >> $GITHUB_ENV + echo "LINUX_GIT_TAG=$RELEVANT_GIT_TAG" >> $GITHUB_ENV + + # Start out pessimistic + echo "unknown" > ../ci.result + echo "Nothing to write home about." > ../ci.commit_extra + + - name: Run a quick Linux kernel defconfig build test + run: | + cd kdevops/linux + git reset --hard ${{ env.LINUX_GIT_TAG }} + make defconfig + make -j$(nproc) + + - name: Run kdevops make defconfig-repo + run: | + LINUX_TREE="https://github.com/${{ github.repository }}" + LINUX_TREE_REF="${{ env.LINUX_GIT_TAG }}" + + # We make the compromise here to use a relevant git tag for the + # host prefix so that folks can easily tell what exact kernel tree + # is being tested by using the relevant git ref. That is, if you + # pushed a tree with the .github/ directory as the top of the tree, + # that commit will not be used, we'll use the last one as that is + # the relevant git ref we want to annotate a test for. + # + # The compromise here we use special KDEVOPS to separete the + # commit ID and github.run_id. Exotic things likes UTF characters + # and dots have problems. + KDEVOPS_HOSTS_PREFIX="${{ env.LINUX_GIT_REF }}KDEVOPS${{ github.run_id }}" + + echo "Going to use defconfig-${{ env.KDEVOPS_DEFCONFIG }}" + + echo "Linux tree: $LINUX_TREE" + echo "Linux trigger ref: $LINUX_TREE_REF" + echo "Linux tag: ${{ env.LINUX_GIT_TAG }}" + echo "Runner ID: ${{ github.run_id }}" + echo "kdevops host prefix: $KDEVOPS_HOSTS_PREFIX" + echo "kdevops defconfig: defconfig-${{ env.KDEVOPS_DEFCONFIG }}" + + KDEVOPS_ARGS="KDEVOPS_HOSTS_PREFIX=$KDEVOPS_HOSTS_PREFIX LINUX_TREE=$LINUX_TREE LINUX_TREE_REF=$LINUX_TREE_REF defconfig-${{ env.KDEVOPS_DEFCONFIG }}" + echo "Going to run:" + echo "make $KDEVOPS_ARGS" + + cd kdevops + make $KDEVOPS_ARGS + + - name: Run kdevops make + run: | + cd kdevops + make -j$(nproc) + + - name: Run kdevops make bringup + run: | + cd kdevops + ls -ld linux + make bringup + + - name: Build linux and boot test nodes on test kernel + run: | + cd kdevops + make linux + + - name: Build required ci tests + run: | + cd kdevops + make ci-build-test From 2415bf1b9d4c7f191f90c5363a5c588e0b9dc9ea Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Thu, 20 Mar 2025 23:27:58 +0000 Subject: [PATCH 2/2] kbuild: Require pahole >v1.29 with GENDWARFKSYMS and BTF on X86 With CONFIG_GENDWARFKSYMS, __gendwarfksyms_ptr variables are added to the kernel in EXPORT_SYMBOL() to ensure DWARF type information is available for exported symbols in the TUs where they're actually exported. These symbols are dropped when linking vmlinux, but dangling references to them remain in DWARF. With CONFIG_DEBUG_INFO_BTF enabled on X86, pahole versions before commit 9810758003ce ("btf_encoder: Verify 0 address DWARF variables are in ELF section") place these symbols in the .data..percpu section, which results in an "Invalid offset" error in btf_datasec_check_meta() during boot, as all the variables are at zero offset and have non-zero size. If CONFIG_DEBUG_INFO_BTF_MODULES is enabled, this also results in a failure to load modules with: failed to validate module [$module] BTF: -22 The pahole commit that adds 0 address DWARF variable verification was merged after v1.29 was released, so later versions of pahole shouldn't have this issue. Require pahole >v1.29 when GENDWARFKSYMS is enabled with DEBUG_INFO_BTF on X86. Reported-by: Paolo Pisati Signed-off-by: Sami Tolvanen --- kernel/module/Kconfig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/module/Kconfig b/kernel/module/Kconfig index d7762ef5949a2c..7c75832aa1af48 100644 --- a/kernel/module/Kconfig +++ b/kernel/module/Kconfig @@ -192,6 +192,10 @@ config GENDWARFKSYMS depends on !DEBUG_INFO_REDUCED && !DEBUG_INFO_SPLIT # Requires ELF object files. depends on !LTO + # Requires pahole commit 9810758003ce ("btf_encoder: Verify 0 address + # DWARF variables are in ELF section") on X86 to avoid conflicts with + # __gendwarfksyms_ptr symbols. + depends on !X86 || !DEBUG_INFO_BTF || PAHOLE_VERSION > 129 help Calculate symbol versions from DWARF debugging information using gendwarfksyms. Requires DEBUG_INFO to be enabled.