From b3da8e89f8771297c33cbe8d0d98ac2054c7ba11 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 19 Jul 2023 15:24:01 +0200 Subject: [PATCH 001/141] secrets! --- .../workflows/build-container-and-deploy.yml | 123 -------------- ...l => build-container-and-test-aarch64.yml} | 61 +++---- .../build-executable-test-deploy.yml | 157 ------------------ .github/workflows/build-on-windows.yml | 18 -- .github/workflows/linters.yml | 19 --- 5 files changed, 26 insertions(+), 352 deletions(-) delete mode 100644 .github/workflows/build-container-and-deploy.yml rename .github/workflows/{build-container-and-test.yml => build-container-and-test-aarch64.yml} (72%) delete mode 100644 .github/workflows/build-executable-test-deploy.yml delete mode 100644 .github/workflows/build-on-windows.yml delete mode 100644 .github/workflows/linters.yml diff --git a/.github/workflows/build-container-and-deploy.yml b/.github/workflows/build-container-and-deploy.yml deleted file mode 100644 index 85091a1c9..000000000 --- a/.github/workflows/build-container-and-deploy.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Build gProfiler container & deploy - -on: - push: - tags: - - '**' - -jobs: - build-docker-x64: - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - - name: Get and verify tag value - run: ./scripts/verify_tag.sh - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - - name: Build and push - run: | - set -x - export DOCKER_BUILDKIT=1 - - BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" - LATEST_IMAGE="${{ env.GH_REPO }}:latest" - X86_64_IMAGE="$BASE_IMAGE-x86_64" - - # build & push x86_64 - ./scripts/build_x86_64_container.sh -t "$X86_64_IMAGE" - docker push "$X86_64_IMAGE" - - build-docker-aarch64: - runs-on: ["self-hosted", "ARM64"] - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - - name: Set up runner requirements - run: scripts/setup_runner_requirements.sh - - - name: Get and verify tag value - run: ./scripts/verify_tag.sh - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - - name: Build and push - run: | - set -x - export DOCKER_BUILDKIT=1 - - BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" - LATEST_IMAGE="${{ env.GH_REPO }}:latest" - AARCH64_IMAGE="$BASE_IMAGE-aarch64" - - # build & push Aarch64 - ./scripts/build_aarch64_container.sh -t "$AARCH64_IMAGE" --push - - push-manifest: - runs-on: ubuntu-latest - needs: - - build-docker-x64 - - build-docker-aarch64 - steps: - # needed to set up env.GH_REPO etc - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - - name: Get and verify tag value - run: ./scripts/verify_tag.sh - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Push manifest - run: | - set -x - export DOCKER_BUILDKIT=1 - - BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" - LATEST_IMAGE="${{ env.GH_REPO }}:latest" - AARCH64_IMAGE="$BASE_IMAGE-aarch64" - X86_64_IMAGE="$BASE_IMAGE-x86_64" - - docker pull $AARCH64_IMAGE - docker pull $X86_64_IMAGE - - # create manifests for the tag + for 'latest' - docker manifest create "$BASE_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" - docker manifest push "$BASE_IMAGE" - docker manifest create "$LATEST_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" - docker manifest push "$LATEST_IMAGE" diff --git a/.github/workflows/build-container-and-test.yml b/.github/workflows/build-container-and-test-aarch64.yml similarity index 72% rename from .github/workflows/build-container-and-test.yml rename to .github/workflows/build-container-and-test-aarch64.yml index c85467b2a..4175301ac 100644 --- a/.github/workflows/build-container-and-test.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -1,51 +1,33 @@ -name: Lint, build gProfiler container and run tests +name: Lint, build gProfiler container and run tests on aarch64 on: pull_request jobs: - lint: - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - runs-on: ubuntu-latest + build-container-aarch64: + runs-on: + - self-hosted + - public + - ARM64 steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Checkout Code uses: actions/checkout@v3 with: submodules: true - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r dev-requirements.txt - pip install -r requirements.txt - - - name: Run linters - run: ./lint.sh --ci - - build-container: - runs-on: ubuntu-latest + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - steps: - - name: Checkout Code - uses: actions/checkout@v3 + - name: Login to DockerHub + uses: docker/login-action@v1 with: - submodules: true + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs # more nicely. the tests will then be able to use the built image. - name: Build gProfiler image - run: ./scripts/build_x86_64_container.sh + run: ./scripts/build_aarch64_container.sh - name: Export gProfiler image run: mkdir -p output && docker image save gprofiler > output/gprofiler.img @@ -58,9 +40,12 @@ jobs: retention-days: 1 test: - needs: build-container + needs: build-container-aarch64 - runs-on: ubuntu-20.04 # for 3.10.0 python tag + runs-on: + - self-hosted + - public + - ARM64 strategy: fail-fast: false # helps detecting flakiness / errors specific to one Python version @@ -81,14 +66,20 @@ jobs: with: java-version: '8.0.275' java-package: jdk - architecture: x64 + architecture: aarch64 - name: Install Node.JS uses: actions/setup-node@v2 with: # same version as used in tests/containers/nodejs/Dockerfile node-version: 10.x - + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Checkout Code uses: actions/checkout@v3 with: diff --git a/.github/workflows/build-executable-test-deploy.yml b/.github/workflows/build-executable-test-deploy.yml deleted file mode 100644 index 4e7e5cc7e..000000000 --- a/.github/workflows/build-executable-test-deploy.yml +++ /dev/null @@ -1,157 +0,0 @@ -name: Build gProfiler executable, test (if PR) and deploy (if tag) - -on: - pull_request: - push: - tags: - - '**' - -jobs: - build-executable: - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - - name: Get and verify tag value - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - ./scripts/cmp_tags.sh - - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - - name: Build gProfiler executable - # Using DOCKER_BUILDKIT=1 although it has another cache mechanism which is not supported by satackey/action-docker-layer-caching@v0.0.11 - # We tried to cache using buildx cache (cache-from, cache-to flags) and got strange behavior when caching, - # decided not to use buildkit here to get nice caches. - # if we enable caching again, we can disable DOCKER_BUILDKIT. - run: | - mkdir -p output - ./scripts/build_x86_64_executable.sh - mv build/x86_64/gprofiler output/gprofiler_x86_64 - cp output/gprofiler_x86_64 output/gprofiler # for backwards compatibility - - - name: Upload the executables as job artifacts - uses: actions/upload-artifact@v2 - with: - name: gprofiler - path: output/ - retention-days: 1 - - test-executable: - if: ${{ !startsWith(github.ref, 'refs/tags/') }} - runs-on: ubuntu-20.04 # the tests which use ruby/node/python etc and run at non-root privs, fail to read the files when running - # on ubuntu-22.04/ubuntu-latest: - # stderr: ruby: Permission denied -- /home/runner/work/gprofiler/gprofiler/tests/containers/ruby/fibonacci.rb (LoadError) - needs: build-executable - strategy: - fail-fast: false - matrix: - containers: - - alpine - - ubuntu:14.04 - - ubuntu:16.04 - - ubuntu:18.04 - - ubuntu:20.04 - - ubuntu:20.10 - - ubuntu:22.04 - - centos:6 - - centos:7 - - centos:8 - - debian:8 - - debian:9 - - debian:10 - - debian:11 - steps: - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - - name: Install Java - uses: actions/setup-java@v1 - with: - java-version: '8.0.275' - java-package: jdk - architecture: x64 - - - name: Install Node.JS - uses: actions/setup-node@v2 - with: - # same version as used in tests/containers/nodejs/Dockerfile - node-version: 10.x - - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Download the executable from previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler - path: dist/ - - - name: Run gProfiler tests - run: | - mv dist/gprofiler_x86_64 dist/gprofiler - chmod +x dist/gprofiler - NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable - - build-executable-aarch64: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ["self-hosted", "ARM64"] - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - - name: Set up requirements - run: scripts/setup_runner_requirements.sh - - - name: Get and verify tag value - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - ./scripts/cmp_tags.sh - - - name: Build gProfiler executable - run: | - mkdir -p output - ./scripts/build_aarch64_executable.sh - mv build/aarch64/gprofiler output/gprofiler_aarch64 - - - name: Upload the executables as job artifacts - uses: actions/upload-artifact@v2 - with: - name: gprofiler - path: output/ - retention-days: 1 - - deploy-executable: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-20.04 - needs: - - build-executable - - build-executable-aarch64 - - steps: - - name: Download executables from the previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler - path: output/ - - name: Get tag value - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Release - uses: softprops/action-gh-release@v1 - with: - files: | - output/gprofiler_x86_64 - output/gprofiler_aarch64 - output/gprofiler - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-on-windows.yml b/.github/workflows/build-on-windows.yml deleted file mode 100644 index 847befc75..000000000 --- a/.github/workflows/build-on-windows.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Build gProfiler on Windows - -on: pull_request - -jobs: - build-executable: - runs-on: windows-latest - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: run the build script - run: .\scripts\windows\build.bat - - - name: check gprofiler build - run: .\app\dist\gprofiler.exe -h diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml deleted file mode 100644 index a04764fb5..000000000 --- a/.github/workflows/linters.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Non-Python linters - -on: pull_request - -jobs: - lint: - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Run shell linter - run: ./shell_lint.sh - - - name: Run Dockerfile linters - run: ./dockerfile_lint.sh From dd3f46439f4c9381ac836cdca1f0808aa9853156 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 13:37:52 +0200 Subject: [PATCH 002/141] correct tag --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 4175301ac..eb26c16f7 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -27,7 +27,7 @@ jobs: # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs # more nicely. the tests will then be able to use the built image. - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh + run: ./scripts/build_aarch64_container.sh -t gprofiler - name: Export gProfiler image run: mkdir -p output && docker image save gprofiler > output/gprofiler.img From 0d865ad96e1d4c9f13d8e407e6cbc4172ee389e6 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 14:16:05 +0200 Subject: [PATCH 003/141] correct tag --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- scripts/build_aarch64_container.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index eb26c16f7..4175301ac 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -27,7 +27,7 @@ jobs: # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs # more nicely. the tests will then be able to use the built image. - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh -t gprofiler + run: ./scripts/build_aarch64_container.sh - name: Export gProfiler image run: mkdir -p output && docker image save gprofiler > output/gprofiler.img diff --git a/scripts/build_aarch64_container.sh b/scripts/build_aarch64_container.sh index 46acb22cb..bd5475d8d 100755 --- a/scripts/build_aarch64_container.sh +++ b/scripts/build_aarch64_container.sh @@ -32,4 +32,4 @@ docker buildx build --platform=linux/arm64 \ --build-arg DOTNET_BUILDER=$DOTNET_BUILDER \ --build-arg NODE_PACKAGE_BUILDER_MUSL=$ALPINE_VERSION \ --build-arg NODE_PACKAGE_BUILDER_GLIBC=$NODE_PACKAGE_BUILDER_GLIBC \ - . "$@" + . -t gprofiler "$@" From 7dd2ae3dc70a3ee2a106cbd9cf486117ab58b755 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 14:46:41 +0200 Subject: [PATCH 004/141] correct tag --- scripts/build_aarch64_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_aarch64_container.sh b/scripts/build_aarch64_container.sh index bd5475d8d..09a8ca522 100755 --- a/scripts/build_aarch64_container.sh +++ b/scripts/build_aarch64_container.sh @@ -32,4 +32,4 @@ docker buildx build --platform=linux/arm64 \ --build-arg DOTNET_BUILDER=$DOTNET_BUILDER \ --build-arg NODE_PACKAGE_BUILDER_MUSL=$ALPINE_VERSION \ --build-arg NODE_PACKAGE_BUILDER_GLIBC=$NODE_PACKAGE_BUILDER_GLIBC \ - . -t gprofiler "$@" + . --output type=docker -t gprofiler "$@" From 06ac8a0eec9da59a80affa5ec94a643f9989a539 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 8 Aug 2023 16:03:01 +0300 Subject: [PATCH 005/141] build: Use https link instead of ftp --- scripts/perf_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/perf_env.sh b/scripts/perf_env.sh index 314bb19aa..3e0e7df48 100755 --- a/scripts/perf_env.sh +++ b/scripts/perf_env.sh @@ -45,7 +45,7 @@ popd rm -r zstd-1.5.2 zstd-1.5.2.tar.gz # install newer versions of elfutils -curl -L ftp://sourceware.org/pub/elfutils/0.187/elfutils-0.187.tar.bz2 -o elfutils-0.187.tar.bz2 +curl -L https://sourceware.org/elfutils/ftp/0.187/elfutils-0.187.tar.bz2 -o elfutils-0.187.tar.bz2 tar -xf elfutils-0.187.tar.bz2 pushd elfutils-0.187 # disable debuginfod, otherwise it will try to dlopen("libdebuginfod.so") in runtime & that can From ef0f2cd3705bdf6393771879a66cf25121458564 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 15:12:55 +0200 Subject: [PATCH 006/141] different tag approach --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- scripts/build_aarch64_container.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 4175301ac..5df379a35 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -27,7 +27,7 @@ jobs: # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs # more nicely. the tests will then be able to use the built image. - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh + run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler - name: Export gProfiler image run: mkdir -p output && docker image save gprofiler > output/gprofiler.img diff --git a/scripts/build_aarch64_container.sh b/scripts/build_aarch64_container.sh index 09a8ca522..46acb22cb 100755 --- a/scripts/build_aarch64_container.sh +++ b/scripts/build_aarch64_container.sh @@ -32,4 +32,4 @@ docker buildx build --platform=linux/arm64 \ --build-arg DOTNET_BUILDER=$DOTNET_BUILDER \ --build-arg NODE_PACKAGE_BUILDER_MUSL=$ALPINE_VERSION \ --build-arg NODE_PACKAGE_BUILDER_GLIBC=$NODE_PACKAGE_BUILDER_GLIBC \ - . --output type=docker -t gprofiler "$@" + . "$@" From 1604ee555393d5603108209a3d80a2ad89f921d0 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 15:25:25 +0200 Subject: [PATCH 007/141] python --- .../build-container-and-test-aarch64.yml | 150 +++++++++--------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 5df379a35..36c005490 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -3,44 +3,44 @@ name: Lint, build gProfiler container and run tests on aarch64 on: pull_request jobs: - build-container-aarch64: - runs-on: - - self-hosted - - public - - ARM64 - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs - # more nicely. the tests will then be able to use the built image. - - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler - - - name: Export gProfiler image - run: mkdir -p output && docker image save gprofiler > output/gprofiler.img - - - name: Upload the image artifact - uses: actions/upload-artifact@v2 - with: - name: gprofiler.img - path: output/ - retention-days: 1 + # build-container-aarch64: + # runs-on: + # - self-hosted + # - public + # - ARM64 + + # steps: + # - name: Checkout Code + # uses: actions/checkout@v3 + # with: + # submodules: true + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + + # - name: Login to DockerHub + # uses: docker/login-action@v1 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + + # # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # # more nicely. the tests will then be able to use the built image. + # - name: Build gProfiler image + # run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler + + # - name: Export gProfiler image + # run: mkdir -p output && docker image save gprofiler > output/gprofiler.img + + # - name: Upload the image artifact + # uses: actions/upload-artifact@v2 + # with: + # name: gprofiler.img + # path: output/ + # retention-days: 1 test: - needs: build-container-aarch64 + # needs: build-container-aarch64 runs-on: - self-hosted @@ -57,47 +57,47 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install Java - uses: actions/setup-java@v1 - with: - java-version: '8.0.275' - java-package: jdk - architecture: aarch64 - - - name: Install Node.JS - uses: actions/setup-node@v2 - with: - # same version as used in tests/containers/nodejs/Dockerfile - node-version: 10.x + # - name: Install Java + # uses: actions/setup-java@v1 + # with: + # java-version: '8.0.275' + # java-package: jdk + # architecture: aarch64 + + # - name: Install Node.JS + # uses: actions/setup-node@v2 + # with: + # # same version as used in tests/containers/nodejs/Dockerfile + # node-version: 10.x - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + # - name: Login to DockerHub + # uses: docker/login-action@v1 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Download the executable from previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler.img - path: output/ - - # used for next step (Copy resources ...) and also used in the tests themselves. - - name: Import gProfiler image - run: docker image load < output/gprofiler.img - - - name: Copy resources from gProfiler image - run: ./scripts/copy_resources_from_image.sh gprofiler - - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - - name: Run gProfiler tests - run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py + # - name: Checkout Code + # uses: actions/checkout@v3 + # with: + # submodules: true + + # - name: Download the executable from previous job + # uses: actions/download-artifact@v2 + # with: + # name: gprofiler.img + # path: output/ + + # # used for next step (Copy resources ...) and also used in the tests themselves. + # - name: Import gProfiler image + # run: docker image load < output/gprofiler.img + + # - name: Copy resources from gProfiler image + # run: ./scripts/copy_resources_from_image.sh gprofiler + + # # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" + # - name: Run gProfiler tests + # run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py From d971d943329229cc20da5900a8e6e20fcf608388 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 15:31:08 +0200 Subject: [PATCH 008/141] python --- .../build-container-and-test-aarch64.yml | 82 +++++++++---------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 36c005490..b09d82a26 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -51,53 +51,51 @@ jobs: fail-fast: false # helps detecting flakiness / errors specific to one Python version matrix: python-version: - - "3.8" - - "3.9" - - "3.10.0" # see https://github.com/Granulate/gprofiler/issues/502, we need 3.10.0, others fail PyPerf tests. + - "3.10.11" steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - # - name: Install Java - # uses: actions/setup-java@v1 - # with: - # java-version: '8.0.275' - # java-package: jdk - # architecture: aarch64 - - # - name: Install Node.JS - # uses: actions/setup-node@v2 - # with: - # # same version as used in tests/containers/nodejs/Dockerfile - # node-version: 10.x + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: '8.0.275' + java-package: jdk + architecture: aarch64 + + - name: Install Node.JS + uses: actions/setup-node@v2 + with: + # same version as used in tests/containers/nodejs/Dockerfile + node-version: 10.x - # - name: Login to DockerHub - # uses: docker/login-action@v1 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - # - name: Checkout Code - # uses: actions/checkout@v3 - # with: - # submodules: true - - # - name: Download the executable from previous job - # uses: actions/download-artifact@v2 - # with: - # name: gprofiler.img - # path: output/ - - # # used for next step (Copy resources ...) and also used in the tests themselves. - # - name: Import gProfiler image - # run: docker image load < output/gprofiler.img - - # - name: Copy resources from gProfiler image - # run: ./scripts/copy_resources_from_image.sh gprofiler - - # # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - # - name: Run gProfiler tests - # run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Download the executable from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler.img + path: output/ + + # used for next step (Copy resources ...) and also used in the tests themselves. + - name: Import gProfiler image + run: docker image load < output/gprofiler.img + + - name: Copy resources from gProfiler image + run: ./scripts/copy_resources_from_image.sh gprofiler + + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" + - name: Run gProfiler tests + run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py From d293c701509af141b32b4de394410235e3aef5f6 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 15:33:29 +0200 Subject: [PATCH 009/141] python --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index b09d82a26..56053a746 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -55,7 +55,7 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} From a877bf8a39f90465c4e34eed768b8ad07562f251 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 15:36:13 +0200 Subject: [PATCH 010/141] python --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 56053a746..ca864f16b 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -55,7 +55,7 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 1f5258e929d733e112c71e91278e6e2e3597b9b0 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 16:51:59 +0200 Subject: [PATCH 011/141] python --- .../build-container-and-test-aarch64.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index ca864f16b..14fa42ec7 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -47,17 +47,18 @@ jobs: - public - ARM64 - strategy: - fail-fast: false # helps detecting flakiness / errors specific to one Python version - matrix: - python-version: - - "3.10.11" + # strategy: + # fail-fast: false # helps detecting flakiness / errors specific to one Python version + # matrix: + # python-version: + # - "3.10.11" steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} + - name: Install python3 + run: apt-get install python3 + + - name: check python installation + run: python --version - name: Install Java uses: actions/setup-java@v1 From c23a3f69d12411f11eb7ebcd06859357f17ea3b8 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 16:53:26 +0200 Subject: [PATCH 012/141] python --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 14fa42ec7..9f715f509 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -55,7 +55,7 @@ jobs: steps: - name: Install python3 - run: apt-get install python3 + run: sudo apt-get install python3 - name: check python installation run: python --version From 604f76b01a44bbec35e8ef28e09ddc3934022ec1 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 16:54:48 +0200 Subject: [PATCH 013/141] python --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 9f715f509..c6f49a6f8 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -55,7 +55,7 @@ jobs: steps: - name: Install python3 - run: sudo apt-get install python3 + run: sudo apt-get install -y python3 - name: check python installation run: python --version From d0b6e7ff0bfdde3144565db5bd1454b73bc17db8 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 16:55:15 +0200 Subject: [PATCH 014/141] python --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index c6f49a6f8..84e7ae77c 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -58,7 +58,7 @@ jobs: run: sudo apt-get install -y python3 - name: check python installation - run: python --version + run: python3 --version - name: Install Java uses: actions/setup-java@v1 From 5dbc3d14b3d552b9ebf171c15cb7af967480820e Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 17:20:47 +0200 Subject: [PATCH 015/141] apt install --- .../build-container-and-test-aarch64.yml | 110 +++++++++--------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 84e7ae77c..4a45c4757 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -3,44 +3,44 @@ name: Lint, build gProfiler container and run tests on aarch64 on: pull_request jobs: - # build-container-aarch64: - # runs-on: - # - self-hosted - # - public - # - ARM64 - - # steps: - # - name: Checkout Code - # uses: actions/checkout@v3 - # with: - # submodules: true - - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v2 - - # - name: Login to DockerHub - # uses: docker/login-action@v1 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - - # # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs - # # more nicely. the tests will then be able to use the built image. - # - name: Build gProfiler image - # run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler - - # - name: Export gProfiler image - # run: mkdir -p output && docker image save gprofiler > output/gprofiler.img - - # - name: Upload the image artifact - # uses: actions/upload-artifact@v2 - # with: - # name: gprofiler.img - # path: output/ - # retention-days: 1 + build-container-aarch64: + runs-on: + - self-hosted + - public + - ARM64 + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # more nicely. the tests will then be able to use the built image. + - name: Build gProfiler image + run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler + + - name: Export gProfiler image + run: mkdir -p output && docker image save gprofiler > output/gprofiler.img + + - name: Upload the image artifact + uses: actions/upload-artifact@v2 + with: + name: gprofiler.img + path: output/ + retention-days: 1 test: - # needs: build-container-aarch64 + needs: build-container-aarch64 runs-on: - self-hosted @@ -54,24 +54,24 @@ jobs: # - "3.10.11" steps: - - name: Install python3 - run: sudo apt-get install -y python3 - - - name: check python installation - run: python3 --version - - - name: Install Java - uses: actions/setup-java@v1 - with: - java-version: '8.0.275' - java-package: jdk - architecture: aarch64 - - - name: Install Node.JS - uses: actions/setup-node@v2 - with: - # same version as used in tests/containers/nodejs/Dockerfile - node-version: 10.x + # - name: Install python3 + # run: sudo apt-get install -y python3 + + # - name: check python installation + # run: python3 --version + + # - name: Install Java + # uses: actions/setup-java@v1 + # with: + # java-version: '8.0.275' + # java-package: jdk + # architecture: aarch64 + + # - name: Install Node.JS + # uses: actions/setup-node@v2 + # with: + # # same version as used in tests/containers/nodejs/Dockerfile + # node-version: 10.x - name: Login to DockerHub uses: docker/login-action@v1 @@ -99,4 +99,4 @@ jobs: # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests - run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py + run: ./tests/test.sh --ignore=tests/test_executable.py From 51a9819bfe7a98a8cc387da0367a4e53209218c2 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 8 Aug 2023 19:58:55 +0200 Subject: [PATCH 016/141] apt install --- .github/workflows/build-container-and-test-aarch64.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 4a45c4757..b8cfd5a62 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -84,6 +84,9 @@ jobs: with: submodules: true + - name: Create python and pip aliases + run: alias pip="pip3" && alias python="python3" + - name: Download the executable from previous job uses: actions/download-artifact@v2 with: From 3a62302c633807195a1ea99bd56d898cb1957ab1 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 08:55:42 +0200 Subject: [PATCH 017/141] apt install --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index b8cfd5a62..bd7e5a20e 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -85,7 +85,7 @@ jobs: submodules: true - name: Create python and pip aliases - run: alias pip="pip3" && alias python="python3" + run: echo 'alias pip="pip3"' >> ~/.bashrc && echo 'alias python="python3"' >> ~/.bashrc - name: Download the executable from previous job uses: actions/download-artifact@v2 From edfa4f82467dc2c9e5f96a666e39108929ad3b45 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 09:26:47 +0200 Subject: [PATCH 018/141] apt install --- .github/workflows/build-container-and-test-aarch64.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index bd7e5a20e..4a45c4757 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -84,9 +84,6 @@ jobs: with: submodules: true - - name: Create python and pip aliases - run: echo 'alias pip="pip3"' >> ~/.bashrc && echo 'alias python="python3"' >> ~/.bashrc - - name: Download the executable from previous job uses: actions/download-artifact@v2 with: From cfc3cb4382ce0a1e58473af0d6cd40f97cd48d37 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 10:10:33 +0200 Subject: [PATCH 019/141] apt install --- .github/workflows/build-container-and-test-aarch64.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 4a45c4757..6b5046a7e 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -54,8 +54,8 @@ jobs: # - "3.10.11" steps: - # - name: Install python3 - # run: sudo apt-get install -y python3 + - name: Install python3 + run: sudo apt-get install -y python3 # - name: check python installation # run: python3 --version From b633cdcbf33f5798fe30c61f6235ce106e48bde5 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 10:27:11 +0200 Subject: [PATCH 020/141] apt install --- .../build-container-and-test-aarch64.yml | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 6b5046a7e..dd1b66699 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -3,44 +3,44 @@ name: Lint, build gProfiler container and run tests on aarch64 on: pull_request jobs: - build-container-aarch64: - runs-on: - - self-hosted - - public - - ARM64 - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs - # more nicely. the tests will then be able to use the built image. - - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler - - - name: Export gProfiler image - run: mkdir -p output && docker image save gprofiler > output/gprofiler.img - - - name: Upload the image artifact - uses: actions/upload-artifact@v2 - with: - name: gprofiler.img - path: output/ - retention-days: 1 + # build-container-aarch64: + # runs-on: + # - self-hosted + # - public + # - ARM64 + + # steps: + # - name: Checkout Code + # uses: actions/checkout@v3 + # with: + # submodules: true + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + + # - name: Login to DockerHub + # uses: docker/login-action@v1 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + + # # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # # more nicely. the tests will then be able to use the built image. + # - name: Build gProfiler image + # run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler + + # - name: Export gProfiler image + # run: mkdir -p output && docker image save gprofiler > output/gprofiler.img + + # - name: Upload the image artifact + # uses: actions/upload-artifact@v2 + # with: + # name: gprofiler.img + # path: output/ + # retention-days: 1 test: - needs: build-container-aarch64 + # needs: build-container-aarch64 runs-on: - self-hosted From 9582dbe7c553a8e10275d75282c5291e5bda5037 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 10:40:57 +0200 Subject: [PATCH 021/141] apt install --- .../build-container-and-test-aarch64.yml | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index dd1b66699..232debb63 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -73,29 +73,29 @@ jobs: # # same version as used in tests/containers/nodejs/Dockerfile # node-version: 10.x - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + # - name: Login to DockerHub + # uses: docker/login-action@v1 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Download the executable from previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler.img - path: output/ - - # used for next step (Copy resources ...) and also used in the tests themselves. - - name: Import gProfiler image - run: docker image load < output/gprofiler.img - - - name: Copy resources from gProfiler image - run: ./scripts/copy_resources_from_image.sh gprofiler + # - name: Checkout Code + # uses: actions/checkout@v3 + # with: + # submodules: true + + # - name: Download the executable from previous job + # uses: actions/download-artifact@v2 + # with: + # name: gprofiler.img + # path: output/ + + # # used for next step (Copy resources ...) and also used in the tests themselves. + # - name: Import gProfiler image + # run: docker image load < output/gprofiler.img + + # - name: Copy resources from gProfiler image + # run: ./scripts/copy_resources_from_image.sh gprofiler # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From e507d6c770b6265f65066aa5bfc1ed6785263282 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 10:45:17 +0200 Subject: [PATCH 022/141] apt install --- .../build-container-and-test-aarch64.yml | 116 +++++++++--------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 232debb63..6b5046a7e 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -3,44 +3,44 @@ name: Lint, build gProfiler container and run tests on aarch64 on: pull_request jobs: - # build-container-aarch64: - # runs-on: - # - self-hosted - # - public - # - ARM64 - - # steps: - # - name: Checkout Code - # uses: actions/checkout@v3 - # with: - # submodules: true - - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v2 - - # - name: Login to DockerHub - # uses: docker/login-action@v1 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - - # # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs - # # more nicely. the tests will then be able to use the built image. - # - name: Build gProfiler image - # run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler - - # - name: Export gProfiler image - # run: mkdir -p output && docker image save gprofiler > output/gprofiler.img - - # - name: Upload the image artifact - # uses: actions/upload-artifact@v2 - # with: - # name: gprofiler.img - # path: output/ - # retention-days: 1 + build-container-aarch64: + runs-on: + - self-hosted + - public + - ARM64 + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # more nicely. the tests will then be able to use the built image. + - name: Build gProfiler image + run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler + + - name: Export gProfiler image + run: mkdir -p output && docker image save gprofiler > output/gprofiler.img + + - name: Upload the image artifact + uses: actions/upload-artifact@v2 + with: + name: gprofiler.img + path: output/ + retention-days: 1 test: - # needs: build-container-aarch64 + needs: build-container-aarch64 runs-on: - self-hosted @@ -73,29 +73,29 @@ jobs: # # same version as used in tests/containers/nodejs/Dockerfile # node-version: 10.x - # - name: Login to DockerHub - # uses: docker/login-action@v1 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - # - name: Checkout Code - # uses: actions/checkout@v3 - # with: - # submodules: true - - # - name: Download the executable from previous job - # uses: actions/download-artifact@v2 - # with: - # name: gprofiler.img - # path: output/ - - # # used for next step (Copy resources ...) and also used in the tests themselves. - # - name: Import gProfiler image - # run: docker image load < output/gprofiler.img - - # - name: Copy resources from gProfiler image - # run: ./scripts/copy_resources_from_image.sh gprofiler + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Download the executable from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler.img + path: output/ + + # used for next step (Copy resources ...) and also used in the tests themselves. + - name: Import gProfiler image + run: docker image load < output/gprofiler.img + + - name: Copy resources from gProfiler image + run: ./scripts/copy_resources_from_image.sh gprofiler # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 9f0e893698125da9aa88cb634826cd0018e3938d Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 11:11:18 +0200 Subject: [PATCH 023/141] apt install --- .../build-container-and-test-aarch64.yml | 116 +++++++++--------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 6b5046a7e..232debb63 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -3,44 +3,44 @@ name: Lint, build gProfiler container and run tests on aarch64 on: pull_request jobs: - build-container-aarch64: - runs-on: - - self-hosted - - public - - ARM64 - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs - # more nicely. the tests will then be able to use the built image. - - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler - - - name: Export gProfiler image - run: mkdir -p output && docker image save gprofiler > output/gprofiler.img - - - name: Upload the image artifact - uses: actions/upload-artifact@v2 - with: - name: gprofiler.img - path: output/ - retention-days: 1 + # build-container-aarch64: + # runs-on: + # - self-hosted + # - public + # - ARM64 + + # steps: + # - name: Checkout Code + # uses: actions/checkout@v3 + # with: + # submodules: true + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + + # - name: Login to DockerHub + # uses: docker/login-action@v1 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + + # # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # # more nicely. the tests will then be able to use the built image. + # - name: Build gProfiler image + # run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler + + # - name: Export gProfiler image + # run: mkdir -p output && docker image save gprofiler > output/gprofiler.img + + # - name: Upload the image artifact + # uses: actions/upload-artifact@v2 + # with: + # name: gprofiler.img + # path: output/ + # retention-days: 1 test: - needs: build-container-aarch64 + # needs: build-container-aarch64 runs-on: - self-hosted @@ -73,29 +73,29 @@ jobs: # # same version as used in tests/containers/nodejs/Dockerfile # node-version: 10.x - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + # - name: Login to DockerHub + # uses: docker/login-action@v1 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Download the executable from previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler.img - path: output/ - - # used for next step (Copy resources ...) and also used in the tests themselves. - - name: Import gProfiler image - run: docker image load < output/gprofiler.img - - - name: Copy resources from gProfiler image - run: ./scripts/copy_resources_from_image.sh gprofiler + # - name: Checkout Code + # uses: actions/checkout@v3 + # with: + # submodules: true + + # - name: Download the executable from previous job + # uses: actions/download-artifact@v2 + # with: + # name: gprofiler.img + # path: output/ + + # # used for next step (Copy resources ...) and also used in the tests themselves. + # - name: Import gProfiler image + # run: docker image load < output/gprofiler.img + + # - name: Copy resources from gProfiler image + # run: ./scripts/copy_resources_from_image.sh gprofiler # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From a9ac503cdbe680fa2dfbf59d47c28910f049ebe4 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 11:21:05 +0200 Subject: [PATCH 024/141] apt install --- tests/test.sh | 53 ++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/tests/test.sh b/tests/test.sh index 47cb770e8..bc27436c1 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -5,36 +5,37 @@ # set -e -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # https://stackoverflow.com/a/246128 +# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # https://stackoverflow.com/a/246128 -if [ -z ${NO_APT_INSTALL+x} ]; then - sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends openjdk-8-jdk python3 python3-pip docker.io php - if [ "$(uname -m)" = "aarch64" ]; then - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends python3-dev ruby build-essential nodejs - if ! [ -L "/usr/bin/dotnet" ] ; then - # there is no dotnet apt package on aarch64 - curl -SL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/master/dotnet-sdk-latest-linux-arm64.tar.gz - sudo mkdir -p /usr/share/dotnet - sudo tar -zxf dotnet.tar.gz -C /usr/share/dotnet - sudo ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet - fi - else - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends ruby2.7 dotnet-sdk-6.0 - fi -fi +# if [ -z ${NO_APT_INSTALL+x} ]; then +# sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update +# sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends openjdk-8-jdk python3 python3-pip docker.io php +# if [ "$(uname -m)" = "aarch64" ]; then +# sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends python3-dev ruby build-essential nodejs +# if ! [ -L "/usr/bin/dotnet" ] ; then +# # there is no dotnet apt package on aarch64 +# curl -SL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/master/dotnet-sdk-latest-linux-arm64.tar.gz +# sudo mkdir -p /usr/share/dotnet +# sudo tar -zxf dotnet.tar.gz -C /usr/share/dotnet +# sudo ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet +# fi +# else +# sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends ruby2.7 dotnet-sdk-6.0 +# fi +# fi -# we'll check 1 file, perf. not if --executable is passed - these are the executable tests, and they don't -# require resources. -# TODO split them to 2 pytest files -PERF_RESOURCE="$SCRIPT_DIR/../gprofiler/resources/perf" -if [ ! -f "$PERF_RESOURCE" ] && [[ "$*" != *"--executable"* ]]; then - echo "perf resource not found: $(readlink -f "$PERF_RESOURCE")" - echo "Please run $(readlink -f "$SCRIPT_DIR/../scripts/copy_resources_from_image.sh") to get all resources" - exit 1 -fi +# # we'll check 1 file, perf. not if --executable is passed - these are the executable tests, and they don't +# # require resources. +# # TODO split them to 2 pytest files +# PERF_RESOURCE="$SCRIPT_DIR/../gprofiler/resources/perf" +# if [ ! -f "$PERF_RESOURCE" ] && [[ "$*" != *"--executable"* ]]; then +# echo "perf resource not found: $(readlink -f "$PERF_RESOURCE")" +# echo "Please run $(readlink -f "$SCRIPT_DIR/../scripts/copy_resources_from_image.sh") to get all resources" +# exit 1 +# fi python3 -m pip install -q --upgrade setuptools pip python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . +python3 -m pip install -U pytest sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 89388c1e79e4e32514aa2f8d623ffdcfdd14376e Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 11:27:14 +0200 Subject: [PATCH 025/141] apt install --- .github/workflows/build-container-and-test-aarch64.yml | 3 ++- tests/test.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 232debb63..662008da0 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -96,7 +96,8 @@ jobs: # - name: Copy resources from gProfiler image # run: ./scripts/copy_resources_from_image.sh gprofiler - + - name: pytest pytest pytest + run: python3 -m pip install -U pytest # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: ./tests/test.sh --ignore=tests/test_executable.py diff --git a/tests/test.sh b/tests/test.sh index bc27436c1..c55029f52 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -37,5 +37,5 @@ set -e python3 -m pip install -q --upgrade setuptools pip python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . -python3 -m pip install -U pytest + sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 281581781b4838fcb25548135ed41b17934db4d9 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 11:32:48 +0200 Subject: [PATCH 026/141] apt install --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 662008da0..65fb27644 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -97,7 +97,7 @@ jobs: # - name: Copy resources from gProfiler image # run: ./scripts/copy_resources_from_image.sh gprofiler - name: pytest pytest pytest - run: python3 -m pip install -U pytest + run: pip install -U pytest # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: ./tests/test.sh --ignore=tests/test_executable.py From 80ffc3c96dbe6f641674ba2ce1e9c74a6bdcac35 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 11:37:24 +0200 Subject: [PATCH 027/141] apt install --- .github/workflows/build-container-and-test-aarch64.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 65fb27644..b9284c68d 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -96,8 +96,8 @@ jobs: # - name: Copy resources from gProfiler image # run: ./scripts/copy_resources_from_image.sh gprofiler - - name: pytest pytest pytest - run: pip install -U pytest + - name: Add user's directory to path + run: export PATH="$HOME/.local/bin:$PATH" # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: ./tests/test.sh --ignore=tests/test_executable.py From 960e7a4271587465c991f7e573b9315a70e8897d Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 11:44:46 +0200 Subject: [PATCH 028/141] apt install --- .github/workflows/build-container-and-test-aarch64.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index b9284c68d..4b82d104d 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -46,7 +46,8 @@ jobs: - self-hosted - public - ARM64 - + env: + PATH: "$HOME/.local/bin:$PATH" # strategy: # fail-fast: false # helps detecting flakiness / errors specific to one Python version # matrix: @@ -96,8 +97,8 @@ jobs: # - name: Copy resources from gProfiler image # run: ./scripts/copy_resources_from_image.sh gprofiler - - name: Add user's directory to path - run: export PATH="$HOME/.local/bin:$PATH" + - name: show user's directory to path + run: echo $PATH # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: ./tests/test.sh --ignore=tests/test_executable.py From 52b9ffb758bdba13002c2690f02ca836a13893cf Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 12:08:06 +0200 Subject: [PATCH 029/141] apt install --- .github/workflows/build-container-and-test-aarch64.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 4b82d104d..3feddd0dd 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -97,8 +97,16 @@ jobs: # - name: Copy resources from gProfiler image # run: ./scripts/copy_resources_from_image.sh gprofiler - - name: show user's directory to path + - name: Add to PATH + shell: bash + run: | + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Check user's directory to path run: echo $PATH + + - name: Check user's directory to path + run: echo $GITHUB_PATH # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: ./tests/test.sh --ignore=tests/test_executable.py From 586e2e6570bacdf8bace1afd41c3306e8c23659e Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 12:18:26 +0200 Subject: [PATCH 030/141] apt install --- .github/workflows/build-container-and-test-aarch64.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 3feddd0dd..4f6f83fb9 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -46,8 +46,6 @@ jobs: - self-hosted - public - ARM64 - env: - PATH: "$HOME/.local/bin:$PATH" # strategy: # fail-fast: false # helps detecting flakiness / errors specific to one Python version # matrix: From 9f72251aaf098c436945faddeada59002188f7ad Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 12:37:43 +0200 Subject: [PATCH 031/141] apt install --- .../build-container-and-test-aarch64.yml | 111 +++++++++--------- 1 file changed, 56 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 4f6f83fb9..a100cd01c 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -3,44 +3,44 @@ name: Lint, build gProfiler container and run tests on aarch64 on: pull_request jobs: - # build-container-aarch64: - # runs-on: - # - self-hosted - # - public - # - ARM64 - - # steps: - # - name: Checkout Code - # uses: actions/checkout@v3 - # with: - # submodules: true - - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v2 - - # - name: Login to DockerHub - # uses: docker/login-action@v1 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - - # # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs - # # more nicely. the tests will then be able to use the built image. - # - name: Build gProfiler image - # run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler - - # - name: Export gProfiler image - # run: mkdir -p output && docker image save gprofiler > output/gprofiler.img - - # - name: Upload the image artifact - # uses: actions/upload-artifact@v2 - # with: - # name: gprofiler.img - # path: output/ - # retention-days: 1 + build-container-aarch64: + runs-on: + - self-hosted + - public + - ARM64 + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # more nicely. the tests will then be able to use the built image. + - name: Build gProfiler image + run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler + + - name: Export gProfiler image + run: mkdir -p output && docker image save gprofiler > output/gprofiler.img + + - name: Upload the image artifact + uses: actions/upload-artifact@v2 + with: + name: gprofiler.img + path: output/ + retention-days: 1 test: - # needs: build-container-aarch64 + needs: build-container-aarch64 runs-on: - self-hosted @@ -72,29 +72,30 @@ jobs: # # same version as used in tests/containers/nodejs/Dockerfile # node-version: 10.x - # - name: Login to DockerHub - # uses: docker/login-action@v1 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - # - name: Checkout Code - # uses: actions/checkout@v3 - # with: - # submodules: true + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true - # - name: Download the executable from previous job - # uses: actions/download-artifact@v2 - # with: - # name: gprofiler.img - # path: output/ + - name: Download the executable from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler.img + path: output/ + + # used for next step (Copy resources ...) and also used in the tests themselves. + - name: Import gProfiler image + run: docker image load < output/gprofiler.img - # # used for next step (Copy resources ...) and also used in the tests themselves. - # - name: Import gProfiler image - # run: docker image load < output/gprofiler.img + - name: Copy resources from gProfiler image + run: ./scripts/copy_resources_from_image.sh gprofiler - # - name: Copy resources from gProfiler image - # run: ./scripts/copy_resources_from_image.sh gprofiler - name: Add to PATH shell: bash run: | From 33eacc3d907d36be428d18a92565ca877eafad6d Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 13:00:48 +0200 Subject: [PATCH 032/141] apt install --- tests/test.sh | 52 +++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/test.sh b/tests/test.sh index c55029f52..3888d03d4 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -5,34 +5,34 @@ # set -e -# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # https://stackoverflow.com/a/246128 +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # https://stackoverflow.com/a/246128 -# if [ -z ${NO_APT_INSTALL+x} ]; then -# sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update -# sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends openjdk-8-jdk python3 python3-pip docker.io php -# if [ "$(uname -m)" = "aarch64" ]; then -# sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends python3-dev ruby build-essential nodejs -# if ! [ -L "/usr/bin/dotnet" ] ; then -# # there is no dotnet apt package on aarch64 -# curl -SL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/master/dotnet-sdk-latest-linux-arm64.tar.gz -# sudo mkdir -p /usr/share/dotnet -# sudo tar -zxf dotnet.tar.gz -C /usr/share/dotnet -# sudo ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet -# fi -# else -# sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends ruby2.7 dotnet-sdk-6.0 -# fi -# fi +if [ -z ${NO_APT_INSTALL+x} ]; then + sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends openjdk-8-jdk python3 python3-pip docker.io php + if [ "$(uname -m)" = "aarch64" ]; then + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends python3-dev ruby build-essential nodejs + if ! [ -L "/usr/bin/dotnet" ] ; then + # there is no dotnet apt package on aarch64 + curl -SL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/master/dotnet-sdk-latest-linux-arm64.tar.gz + sudo mkdir -p /usr/share/dotnet + sudo tar -zxf dotnet.tar.gz -C /usr/share/dotnet + sudo ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet + fi + else + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends ruby2.7 dotnet-sdk-6.0 + fi +fi -# # we'll check 1 file, perf. not if --executable is passed - these are the executable tests, and they don't -# # require resources. -# # TODO split them to 2 pytest files -# PERF_RESOURCE="$SCRIPT_DIR/../gprofiler/resources/perf" -# if [ ! -f "$PERF_RESOURCE" ] && [[ "$*" != *"--executable"* ]]; then -# echo "perf resource not found: $(readlink -f "$PERF_RESOURCE")" -# echo "Please run $(readlink -f "$SCRIPT_DIR/../scripts/copy_resources_from_image.sh") to get all resources" -# exit 1 -# fi +# we'll check 1 file, perf. not if --executable is passed - these are the executable tests, and they don't +# require resources. +# TODO split them to 2 pytest files +PERF_RESOURCE="$SCRIPT_DIR/../gprofiler/resources/perf" +if [ ! -f "$PERF_RESOURCE" ] && [[ "$*" != *"--executable"* ]]; then + echo "perf resource not found: $(readlink -f "$PERF_RESOURCE")" + echo "Please run $(readlink -f "$SCRIPT_DIR/../scripts/copy_resources_from_image.sh") to get all resources" + exit 1 +fi python3 -m pip install -q --upgrade setuptools pip python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt From 5524ea5a0b25c147edf5a61b1809ea95eff07557 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 13:30:22 +0200 Subject: [PATCH 033/141] apt install --- tests/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test.sh b/tests/test.sh index 3888d03d4..d90970b52 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -35,6 +35,7 @@ if [ ! -f "$PERF_RESOURCE" ] && [[ "$*" != *"--executable"* ]]; then fi python3 -m pip install -q --upgrade setuptools pip +python3 -m pip install testresources python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . From ceeb6f6b0a0cd733be0df156f423ed7f5c824cb8 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 14:01:42 +0200 Subject: [PATCH 034/141] pytest? --- tests/test.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test.sh b/tests/test.sh index d90970b52..0364a482e 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -38,5 +38,11 @@ python3 -m pip install -q --upgrade setuptools pip python3 -m pip install testresources python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . - +python3 -c "from pip import _internal; _internal.main(['list'])" +echo "PRZERWATESTTESTTEST" +sudo env "PATH=$PATH" python3 -c "from pip import _internal; _internal.main(['list'])" +which python +which python3 +which pip +which pip3 sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 36c6c0fb26dc3c45b457bfbe1f69094c445e2222 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 9 Aug 2023 16:15:36 +0200 Subject: [PATCH 035/141] pytest? --- .github/workflows/build-container-and-test-aarch64.yml | 2 -- tests/test.sh | 10 ++-------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index a100cd01c..c95ee6e91 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -53,8 +53,6 @@ jobs: # - "3.10.11" steps: - - name: Install python3 - run: sudo apt-get install -y python3 # - name: check python installation # run: python3 --version diff --git a/tests/test.sh b/tests/test.sh index 0364a482e..47cd28b40 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -38,11 +38,5 @@ python3 -m pip install -q --upgrade setuptools pip python3 -m pip install testresources python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . -python3 -c "from pip import _internal; _internal.main(['list'])" -echo "PRZERWATESTTESTTEST" -sudo env "PATH=$PATH" python3 -c "from pip import _internal; _internal.main(['list'])" -which python -which python3 -which pip -which pip3 -sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" + +sudo python3 -m pytest -v tests/ "$@" From 3437f8848eda18daf202d23fae62c10c32908582 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 07:56:09 +0200 Subject: [PATCH 036/141] pytest? --- .../workflows/build-container-and-test-aarch64.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index c95ee6e91..4b29ba08a 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -104,6 +104,18 @@ jobs: - name: Check user's directory to path run: echo $GITHUB_PATH + - name: test1 + run: which pip + - name: test1 + run: sudo which pip + - name: test3 + run: which python3 + - name: test4 + run: sudo which python3 + - name: test5 + run: python3 -c 'import site; print(site.getsitepackages())' + - name: test6 + run: sudo python3 -c 'import site; print(site.getsitepackages())' # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: ./tests/test.sh --ignore=tests/test_executable.py From c316a3bfd5fa33cd32a40ab0410aac7b6a3dc861 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 08:40:51 +0200 Subject: [PATCH 037/141] pytest? --- .../build-container-and-test-aarch64.yml | 116 +++++++++--------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 4b29ba08a..47dcdd268 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -3,44 +3,44 @@ name: Lint, build gProfiler container and run tests on aarch64 on: pull_request jobs: - build-container-aarch64: - runs-on: - - self-hosted - - public - - ARM64 - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs - # more nicely. the tests will then be able to use the built image. - - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler - - - name: Export gProfiler image - run: mkdir -p output && docker image save gprofiler > output/gprofiler.img - - - name: Upload the image artifact - uses: actions/upload-artifact@v2 - with: - name: gprofiler.img - path: output/ - retention-days: 1 + # build-container-aarch64: + # runs-on: + # - self-hosted + # - public + # - ARM64 + + # steps: + # - name: Checkout Code + # uses: actions/checkout@v3 + # with: + # submodules: true + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + + # - name: Login to DockerHub + # uses: docker/login-action@v1 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + + # # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # # more nicely. the tests will then be able to use the built image. + # - name: Build gProfiler image + # run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler + + # - name: Export gProfiler image + # run: mkdir -p output && docker image save gprofiler > output/gprofiler.img + + # - name: Upload the image artifact + # uses: actions/upload-artifact@v2 + # with: + # name: gprofiler.img + # path: output/ + # retention-days: 1 test: - needs: build-container-aarch64 + # needs: build-container-aarch64 runs-on: - self-hosted @@ -70,29 +70,29 @@ jobs: # # same version as used in tests/containers/nodejs/Dockerfile # node-version: 10.x - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + # - name: Login to DockerHub + # uses: docker/login-action@v1 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Download the executable from previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler.img - path: output/ - - # used for next step (Copy resources ...) and also used in the tests themselves. - - name: Import gProfiler image - run: docker image load < output/gprofiler.img - - - name: Copy resources from gProfiler image - run: ./scripts/copy_resources_from_image.sh gprofiler + # - name: Checkout Code + # uses: actions/checkout@v3 + # with: + # submodules: true + + # - name: Download the executable from previous job + # uses: actions/download-artifact@v2 + # with: + # name: gprofiler.img + # path: output/ + + # # used for next step (Copy resources ...) and also used in the tests themselves. + # - name: Import gProfiler image + # run: docker image load < output/gprofiler.img + + # - name: Copy resources from gProfiler image + # run: ./scripts/copy_resources_from_image.sh gprofiler - name: Add to PATH shell: bash From 4eb943314416f6edddbe88946924f2e6597e1679 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 08:44:48 +0200 Subject: [PATCH 038/141] pytest? --- .github/workflows/build-container-and-test-aarch64.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 47dcdd268..7d54a2a3d 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -105,13 +105,13 @@ jobs: - name: Check user's directory to path run: echo $GITHUB_PATH - name: test1 - run: which pip + run: type -a pip - name: test1 - run: sudo which pip + run: sudo type -a pip - name: test3 - run: which python3 + run: type -a python3 - name: test4 - run: sudo which python3 + run: sudo type -a python3 - name: test5 run: python3 -c 'import site; print(site.getsitepackages())' - name: test6 From b279df1179e9dcb2808d4ae0ae85c48f0c744b45 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 08:49:06 +0200 Subject: [PATCH 039/141] pytest? --- .../workflows/build-container-and-test-aarch64.yml | 13 +------------ tests/test.sh | 7 ++++++- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 7d54a2a3d..e22d86692 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -104,18 +104,7 @@ jobs: - name: Check user's directory to path run: echo $GITHUB_PATH - - name: test1 - run: type -a pip - - name: test1 - run: sudo type -a pip - - name: test3 - run: type -a python3 - - name: test4 - run: sudo type -a python3 - - name: test5 - run: python3 -c 'import site; print(site.getsitepackages())' - - name: test6 - run: sudo python3 -c 'import site; print(site.getsitepackages())' + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: ./tests/test.sh --ignore=tests/test_executable.py diff --git a/tests/test.sh b/tests/test.sh index 47cd28b40..b44f673d7 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -38,5 +38,10 @@ python3 -m pip install -q --upgrade setuptools pip python3 -m pip install testresources python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . - +type -a pip +sudo type -a pip +type -a python3 +sudo type -a python3 +python3 -c 'import site; print(site.getsitepackages())' +sudo python3 -c 'import site; print(site.getsitepackages())' sudo python3 -m pytest -v tests/ "$@" From 74f6e20c9bc6c587d953c75caf92121b31523fe1 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 08:56:32 +0200 Subject: [PATCH 040/141] pytest? --- tests/test.sh | 52 +++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/test.sh b/tests/test.sh index b44f673d7..bdad960a8 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -5,34 +5,34 @@ # set -e -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # https://stackoverflow.com/a/246128 +# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # https://stackoverflow.com/a/246128 -if [ -z ${NO_APT_INSTALL+x} ]; then - sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends openjdk-8-jdk python3 python3-pip docker.io php - if [ "$(uname -m)" = "aarch64" ]; then - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends python3-dev ruby build-essential nodejs - if ! [ -L "/usr/bin/dotnet" ] ; then - # there is no dotnet apt package on aarch64 - curl -SL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/master/dotnet-sdk-latest-linux-arm64.tar.gz - sudo mkdir -p /usr/share/dotnet - sudo tar -zxf dotnet.tar.gz -C /usr/share/dotnet - sudo ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet - fi - else - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends ruby2.7 dotnet-sdk-6.0 - fi -fi +# if [ -z ${NO_APT_INSTALL+x} ]; then +# sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update +# sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends openjdk-8-jdk python3 python3-pip docker.io php +# if [ "$(uname -m)" = "aarch64" ]; then +# sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends python3-dev ruby build-essential nodejs +# if ! [ -L "/usr/bin/dotnet" ] ; then +# # there is no dotnet apt package on aarch64 +# curl -SL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/master/dotnet-sdk-latest-linux-arm64.tar.gz +# sudo mkdir -p /usr/share/dotnet +# sudo tar -zxf dotnet.tar.gz -C /usr/share/dotnet +# sudo ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet +# fi +# else +# sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends ruby2.7 dotnet-sdk-6.0 +# fi +# fi -# we'll check 1 file, perf. not if --executable is passed - these are the executable tests, and they don't -# require resources. -# TODO split them to 2 pytest files -PERF_RESOURCE="$SCRIPT_DIR/../gprofiler/resources/perf" -if [ ! -f "$PERF_RESOURCE" ] && [[ "$*" != *"--executable"* ]]; then - echo "perf resource not found: $(readlink -f "$PERF_RESOURCE")" - echo "Please run $(readlink -f "$SCRIPT_DIR/../scripts/copy_resources_from_image.sh") to get all resources" - exit 1 -fi +# # we'll check 1 file, perf. not if --executable is passed - these are the executable tests, and they don't +# # require resources. +# # TODO split them to 2 pytest files +# PERF_RESOURCE="$SCRIPT_DIR/../gprofiler/resources/perf" +# if [ ! -f "$PERF_RESOURCE" ] && [[ "$*" != *"--executable"* ]]; then +# echo "perf resource not found: $(readlink -f "$PERF_RESOURCE")" +# echo "Please run $(readlink -f "$SCRIPT_DIR/../scripts/copy_resources_from_image.sh") to get all resources" +# exit 1 +# fi python3 -m pip install -q --upgrade setuptools pip python3 -m pip install testresources From 46ec91304b2ff7976fa7c3772db0c4507a1c9eb5 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 09:12:31 +0200 Subject: [PATCH 041/141] pytest? --- tests/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.sh b/tests/test.sh index bdad960a8..7554be394 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -44,4 +44,4 @@ type -a python3 sudo type -a python3 python3 -c 'import site; print(site.getsitepackages())' sudo python3 -c 'import site; print(site.getsitepackages())' -sudo python3 -m pytest -v tests/ "$@" +sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 8497de2401be85a9bd94a2c621a768f97d9dac27 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 09:15:38 +0200 Subject: [PATCH 042/141] pytest? --- .github/workflows/build-container-and-test-aarch64.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index e22d86692..85fcd980a 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -76,10 +76,10 @@ jobs: # username: ${{ secrets.DOCKERHUB_USERNAME }} # password: ${{ secrets.DOCKERHUB_TOKEN }} - # - name: Checkout Code - # uses: actions/checkout@v3 - # with: - # submodules: true + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true # - name: Download the executable from previous job # uses: actions/download-artifact@v2 From 68bdeefd577a846b344994950fa129aa7c132307 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 09:18:23 +0200 Subject: [PATCH 043/141] pytest? --- tests/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test.sh b/tests/test.sh index 7554be394..68d2866d7 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -39,9 +39,9 @@ python3 -m pip install testresources python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . type -a pip -sudo type -a pip + type -a python3 -sudo type -a python3 +echo $PATH python3 -c 'import site; print(site.getsitepackages())' sudo python3 -c 'import site; print(site.getsitepackages())' sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 04354c7ffcc04603421d2cca11e61ee40eb712f8 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 09:21:51 +0200 Subject: [PATCH 044/141] pytest? --- tests/test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test.sh b/tests/test.sh index 68d2866d7..14f262ee9 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -43,5 +43,6 @@ type -a pip type -a python3 echo $PATH python3 -c 'import site; print(site.getsitepackages())' -sudo python3 -c 'import site; print(site.getsitepackages())' +sudo python3 -c 'import site; print(site.getsitepackages())' +sudo env "PATH=$PATH" python3 -c 'import site; print(site.getsitepackages())' sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From d3b5e9c2add5e01acef65eeb95b07a277fef964d Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 10:30:30 +0200 Subject: [PATCH 045/141] pytest? --- .github/workflows/build-container-and-test-aarch64.yml | 4 ++-- tests/test.sh | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 85fcd980a..8a9157eb6 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -94,10 +94,10 @@ jobs: # - name: Copy resources from gProfiler image # run: ./scripts/copy_resources_from_image.sh gprofiler - - name: Add to PATH + - name: Add to PYTHONPATH shell: bash run: | - echo "$HOME/.local/bin" >> $GITHUB_PATH + echo "PYTHONPATH=$HOME/.local/lib:$PYTHONPATH" >> $GITHUB_ENV - name: Check user's directory to path run: echo $PATH diff --git a/tests/test.sh b/tests/test.sh index 14f262ee9..a447ce0cf 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -38,11 +38,4 @@ python3 -m pip install -q --upgrade setuptools pip python3 -m pip install testresources python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . -type -a pip - -type -a python3 -echo $PATH -python3 -c 'import site; print(site.getsitepackages())' -sudo python3 -c 'import site; print(site.getsitepackages())' -sudo env "PATH=$PATH" python3 -c 'import site; print(site.getsitepackages())' sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From a3811c8591d230d9762d0bfa612dc24dc732aa94 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 10:33:59 +0200 Subject: [PATCH 046/141] pytest? --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- tests/test.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 8a9157eb6..b2c37336b 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -103,7 +103,7 @@ jobs: run: echo $PATH - name: Check user's directory to path - run: echo $GITHUB_PATH + run: echo $PYTHONPATH # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests diff --git a/tests/test.sh b/tests/test.sh index a447ce0cf..fafe73307 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -38,4 +38,5 @@ python3 -m pip install -q --upgrade setuptools pip python3 -m pip install testresources python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . +python3 -m pip show pytest sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 356bc6f3e50b841e7a90fee9e96e13315a75c90f Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 10:54:35 +0200 Subject: [PATCH 047/141] PYTHONHOME? --- .github/workflows/build-container-and-test-aarch64.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index b2c37336b..e12a92c5e 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -97,13 +97,13 @@ jobs: - name: Add to PYTHONPATH shell: bash run: | - echo "PYTHONPATH=$HOME/.local/lib:$PYTHONPATH" >> $GITHUB_ENV + echo "PYTHONHOME=$HOME/.local/lib/python`(python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')`:$PYTHONHOME" >> $GITHUB_ENV - name: Check user's directory to path run: echo $PATH - name: Check user's directory to path - run: echo $PYTHONPATH + run: echo $PYTHONHOME # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 0b1fc9ba689f583c4565bfe04d7bbc6e81c1ca97 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:02:53 +0200 Subject: [PATCH 048/141] PYTHONHOME? --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index e12a92c5e..133059fc9 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -97,7 +97,7 @@ jobs: - name: Add to PYTHONPATH shell: bash run: | - echo "PYTHONHOME=$HOME/.local/lib/python`(python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')`:$PYTHONHOME" >> $GITHUB_ENV + echo "PYTHONHOME=$HOME/.local/lib/python`(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')`:$PYTHONHOME" >> $GITHUB_ENV - name: Check user's directory to path run: echo $PATH From 4353bac42720d998dc34305572a0406b0ec46221 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:05:05 +0200 Subject: [PATCH 049/141] PYTHONHOME? --- .github/workflows/build-container-and-test-aarch64.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 133059fc9..c176306a4 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -97,13 +97,13 @@ jobs: - name: Add to PYTHONPATH shell: bash run: | - echo "PYTHONHOME=$HOME/.local/lib/python`(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')`:$PYTHONHOME" >> $GITHUB_ENV + echo "PYTHONPATH=$HOME/.local/lib/python`(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')`:$PYTHONPATH" >> $GITHUB_ENV - name: Check user's directory to path run: echo $PATH - name: Check user's directory to path - run: echo $PYTHONHOME + run: echo $PYTHONPATH # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 1617c95976ac4d1b3036a611ca2c773eb8423505 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:11:32 +0200 Subject: [PATCH 050/141] PYTHONHOME? --- tests/test.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test.sh b/tests/test.sh index fafe73307..5d34d4337 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -5,24 +5,24 @@ # set -e -# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # https://stackoverflow.com/a/246128 +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # https://stackoverflow.com/a/246128 -# if [ -z ${NO_APT_INSTALL+x} ]; then -# sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update -# sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends openjdk-8-jdk python3 python3-pip docker.io php -# if [ "$(uname -m)" = "aarch64" ]; then -# sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends python3-dev ruby build-essential nodejs -# if ! [ -L "/usr/bin/dotnet" ] ; then -# # there is no dotnet apt package on aarch64 -# curl -SL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/master/dotnet-sdk-latest-linux-arm64.tar.gz -# sudo mkdir -p /usr/share/dotnet -# sudo tar -zxf dotnet.tar.gz -C /usr/share/dotnet -# sudo ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet -# fi -# else -# sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends ruby2.7 dotnet-sdk-6.0 -# fi -# fi +if [ -z ${NO_APT_INSTALL+x} ]; then + sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends openjdk-8-jdk python3 python3-pip docker.io php + if [ "$(uname -m)" = "aarch64" ]; then + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends python3-dev ruby build-essential nodejs + if ! [ -L "/usr/bin/dotnet" ] ; then + # there is no dotnet apt package on aarch64 + curl -SL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/master/dotnet-sdk-latest-linux-arm64.tar.gz + sudo mkdir -p /usr/share/dotnet + sudo tar -zxf dotnet.tar.gz -C /usr/share/dotnet + sudo ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet + fi + else + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends ruby2.7 dotnet-sdk-6.0 + fi +fi # # we'll check 1 file, perf. not if --executable is passed - these are the executable tests, and they don't # # require resources. From 7672787695d1c9838cba6df50c6210549d6b4f7d Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:22:39 +0200 Subject: [PATCH 051/141] PYTHONHOME? --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index c176306a4..b2e8b33f6 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -97,7 +97,7 @@ jobs: - name: Add to PYTHONPATH shell: bash run: | - echo "PYTHONPATH=$HOME/.local/lib/python`(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')`:$PYTHONPATH" >> $GITHUB_ENV + echo "PYTHONPATH=$HOME/.local/lib/python`(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')`" >> $GITHUB_ENV - name: Check user's directory to path run: echo $PATH From ccef714cf0da7fe0c2e1050c51c1de2f2234d7ce Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:29:31 +0200 Subject: [PATCH 052/141] PYTHONPATH? --- tests/test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test.sh b/tests/test.sh index 5d34d4337..dc2b9406b 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -38,5 +38,6 @@ python3 -m pip install -q --upgrade setuptools pip python3 -m pip install testresources python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . -python3 -m pip show pytest +sudo python3 -m pip show pytest +sudo env "PATH=$PATH" python3 -m pip show pytest sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 8985c8f47165e6bd2abb3d81501c4bb83fd3144c Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:32:14 +0200 Subject: [PATCH 053/141] PYTHONPATH? --- tests/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.sh b/tests/test.sh index dc2b9406b..73b6b016c 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -39,5 +39,5 @@ python3 -m pip install testresources python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . sudo python3 -m pip show pytest -sudo env "PATH=$PATH" python3 -m pip show pytest +sudo env "PYTHONPATH=$PYTHONPATH" python3 -m pip show pytest sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 50459982f12446ff4f23d8601d175c234d77b009 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:35:07 +0200 Subject: [PATCH 054/141] PYTHONPATH? --- tests/test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test.sh b/tests/test.sh index 73b6b016c..853ae830a 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -38,6 +38,5 @@ python3 -m pip install -q --upgrade setuptools pip python3 -m pip install testresources python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . -sudo python3 -m pip show pytest sudo env "PYTHONPATH=$PYTHONPATH" python3 -m pip show pytest sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 976799693d5b48b773cf8464d9b6302a37551ad4 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:39:01 +0200 Subject: [PATCH 055/141] PYTHONPATH? --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- tests/test.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index b2e8b33f6..3d8d163e7 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -97,7 +97,7 @@ jobs: - name: Add to PYTHONPATH shell: bash run: | - echo "PYTHONPATH=$HOME/.local/lib/python`(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')`" >> $GITHUB_ENV + echo "PYTHONPATH=$HOME/.local/lib/python`(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')`/site-packages" >> $GITHUB_ENV - name: Check user's directory to path run: echo $PATH diff --git a/tests/test.sh b/tests/test.sh index 853ae830a..7c24b136b 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -38,5 +38,6 @@ python3 -m pip install -q --upgrade setuptools pip python3 -m pip install testresources python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . +python3 -m pip show pytest sudo env "PYTHONPATH=$PYTHONPATH" python3 -m pip show pytest sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From fdd60840d73585b7512a2391aa392e1adb5bd59e Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:43:46 +0200 Subject: [PATCH 056/141] PYTHONPATH? --- tests/test.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test.sh b/tests/test.sh index 7c24b136b..b419d0567 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -34,10 +34,8 @@ fi # exit 1 # fi -python3 -m pip install -q --upgrade setuptools pip -python3 -m pip install testresources -python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt +sudo python3 -m pip install -q --upgrade setuptools pip +sudo python3 -m pip install testresources +sudo python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . -python3 -m pip show pytest -sudo env "PYTHONPATH=$PYTHONPATH" python3 -m pip show pytest sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 0288e87e0ffbf77cec4acb6cde84966786556787 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:46:52 +0200 Subject: [PATCH 057/141] PYTHONPATH? --- tests/test.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test.sh b/tests/test.sh index b419d0567..0da41165e 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -33,9 +33,12 @@ fi # echo "Please run $(readlink -f "$SCRIPT_DIR/../scripts/copy_resources_from_image.sh") to get all resources" # exit 1 # fi - -sudo python3 -m pip install -q --upgrade setuptools pip -sudo python3 -m pip install testresources -sudo python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt +if [ "$(uname -m)" = "aarch64" ]; then + sudo python3 -m pip install -q --upgrade setuptools pip + sudo python3 -m pip install --ignore-installed -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt +else + python3 -m pip install -q --upgrade setuptools pip + python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt +fi # TODO: python3 -m pip install . sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From e258e9aed3ef9dab83d15a47640a55a8f3c828e7 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:49:50 +0200 Subject: [PATCH 058/141] PYTHONPATH? --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- tests/test.sh | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 3d8d163e7..7b8fb083b 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -107,4 +107,4 @@ jobs: # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests - run: ./tests/test.sh --ignore=tests/test_executable.py + run: sudo ./tests/test.sh --ignore=tests/test_executable.py diff --git a/tests/test.sh b/tests/test.sh index 0da41165e..3de948029 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -33,12 +33,8 @@ fi # echo "Please run $(readlink -f "$SCRIPT_DIR/../scripts/copy_resources_from_image.sh") to get all resources" # exit 1 # fi -if [ "$(uname -m)" = "aarch64" ]; then - sudo python3 -m pip install -q --upgrade setuptools pip - sudo python3 -m pip install --ignore-installed -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt -else - python3 -m pip install -q --upgrade setuptools pip - python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt -fi +python3 -m pip install -q --upgrade setuptools pip +python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt + # TODO: python3 -m pip install . sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From b0420e707af3099fefcd05fb9ef67934787c12ff Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:53:43 +0200 Subject: [PATCH 059/141] PYTHONPATH? --- .github/workflows/build-container-and-test-aarch64.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 7b8fb083b..b238b39de 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -75,7 +75,8 @@ jobs: # with: # username: ${{ secrets.DOCKERHUB_USERNAME }} # password: ${{ secrets.DOCKERHUB_TOKEN }} - + - name: clean sudo python shit + run: sudo rm -r /home/ubuntu/actions-runner/_work/gprofiler/gprofiler/gprofiler/__pycache__ - name: Checkout Code uses: actions/checkout@v3 with: From 75e716c57d8fca6bf389467241784aadc77698dc Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:56:01 +0200 Subject: [PATCH 060/141] PYTHONPATH? --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index b238b39de..6f8010104 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -76,7 +76,7 @@ jobs: # username: ${{ secrets.DOCKERHUB_USERNAME }} # password: ${{ secrets.DOCKERHUB_TOKEN }} - name: clean sudo python shit - run: sudo rm -r /home/ubuntu/actions-runner/_work/gprofiler/gprofiler/gprofiler/__pycache__ + run: sudo rm -r /home/ubuntu/actions-runner/_work/gprofiler/ - name: Checkout Code uses: actions/checkout@v3 with: From 3664265777fed34bbccdb6427703628a0c2d4ca9 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 11:57:55 +0200 Subject: [PATCH 061/141] PYTHONPATH? --- .github/workflows/build-container-and-test-aarch64.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 6f8010104..b27be057c 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -75,8 +75,6 @@ jobs: # with: # username: ${{ secrets.DOCKERHUB_USERNAME }} # password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: clean sudo python shit - run: sudo rm -r /home/ubuntu/actions-runner/_work/gprofiler/ - name: Checkout Code uses: actions/checkout@v3 with: From 620b8d0759f7ef1bbdef8b147454c9f3b3bba029 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 12:19:33 +0200 Subject: [PATCH 062/141] BUILD MAN --- .../build-container-and-test-aarch64.yml | 113 ++++++++---------- tests/test.sh | 18 +-- 2 files changed, 60 insertions(+), 71 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index b27be057c..a26f46591 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -3,44 +3,44 @@ name: Lint, build gProfiler container and run tests on aarch64 on: pull_request jobs: - # build-container-aarch64: - # runs-on: - # - self-hosted - # - public - # - ARM64 - - # steps: - # - name: Checkout Code - # uses: actions/checkout@v3 - # with: - # submodules: true - - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v2 - - # - name: Login to DockerHub - # uses: docker/login-action@v1 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - - # # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs - # # more nicely. the tests will then be able to use the built image. - # - name: Build gProfiler image - # run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler - - # - name: Export gProfiler image - # run: mkdir -p output && docker image save gprofiler > output/gprofiler.img - - # - name: Upload the image artifact - # uses: actions/upload-artifact@v2 - # with: - # name: gprofiler.img - # path: output/ - # retention-days: 1 + build-container-aarch64: + runs-on: + - self-hosted + - public + - ARM64 + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # more nicely. the tests will then be able to use the built image. + - name: Build gProfiler image + run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler + + - name: Export gProfiler image + run: mkdir -p output && docker image save gprofiler > output/gprofiler.img + + - name: Upload the image artifact + uses: actions/upload-artifact@v2 + with: + name: gprofiler.img + path: output/ + retention-days: 1 test: - # needs: build-container-aarch64 + needs: build-container-aarch64 runs-on: - self-hosted @@ -70,39 +70,28 @@ jobs: # # same version as used in tests/containers/nodejs/Dockerfile # node-version: 10.x - # - name: Login to DockerHub - # uses: docker/login-action@v1 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Checkout Code uses: actions/checkout@v3 with: submodules: true - # - name: Download the executable from previous job - # uses: actions/download-artifact@v2 - # with: - # name: gprofiler.img - # path: output/ - - # # used for next step (Copy resources ...) and also used in the tests themselves. - # - name: Import gProfiler image - # run: docker image load < output/gprofiler.img - - # - name: Copy resources from gProfiler image - # run: ./scripts/copy_resources_from_image.sh gprofiler - - - name: Add to PYTHONPATH - shell: bash - run: | - echo "PYTHONPATH=$HOME/.local/lib/python`(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')`/site-packages" >> $GITHUB_ENV + - name: Download the executable from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler.img + path: output/ - - name: Check user's directory to path - run: echo $PATH + # used for next step (Copy resources ...) and also used in the tests themselves. + - name: Import gProfiler image + run: docker image load < output/gprofiler.img - - name: Check user's directory to path - run: echo $PYTHONPATH + - name: Copy resources from gProfiler image + run: ./scripts/copy_resources_from_image.sh gprofiler # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests diff --git a/tests/test.sh b/tests/test.sh index 3de948029..467111db7 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -24,15 +24,15 @@ if [ -z ${NO_APT_INSTALL+x} ]; then fi fi -# # we'll check 1 file, perf. not if --executable is passed - these are the executable tests, and they don't -# # require resources. -# # TODO split them to 2 pytest files -# PERF_RESOURCE="$SCRIPT_DIR/../gprofiler/resources/perf" -# if [ ! -f "$PERF_RESOURCE" ] && [[ "$*" != *"--executable"* ]]; then -# echo "perf resource not found: $(readlink -f "$PERF_RESOURCE")" -# echo "Please run $(readlink -f "$SCRIPT_DIR/../scripts/copy_resources_from_image.sh") to get all resources" -# exit 1 -# fi +# we'll check 1 file, perf. not if --executable is passed - these are the executable tests, and they don't +# require resources. +# TODO split them to 2 pytest files +PERF_RESOURCE="$SCRIPT_DIR/../gprofiler/resources/perf" +if [ ! -f "$PERF_RESOURCE" ] && [[ "$*" != *"--executable"* ]]; then + echo "perf resource not found: $(readlink -f "$PERF_RESOURCE")" + echo "Please run $(readlink -f "$SCRIPT_DIR/../scripts/copy_resources_from_image.sh") to get all resources" + exit 1 +fi python3 -m pip install -q --upgrade setuptools pip python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt From 00437e3ea240d4e5d1c8d5f902441d716a058da6 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 12:23:12 +0200 Subject: [PATCH 063/141] BUILD MAN --- .github/workflows/build-container-and-test-aarch64.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index a26f46591..6f968615e 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -10,6 +10,9 @@ jobs: - ARM64 steps: + - name: remove incorrect pycache permissions + run: chmod -R 755 /home/ubuntu/actions-runner/_work/gprofiler/gprofiler + - name: Checkout Code uses: actions/checkout@v3 with: From b93f1b3027a3239af12dc90599888257cfaa92bd Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 12:25:02 +0200 Subject: [PATCH 064/141] BUILD MAN --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 6f968615e..5071026b2 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -11,7 +11,7 @@ jobs: steps: - name: remove incorrect pycache permissions - run: chmod -R 755 /home/ubuntu/actions-runner/_work/gprofiler/gprofiler + run: sudo chmod -R 755 /home/ubuntu/actions-runner/_work/gprofiler/gprofiler - name: Checkout Code uses: actions/checkout@v3 From c724215c8b0ada2e2ae921cd48efe52cf8511bfa Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 10 Aug 2023 12:28:34 +0200 Subject: [PATCH 065/141] BUILD MAN --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 5071026b2..75884fc57 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -11,7 +11,7 @@ jobs: steps: - name: remove incorrect pycache permissions - run: sudo chmod -R 755 /home/ubuntu/actions-runner/_work/gprofiler/gprofiler + run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler - name: Checkout Code uses: actions/checkout@v3 From f56031c4ea98ea990920b20043df74b48dd5c4d9 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Fri, 11 Aug 2023 09:32:57 +0200 Subject: [PATCH 066/141] ignore installed --- tests/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.sh b/tests/test.sh index 467111db7..b1e90eea7 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -34,7 +34,7 @@ if [ ! -f "$PERF_RESOURCE" ] && [[ "$*" != *"--executable"* ]]; then exit 1 fi python3 -m pip install -q --upgrade setuptools pip -python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt +python3 -m pip install --ignore-installed -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 81d88972182eed36a2e4cd52e2664f2a50ef339e Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Fri, 11 Aug 2023 11:09:50 +0200 Subject: [PATCH 067/141] other flows back --- .../workflows/build-container-and-deploy.yml | 114 +++++++++++++ .../build-container-and-test-aarch64.yml | 22 --- .../workflows/build-container-and-test.yml | 0 .../build-executable-test-deploy.yml | 152 ++++++++++++++++++ .github/workflows/build-on-windows.yml | 18 +++ .github/workflows/linters.yml | 19 +++ tests/test.sh | 1 + 7 files changed, 304 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/build-container-and-deploy.yml create mode 100644 .github/workflows/build-container-and-test.yml create mode 100644 .github/workflows/build-executable-test-deploy.yml create mode 100644 .github/workflows/build-on-windows.yml create mode 100644 .github/workflows/linters.yml diff --git a/.github/workflows/build-container-and-deploy.yml b/.github/workflows/build-container-and-deploy.yml new file mode 100644 index 000000000..6c73340bd --- /dev/null +++ b/.github/workflows/build-container-and-deploy.yml @@ -0,0 +1,114 @@ +name: Build gProfiler container & deploy + +on: + push: + tags: + - '**' + +jobs: + build-docker-x64: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + - name: Get and verify tag value + run: ./scripts/verify_tag.sh + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" + - name: Build and push + run: | + set -x + export DOCKER_BUILDKIT=1 + BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" + LATEST_IMAGE="${{ env.GH_REPO }}:latest" + X86_64_IMAGE="$BASE_IMAGE-x86_64" + # build & push x86_64 + ./scripts/build_x86_64_container.sh -t "$X86_64_IMAGE" + docker push "$X86_64_IMAGE" + build-docker-aarch64: + runs-on: ["self-hosted", "ARM64"] + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + - name: Set up runner requirements + run: scripts/setup_runner_requirements.sh + + - name: Get and verify tag value + run: ./scripts/verify_tag.sh + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" + - name: Build and push + run: | + set -x + export DOCKER_BUILDKIT=1 + BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" + LATEST_IMAGE="${{ env.GH_REPO }}:latest" + AARCH64_IMAGE="$BASE_IMAGE-aarch64" + # build & push Aarch64 + ./scripts/build_aarch64_container.sh -t "$AARCH64_IMAGE" --push + push-manifest: + runs-on: ubuntu-latest + needs: + - build-docker-x64 + - build-docker-aarch64 + steps: + # needed to set up env.GH_REPO etc + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + - name: Get and verify tag value + run: ./scripts/verify_tag.sh + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Push manifest + run: | + set -x + export DOCKER_BUILDKIT=1 + BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" + LATEST_IMAGE="${{ env.GH_REPO }}:latest" + AARCH64_IMAGE="$BASE_IMAGE-aarch64" + X86_64_IMAGE="$BASE_IMAGE-x86_64" + docker pull $AARCH64_IMAGE + docker pull $X86_64_IMAGE + # create manifests for the tag + for 'latest' + docker manifest create "$BASE_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" + docker manifest push "$BASE_IMAGE" + docker manifest create "$LATEST_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" + docker manifest push "$LATEST_IMAGE" \ No newline at end of file diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 75884fc57..366b94845 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -49,29 +49,7 @@ jobs: - self-hosted - public - ARM64 - # strategy: - # fail-fast: false # helps detecting flakiness / errors specific to one Python version - # matrix: - # python-version: - # - "3.10.11" - steps: - - # - name: check python installation - # run: python3 --version - - # - name: Install Java - # uses: actions/setup-java@v1 - # with: - # java-version: '8.0.275' - # java-package: jdk - # architecture: aarch64 - - # - name: Install Node.JS - # uses: actions/setup-node@v2 - # with: - # # same version as used in tests/containers/nodejs/Dockerfile - # node-version: 10.x - name: Login to DockerHub uses: docker/login-action@v1 diff --git a/.github/workflows/build-container-and-test.yml b/.github/workflows/build-container-and-test.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/build-executable-test-deploy.yml b/.github/workflows/build-executable-test-deploy.yml new file mode 100644 index 000000000..04a7bf295 --- /dev/null +++ b/.github/workflows/build-executable-test-deploy.yml @@ -0,0 +1,152 @@ +name: Build gProfiler executable, test (if PR) and deploy (if tag) + +on: + pull_request: + push: + tags: + - '**' + +jobs: + build-executable: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + - name: Get and verify tag value + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + ./scripts/cmp_tags.sh + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" + - name: Build gProfiler executable + # Using DOCKER_BUILDKIT=1 although it has another cache mechanism which is not supported by satackey/action-docker-layer-caching@v0.0.11 + # We tried to cache using buildx cache (cache-from, cache-to flags) and got strange behavior when caching, + # decided not to use buildkit here to get nice caches. + # if we enable caching again, we can disable DOCKER_BUILDKIT. + run: | + mkdir -p output + ./scripts/build_x86_64_executable.sh + mv build/x86_64/gprofiler output/gprofiler_x86_64 + cp output/gprofiler_x86_64 output/gprofiler # for backwards compatibility + - name: Upload the executables as job artifacts + uses: actions/upload-artifact@v2 + with: + name: gprofiler + path: output/ + retention-days: 1 + + test-executable: + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + runs-on: ubuntu-20.04 # the tests which use ruby/node/python etc and run at non-root privs, fail to read the files when running + # on ubuntu-22.04/ubuntu-latest: + # stderr: ruby: Permission denied -- /home/runner/work/gprofiler/gprofiler/tests/containers/ruby/fibonacci.rb (LoadError) + needs: build-executable + strategy: + fail-fast: false + matrix: + containers: + - alpine + - ubuntu:14.04 + - ubuntu:16.04 + - ubuntu:18.04 + - ubuntu:20.04 + - ubuntu:20.10 + - ubuntu:22.04 + - centos:6 + - centos:7 + - centos:8 + - debian:8 + - debian:9 + - debian:10 + - debian:11 + steps: + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: '8.0.275' + java-package: jdk + architecture: x64 + + - name: Install Node.JS + uses: actions/setup-node@v2 + with: + # same version as used in tests/containers/nodejs/Dockerfile + node-version: 10.x + + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Download the executable from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler + path: dist/ + + - name: Run gProfiler tests + run: | + mv dist/gprofiler_x86_64 dist/gprofiler + chmod +x dist/gprofiler + NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable + build-executable-aarch64: + if: startsWith(github.ref, 'refs/tags/') + runs-on: ["self-hosted", "ARM64"] + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + - name: Set up requirements + run: scripts/setup_runner_requirements.sh + + - name: Get and verify tag value + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + ./scripts/cmp_tags.sh + - name: Build gProfiler executable + run: | + mkdir -p output + ./scripts/build_aarch64_executable.sh + mv build/aarch64/gprofiler output/gprofiler_aarch64 + - name: Upload the executables as job artifacts + uses: actions/upload-artifact@v2 + with: + name: gprofiler + path: output/ + retention-days: 1 + + deploy-executable: + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-20.04 + needs: + - build-executable + - build-executable-aarch64 + + steps: + - name: Download executables from the previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler + path: output/ + - name: Get tag value + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Release + uses: softprops/action-gh-release@v1 + with: + files: | + output/gprofiler_x86_64 + output/gprofiler_aarch64 + output/gprofiler + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/build-on-windows.yml b/.github/workflows/build-on-windows.yml new file mode 100644 index 000000000..8c0c8c1c3 --- /dev/null +++ b/.github/workflows/build-on-windows.yml @@ -0,0 +1,18 @@ +name: Build gProfiler on Windows + +on: pull_request + +jobs: + build-executable: + runs-on: windows-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: run the build script + run: .\scripts\windows\build.bat + + - name: check gprofiler build + run: .\app\dist\gprofiler.exe -h \ No newline at end of file diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml new file mode 100644 index 000000000..1b75466db --- /dev/null +++ b/.github/workflows/linters.yml @@ -0,0 +1,19 @@ +name: Non-Python linters + +on: pull_request + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Run shell linter + run: ./shell_lint.sh + + - name: Run Dockerfile linters + run: ./dockerfile_lint.sh \ No newline at end of file diff --git a/tests/test.sh b/tests/test.sh index b1e90eea7..8df72942e 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -33,6 +33,7 @@ if [ ! -f "$PERF_RESOURCE" ] && [[ "$*" != *"--executable"* ]]; then echo "Please run $(readlink -f "$SCRIPT_DIR/../scripts/copy_resources_from_image.sh") to get all resources" exit 1 fi + python3 -m pip install -q --upgrade setuptools pip python3 -m pip install --ignore-installed -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt From c6c61ae08badd788e280a6f00ac2352251211161 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Fri, 11 Aug 2023 11:35:00 +0200 Subject: [PATCH 068/141] other flows back --- .../workflows/build-container-and-deploy.yml | 10 +- .../workflows/build-container-and-test.yml | 111 ++++++++++++++++++ .../build-executable-test-deploy.yml | 7 +- .github/workflows/build-on-windows.yml | 2 +- .github/workflows/linters.yml | 2 +- 5 files changed, 128 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-container-and-deploy.yml b/.github/workflows/build-container-and-deploy.yml index 6c73340bd..096f9b351 100644 --- a/.github/workflows/build-container-and-deploy.yml +++ b/.github/workflows/build-container-and-deploy.yml @@ -32,9 +32,11 @@ jobs: run: | set -x export DOCKER_BUILDKIT=1 + BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" LATEST_IMAGE="${{ env.GH_REPO }}:latest" X86_64_IMAGE="$BASE_IMAGE-x86_64" + # build & push x86_64 ./scripts/build_x86_64_container.sh -t "$X86_64_IMAGE" docker push "$X86_64_IMAGE" @@ -67,11 +69,14 @@ jobs: run: | set -x export DOCKER_BUILDKIT=1 + BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" LATEST_IMAGE="${{ env.GH_REPO }}:latest" AARCH64_IMAGE="$BASE_IMAGE-aarch64" + # build & push Aarch64 ./scripts/build_aarch64_container.sh -t "$AARCH64_IMAGE" --push + push-manifest: runs-on: ubuntu-latest needs: @@ -101,14 +106,17 @@ jobs: run: | set -x export DOCKER_BUILDKIT=1 + BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" LATEST_IMAGE="${{ env.GH_REPO }}:latest" AARCH64_IMAGE="$BASE_IMAGE-aarch64" X86_64_IMAGE="$BASE_IMAGE-x86_64" + docker pull $AARCH64_IMAGE docker pull $X86_64_IMAGE + # create manifests for the tag + for 'latest' docker manifest create "$BASE_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" docker manifest push "$BASE_IMAGE" docker manifest create "$LATEST_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" - docker manifest push "$LATEST_IMAGE" \ No newline at end of file + docker manifest push "$LATEST_IMAGE" diff --git a/.github/workflows/build-container-and-test.yml b/.github/workflows/build-container-and-test.yml index e69de29bb..eb5e6a0e3 100644 --- a/.github/workflows/build-container-and-test.yml +++ b/.github/workflows/build-container-and-test.yml @@ -0,0 +1,111 @@ +name: Lint, build gProfiler container and run tests + +on: pull_request + +jobs: + lint: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + + runs-on: ubuntu-latest + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r dev-requirements.txt + pip install -r requirements.txt + - name: Run linters + run: ./lint.sh --ci + + build-container: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # more nicely. the tests will then be able to use the built image. + - name: Build gProfiler image + run: ./scripts/build_x86_64_container.sh + + - name: Export gProfiler image + run: mkdir -p output && docker image save gprofiler > output/gprofiler.img + + - name: Upload the image artifact + uses: actions/upload-artifact@v2 + with: + name: gprofiler.img + path: output/ + retention-days: 1 + + test: + needs: build-container + + runs-on: ubuntu-20.04 # for 3.10.0 python tag + + strategy: + fail-fast: false # helps detecting flakiness / errors specific to one Python version + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10.0" # see https://github.com/Granulate/gprofiler/issues/502, we need 3.10.0, others fail PyPerf tests. + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: '8.0.275' + java-package: jdk + architecture: x64 + + - name: Install Node.JS + uses: actions/setup-node@v2 + with: + # same version as used in tests/containers/nodejs/Dockerfile + node-version: 10.x + + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Download the executable from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler.img + path: output/ + + # used for next step (Copy resources ...) and also used in the tests themselves. + - name: Import gProfiler image + run: docker image load < output/gprofiler.img + + - name: Copy resources from gProfiler image + run: ./scripts/copy_resources_from_image.sh gprofiler + + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" + - name: Run gProfiler tests + run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py diff --git a/.github/workflows/build-executable-test-deploy.yml b/.github/workflows/build-executable-test-deploy.yml index 04a7bf295..4e7e5cc7e 100644 --- a/.github/workflows/build-executable-test-deploy.yml +++ b/.github/workflows/build-executable-test-deploy.yml @@ -20,6 +20,7 @@ jobs: run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV ./scripts/cmp_tags.sh + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Build gProfiler executable # Using DOCKER_BUILDKIT=1 although it has another cache mechanism which is not supported by satackey/action-docker-layer-caching@v0.0.11 @@ -31,6 +32,7 @@ jobs: ./scripts/build_x86_64_executable.sh mv build/x86_64/gprofiler output/gprofiler_x86_64 cp output/gprofiler_x86_64 output/gprofiler # for backwards compatibility + - name: Upload the executables as job artifacts uses: actions/upload-artifact@v2 with: @@ -97,6 +99,7 @@ jobs: mv dist/gprofiler_x86_64 dist/gprofiler chmod +x dist/gprofiler NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable + build-executable-aarch64: if: startsWith(github.ref, 'refs/tags/') runs-on: ["self-hosted", "ARM64"] @@ -114,11 +117,13 @@ jobs: run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV ./scripts/cmp_tags.sh + - name: Build gProfiler executable run: | mkdir -p output ./scripts/build_aarch64_executable.sh mv build/aarch64/gprofiler output/gprofiler_aarch64 + - name: Upload the executables as job artifacts uses: actions/upload-artifact@v2 with: @@ -149,4 +154,4 @@ jobs: output/gprofiler_aarch64 output/gprofiler env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-on-windows.yml b/.github/workflows/build-on-windows.yml index 8c0c8c1c3..279e5f740 100644 --- a/.github/workflows/build-on-windows.yml +++ b/.github/workflows/build-on-windows.yml @@ -15,4 +15,4 @@ jobs: run: .\scripts\windows\build.bat - name: check gprofiler build - run: .\app\dist\gprofiler.exe -h \ No newline at end of file + run: .\app\dist\gprofiler.exe -h diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 1b75466db..a04764fb5 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -16,4 +16,4 @@ jobs: run: ./shell_lint.sh - name: Run Dockerfile linters - run: ./dockerfile_lint.sh \ No newline at end of file + run: ./dockerfile_lint.sh From fa85bd93196f7f8bee188e496b038eea8a7136d1 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Fri, 11 Aug 2023 11:47:48 +0200 Subject: [PATCH 069/141] other flows back --- .github/workflows/build-container-and-deploy.yml | 1 + .github/workflows/build-container-and-test.yml | 1 + .github/workflows/build-on-windows.yml | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-deploy.yml b/.github/workflows/build-container-and-deploy.yml index 096f9b351..85091a1c9 100644 --- a/.github/workflows/build-container-and-deploy.yml +++ b/.github/workflows/build-container-and-deploy.yml @@ -40,6 +40,7 @@ jobs: # build & push x86_64 ./scripts/build_x86_64_container.sh -t "$X86_64_IMAGE" docker push "$X86_64_IMAGE" + build-docker-aarch64: runs-on: ["self-hosted", "ARM64"] steps: diff --git a/.github/workflows/build-container-and-test.yml b/.github/workflows/build-container-and-test.yml index eb5e6a0e3..c85467b2a 100644 --- a/.github/workflows/build-container-and-test.yml +++ b/.github/workflows/build-container-and-test.yml @@ -29,6 +29,7 @@ jobs: python -m pip install --upgrade pip pip install -r dev-requirements.txt pip install -r requirements.txt + - name: Run linters run: ./lint.sh --ci diff --git a/.github/workflows/build-on-windows.yml b/.github/workflows/build-on-windows.yml index 279e5f740..847befc75 100644 --- a/.github/workflows/build-on-windows.yml +++ b/.github/workflows/build-on-windows.yml @@ -10,7 +10,7 @@ jobs: uses: actions/checkout@v3 with: submodules: true - + - name: run the build script run: .\scripts\windows\build.bat From 84834c264c6b3823fe1ee588fa2271bb85c7c267 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Fri, 11 Aug 2023 12:30:53 +0200 Subject: [PATCH 070/141] other flows back --- .github/workflows/build-container-and-test-aarch64.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 366b94845..06692583a 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -10,7 +10,7 @@ jobs: - ARM64 steps: - - name: remove incorrect pycache permissions + - name: change pycache permissions run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler - name: Checkout Code @@ -56,6 +56,10 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: change pycache permissions + run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler + - name: Checkout Code uses: actions/checkout@v3 with: From daac8bc24c66824fe70a8d764d9de153ce63f6bd Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 16 Aug 2023 10:35:49 +0200 Subject: [PATCH 071/141] is it working with current master? --- .../workflows/build-container-and-test-aarch64.yml | 12 ++++++------ tests/test.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 06692583a..945f7007c 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -30,15 +30,15 @@ jobs: # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs # more nicely. the tests will then be able to use the built image. - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler + run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler_aarch64 - name: Export gProfiler image - run: mkdir -p output && docker image save gprofiler > output/gprofiler.img + run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img - name: Upload the image artifact uses: actions/upload-artifact@v2 with: - name: gprofiler.img + name: gprofiler_aarch64.img path: output/ retention-days: 1 @@ -68,15 +68,15 @@ jobs: - name: Download the executable from previous job uses: actions/download-artifact@v2 with: - name: gprofiler.img + name: gprofiler_aarch64.img path: output/ # used for next step (Copy resources ...) and also used in the tests themselves. - name: Import gProfiler image - run: docker image load < output/gprofiler.img + run: docker image load < output/gprofiler_aarch64.img - name: Copy resources from gProfiler image - run: ./scripts/copy_resources_from_image.sh gprofiler + run: ./scripts/copy_resources_from_image.sh gprofiler_aarch64 # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests diff --git a/tests/test.sh b/tests/test.sh index 0270f28e8..4f62a9b17 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -35,7 +35,7 @@ if [ ! -f "$PERF_RESOURCE" ] && [[ "$*" != *"--executable"* ]]; then fi python3 -m pip install -q --upgrade setuptools pip -python3 -m pip install --ignore-installed -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt +python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt # TODO: python3 -m pip install . sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 0ee2fa160bf070ea57e8c2037c9a5d9ce3a8199a Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 16 Aug 2023 10:40:34 +0200 Subject: [PATCH 072/141] is it working with current master? --- tests/test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test.sh b/tests/test.sh index 4f62a9b17..377e7cf1e 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -36,6 +36,5 @@ fi python3 -m pip install -q --upgrade setuptools pip python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt - # TODO: python3 -m pip install . sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 0bb6425688369acc3bfa397270acf70435af8df3 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 16 Aug 2023 13:54:08 +0200 Subject: [PATCH 073/141] is it working with current master? --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 945f7007c..0afe3f110 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -30,7 +30,7 @@ jobs: # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs # more nicely. the tests will then be able to use the built image. - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh --output type=docker -t gprofiler_aarch64 + run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 - name: Export gProfiler image run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img From 03708900ca4a53b07e2548817a1b5541784bc0e2 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 16 Aug 2023 14:37:27 +0200 Subject: [PATCH 074/141] is it working with current master? --- .github/workflows/build-container-and-test-aarch64.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 0afe3f110..a1d1ce2fa 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -71,13 +71,12 @@ jobs: name: gprofiler_aarch64.img path: output/ - # used for next step (Copy resources ...) and also used in the tests themselves. + - name: Extract resources from gProfiler executable + run: sudo ./output/gprofiler_aarch64.img extract-resources --resources-dest=./gprofiler/resources + - name: Import gProfiler image run: docker image load < output/gprofiler_aarch64.img - - name: Copy resources from gProfiler image - run: ./scripts/copy_resources_from_image.sh gprofiler_aarch64 - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: sudo ./tests/test.sh --ignore=tests/test_executable.py From da0ff0d3a221923c20d344ee0cc10ec7fcb5c422 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 16 Aug 2023 15:13:44 +0200 Subject: [PATCH 075/141] is it working with current master? --- .github/workflows/build-container-and-test-aarch64.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index a1d1ce2fa..4b7805f59 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -71,12 +71,12 @@ jobs: name: gprofiler_aarch64.img path: output/ - - name: Extract resources from gProfiler executable - run: sudo ./output/gprofiler_aarch64.img extract-resources --resources-dest=./gprofiler/resources - - name: Import gProfiler image run: docker image load < output/gprofiler_aarch64.img + - name: Extract resources from gProfiler executable + run: sudo ./output/gprofiler_aarch64 extract-resources --resources-dest=./gprofiler/resources + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: sudo ./tests/test.sh --ignore=tests/test_executable.py From f07d9a1b9fac4b32128f1c70c7b5c79d4bcaed88 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 16 Aug 2023 15:49:59 +0200 Subject: [PATCH 076/141] is it working with current master? --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 4b7805f59..f84ea23c7 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -75,7 +75,7 @@ jobs: run: docker image load < output/gprofiler_aarch64.img - name: Extract resources from gProfiler executable - run: sudo ./output/gprofiler_aarch64 extract-resources --resources-dest=./gprofiler/resources + run: sudo -v ./gprofiler/resources:/app/gprofiler/resources ./output/gprofiler_aarch64 extract-resources --resources-dest=./gprofiler/resources # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From f9607af5454d912e78c01f01a77fd26f25b3d5d1 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 16 Aug 2023 16:58:13 +0200 Subject: [PATCH 077/141] is it working with current master?[A --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index f84ea23c7..bd75a165d 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -75,7 +75,7 @@ jobs: run: docker image load < output/gprofiler_aarch64.img - name: Extract resources from gProfiler executable - run: sudo -v ./gprofiler/resources:/app/gprofiler/resources ./output/gprofiler_aarch64 extract-resources --resources-dest=./gprofiler/resources + run: sudo docker run -v ./gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=./gprofiler/resources # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From c2dc530a44bc424d0651f7a8696859b53ce19580 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 16 Aug 2023 19:04:01 +0200 Subject: [PATCH 078/141] Update build-container-and-test-aarch64.yml --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index bd75a165d..4718b11f2 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -75,7 +75,7 @@ jobs: run: docker image load < output/gprofiler_aarch64.img - name: Extract resources from gProfiler executable - run: sudo docker run -v ./gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=./gprofiler/resources + run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=./gprofiler/resources # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 0c04986011da75968dc1b0df843d2060d46a0800 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 17 Aug 2023 09:40:59 +0200 Subject: [PATCH 079/141] lets extract resources --- .github/workflows/build-test-deploy.yml | 2 +- scripts/prepare_machine-unknown-linux-musl.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 39caf843b..4f86c3413 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -250,7 +250,7 @@ jobs: run: chmod +x ./output/gprofiler - name: Extract resources from gProfiler executable - run: sudo ./output/gprofiler extract-resources --resources-dest=./gprofiler/resources + run: sudo ./output/gprofiler extract-resources --resources-dest=/app/gprofiler/resources # used in the tests - name: Import gProfiler image diff --git a/scripts/prepare_machine-unknown-linux-musl.sh b/scripts/prepare_machine-unknown-linux-musl.sh index 29d7aefd4..e109521b9 100755 --- a/scripts/prepare_machine-unknown-linux-musl.sh +++ b/scripts/prepare_machine-unknown-linux-musl.sh @@ -19,7 +19,7 @@ target="$(uname -m)"-unknown-linux-musl target_dir="/usr/local/musl/$target/" # as searched for by the remoteprocess create. rustup target add "$target" -apk add --no-cache musl-dev make git curl # git & curl used by next scripts +apk add --no-cache musl-dev make git curl wget # git & curl used by next scripts mkdir builds && cd builds From 95019ca490c0e25bd5f00ab6f64b16cf85176265 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 17 Aug 2023 09:44:38 +0200 Subject: [PATCH 080/141] lets extract resources --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- .github/workflows/build-test-deploy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 4718b11f2..0a7842fdd 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -75,7 +75,7 @@ jobs: run: docker image load < output/gprofiler_aarch64.img - name: Extract resources from gProfiler executable - run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=./gprofiler/resources + run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 4f86c3413..39caf843b 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -250,7 +250,7 @@ jobs: run: chmod +x ./output/gprofiler - name: Extract resources from gProfiler executable - run: sudo ./output/gprofiler extract-resources --resources-dest=/app/gprofiler/resources + run: sudo ./output/gprofiler extract-resources --resources-dest=./gprofiler/resources # used in the tests - name: Import gProfiler image From 3f1f2a3ba09b0b0074525b5023c4706e5ef9828e Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 17 Aug 2023 10:08:46 +0200 Subject: [PATCH 081/141] lets extract resources --- scripts/prepare_machine-unknown-linux-musl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_machine-unknown-linux-musl.sh b/scripts/prepare_machine-unknown-linux-musl.sh index e109521b9..6699f96ed 100755 --- a/scripts/prepare_machine-unknown-linux-musl.sh +++ b/scripts/prepare_machine-unknown-linux-musl.sh @@ -19,7 +19,7 @@ target="$(uname -m)"-unknown-linux-musl target_dir="/usr/local/musl/$target/" # as searched for by the remoteprocess create. rustup target add "$target" -apk add --no-cache musl-dev make git curl wget # git & curl used by next scripts +apk add --no-cache musl-dev make git curl # git & curl used by next scripts mkdir builds && cd builds From e3bfa846e2fa42503d3c6ea2112270b4ee0b4150 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 17 Aug 2023 10:09:25 +0200 Subject: [PATCH 082/141] lets extract resources --- scripts/prepare_machine-unknown-linux-musl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_machine-unknown-linux-musl.sh b/scripts/prepare_machine-unknown-linux-musl.sh index 6699f96ed..29d7aefd4 100755 --- a/scripts/prepare_machine-unknown-linux-musl.sh +++ b/scripts/prepare_machine-unknown-linux-musl.sh @@ -19,7 +19,7 @@ target="$(uname -m)"-unknown-linux-musl target_dir="/usr/local/musl/$target/" # as searched for by the remoteprocess create. rustup target add "$target" -apk add --no-cache musl-dev make git curl # git & curl used by next scripts +apk add --no-cache musl-dev make git curl # git & curl used by next scripts mkdir builds && cd builds From 8546a66f8063adae008e30ea552fe16346448871 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 17 Aug 2023 11:10:35 +0200 Subject: [PATCH 083/141] lets extract resources --- .../build-container-and-test-aarch64.yml | 134 ++-- .github/workflows/build-on-windows.yml | 28 +- .github/workflows/build-test-deploy.yml | 722 +++++++++--------- .github/workflows/linters.yml | 98 +-- 4 files changed, 492 insertions(+), 490 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 0a7842fdd..9c2e967e2 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -9,73 +9,75 @@ jobs: - public - ARM64 - steps: - - name: change pycache permissions - run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler - - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs - # more nicely. the tests will then be able to use the built image. - - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 - - - name: Export gProfiler image - run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img - - - name: Upload the image artifact - uses: actions/upload-artifact@v2 - with: - name: gprofiler_aarch64.img - path: output/ - retention-days: 1 - - test: - needs: build-container-aarch64 - - runs-on: - - self-hosted - - public - - ARM64 - steps: + steps: +# - name: change pycache permissions +# run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler + +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# submodules: true + +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v2 + +# - name: Login to DockerHub +# uses: docker/login-action@v1 +# with: +# username: ${{ secrets.DOCKERHUB_USERNAME }} +# password: ${{ secrets.DOCKERHUB_TOKEN }} + +# # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs +# # more nicely. the tests will then be able to use the built image. +# - name: Build gProfiler image +# run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 + +# - name: Export gProfiler image +# run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img + +# - name: Upload the image artifact +# uses: actions/upload-artifact@v2 +# with: +# name: gprofiler_aarch64.img +# path: output/ +# retention-days: 1 + +# test: +# needs: build-container-aarch64 + +# runs-on: +# - self-hosted +# - public +# - ARM64 +# steps: - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: change pycache permissions - run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler - - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Download the executable from previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler_aarch64.img - path: output/ - - - name: Import gProfiler image - run: docker image load < output/gprofiler_aarch64.img - - - name: Extract resources from gProfiler executable - run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources +# - name: Login to DockerHub +# uses: docker/login-action@v1 +# with: +# username: ${{ secrets.DOCKERHUB_USERNAME }} +# password: ${{ secrets.DOCKERHUB_TOKEN }} + +# - name: change pycache permissions +# run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler + +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# submodules: true + +# - name: Download the executable from previous job +# uses: actions/download-artifact@v2 +# with: +# name: gprofiler_aarch64.img +# path: output/ + +# - name: Import gProfiler image +# run: docker image load < output/gprofiler_aarch64.img + +# - name: Extract resources from gProfiler executable +# run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources + - name: Show site-packages + run: python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])' # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests diff --git a/.github/workflows/build-on-windows.yml b/.github/workflows/build-on-windows.yml index 847befc75..5544fe6c1 100644 --- a/.github/workflows/build-on-windows.yml +++ b/.github/workflows/build-on-windows.yml @@ -1,18 +1,18 @@ -name: Build gProfiler on Windows +# name: Build gProfiler on Windows -on: pull_request +# on: pull_request -jobs: - build-executable: - runs-on: windows-latest - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true +# jobs: +# build-executable: +# runs-on: windows-latest +# steps: +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# submodules: true - - name: run the build script - run: .\scripts\windows\build.bat +# - name: run the build script +# run: .\scripts\windows\build.bat - - name: check gprofiler build - run: .\app\dist\gprofiler.exe -h +# - name: check gprofiler build +# run: .\app\dist\gprofiler.exe -h diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 39caf843b..04ecbb688 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -1,361 +1,361 @@ -# build-executable-{arch} job builds the gprofiler executable by running build_{arch}_executable.sh scripts, then uploads it as a job artifact -# test-executable-{arch} job downloads gprofiler exe artifact and then runs test_executable.py tests from tests dir -# deploy-executable job runs only on tagged commits and it deploys gprofiler executables as release assets -# build-container-{arch} job downloads gprofiler exe artifact, produces gprofiler docker image from it, exports it into the file, and uploads image as a job artifact -# test-container-{arch} job downloads gprofiler image artifact and runs tests/test.sh script -# deploy-container job runs only on tagged commits and it deploys gprofiler images to dockerhub - -name: Build, test and deploy - -on: - pull_request: - push: - tags: - - '**' - -jobs: - build-executable-x64: - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - - name: Get and verify tag value - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - ./scripts/cmp_tags.sh - - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - - name: Build gProfiler executable - # Using DOCKER_BUILDKIT=1 although it has another cache mechanism which is not supported by satackey/action-docker-layer-caching@v0.0.11 - # We tried to cache using buildx cache (cache-from, cache-to flags) and got strange behavior when caching, - # decided not to use buildkit here to get nice caches. - # if we enable caching again, we can disable DOCKER_BUILDKIT. - run: | - mkdir -p output - ./scripts/build_x86_64_executable.sh - mv build/x86_64/gprofiler output/gprofiler_x86_64 - cp output/gprofiler_x86_64 output/gprofiler # for backwards compatibility - - - name: Upload the executables as job artifacts - uses: actions/upload-artifact@v2 - with: - name: gprofiler - path: output/ - retention-days: 1 - - test-executable-x64: - if: ${{ !startsWith(github.ref, 'refs/tags/') }} - runs-on: ubuntu-20.04 # the tests which use ruby/node/python etc and run at non-root privs, fail to read the files when running - # on ubuntu-22.04/ubuntu-latest: - # stderr: ruby: Permission denied -- /home/runner/work/gprofiler/gprofiler/tests/containers/ruby/fibonacci.rb (LoadError) - needs: build-executable-x64 - strategy: - fail-fast: false - matrix: - containers: - - alpine - - ubuntu:14.04 - - ubuntu:16.04 - - ubuntu:18.04 - - ubuntu:20.04 - - ubuntu:20.10 - - ubuntu:22.04 - - centos:6 - - centos:7 - - centos:8 - - debian:8 - - debian:9 - - debian:10 - - debian:11 - steps: - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - - name: Install Java - uses: actions/setup-java@v1 - with: - java-version: '8.0.275' - java-package: jdk - architecture: x64 - - - name: Install Node.JS - uses: actions/setup-node@v2 - with: - # same version as used in tests/containers/nodejs/Dockerfile - node-version: 10.x - - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Download the executable from previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler - path: dist/ - - - name: Run gProfiler tests - run: | - mv dist/gprofiler_x86_64 dist/gprofiler - chmod +x dist/gprofiler - NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable - - build-executable-aarch64: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ["self-hosted", "ARM64"] - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - - name: Set up requirements - run: scripts/setup_runner_requirements.sh - - - name: Get and verify tag value - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - ./scripts/cmp_tags.sh - - - name: Build gProfiler executable - run: | - mkdir -p output - ./scripts/build_aarch64_executable.sh - mv build/aarch64/gprofiler output/gprofiler_aarch64 - - - name: Upload the executables as job artifacts - uses: actions/upload-artifact@v2 - with: - name: gprofiler - path: output/ - retention-days: 1 - - deploy-executable: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-20.04 - needs: - - build-executable-x64 - - build-executable-aarch64 - - steps: - - name: Download executables from the previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler - path: output/ - - name: Get tag value - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Release - uses: softprops/action-gh-release@v1 - with: - files: | - output/gprofiler_x86_64 - output/gprofiler_aarch64 - output/gprofiler - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - build-container-x64: - runs-on: ubuntu-latest - needs: - - build-executable-x64 - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - - - name: Download executables from the previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler - path: output/ - - # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs - # more nicely. the tests will then be able to use the built image. - - name: Build gProfiler image - run: ./scripts/build_x86_64_container.sh --skip-exe-build --build-arg EXE_PATH=output/gprofiler_x86_64 -t gprofiler_x86_64 - - - name: Export gProfiler image - run: mkdir -p output && docker image save gprofiler_x86_64 > output/gprofiler_x86_64.img - - - name: Upload the image artifact - uses: actions/upload-artifact@v2 - with: - name: gprofiler_x86_64.img - path: output/ - retention-days: 1 - - test-container-x64: - if: ${{ !startsWith(github.ref, 'refs/tags/') }} - needs: build-container-x64 - - runs-on: ubuntu-20.04 # for 3.10.0 python tag - - strategy: - fail-fast: false # helps detecting flakiness / errors specific to one Python version - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10.0" # see https://github.com/Granulate/gprofiler/issues/502, we need 3.10.0, others fail PyPerf tests. - - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Java - uses: actions/setup-java@v1 - with: - java-version: '8.0.275' - java-package: jdk - architecture: x64 - - - name: Install Node.JS - uses: actions/setup-node@v2 - with: - # same version as used in tests/containers/nodejs/Dockerfile - node-version: 10.x - - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Download the executable from previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler - path: output/ - - - name: Download the image from previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler_x86_64.img - path: output/ - - - name: Add +x to gprofiler - run: chmod +x ./output/gprofiler - - - name: Extract resources from gProfiler executable - run: sudo ./output/gprofiler extract-resources --resources-dest=./gprofiler/resources - - # used in the tests - - name: Import gProfiler image - run: docker image load < output/gprofiler_x86_64.img - - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - - name: Run gProfiler tests - run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py - - build-container-aarch64: - if: startsWith(github.ref, 'refs/tags/') - needs: build-executable-aarch64 - runs-on: ["self-hosted", "ARM64"] - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - - name: Set up runner requirements - run: scripts/setup_runner_requirements.sh - - - name: Get and verify tag value - run: ./scripts/verify_tag.sh - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Download executables from the previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler - path: output/ - - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - - name: Build and push - run: | - set -x - export DOCKER_BUILDKIT=1 - - BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" - AARCH64_IMAGE="$BASE_IMAGE-aarch64" - - # build & push Aarch64 - ./scripts/build_aarch64_container.sh --skip-exe-build --build-arg EXE_PATH=output/gprofiler_aarch64 -t "$AARCH64_IMAGE" --push - - deploy-container: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-latest - needs: - - build-container-x64 - - build-container-aarch64 - steps: - # needed to set up env.GH_REPO etc - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - # build-container-x64 has uploaded the image as an artifact, we donwload it here. - # build-container-aarch64 has pushed the image to DockerHub, so we'll pull it later when creating - # the manifest. - - name: Download the x86_64 image from previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler_x86_64.img - path: output/ - - - name: Get and verify tag value - run: ./scripts/verify_tag.sh - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Push manifest - run: | - set -x - export DOCKER_BUILDKIT=1 - - BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" - LATEST_IMAGE="${{ env.GH_REPO }}:latest" - AARCH64_IMAGE="$BASE_IMAGE-aarch64" - X86_64_IMAGE="$BASE_IMAGE-x86_64" - - docker pull $AARCH64_IMAGE - docker image load < output/gprofiler_x86_64.img - docker tag gprofiler_x86_64 $X86_64_IMAGE - docker push $X86_64_IMAGE - - # create manifests for the tag + for 'latest' - docker manifest create "$BASE_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" - docker manifest push "$BASE_IMAGE" - docker manifest create "$LATEST_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" - docker manifest push "$LATEST_IMAGE" +# # build-executable-{arch} job builds the gprofiler executable by running build_{arch}_executable.sh scripts, then uploads it as a job artifact +# # test-executable-{arch} job downloads gprofiler exe artifact and then runs test_executable.py tests from tests dir +# # deploy-executable job runs only on tagged commits and it deploys gprofiler executables as release assets +# # build-container-{arch} job downloads gprofiler exe artifact, produces gprofiler docker image from it, exports it into the file, and uploads image as a job artifact +# # test-container-{arch} job downloads gprofiler image artifact and runs tests/test.sh script +# # deploy-container job runs only on tagged commits and it deploys gprofiler images to dockerhub + +# name: Build, test and deploy + +# on: +# pull_request: +# push: +# tags: +# - '**' + +# jobs: +# build-executable-x64: +# runs-on: ubuntu-latest +# steps: +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# submodules: true + +# - name: Get and verify tag value +# run: | +# echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV +# ./scripts/cmp_tags.sh + +# # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" +# - name: Build gProfiler executable +# # Using DOCKER_BUILDKIT=1 although it has another cache mechanism which is not supported by satackey/action-docker-layer-caching@v0.0.11 +# # We tried to cache using buildx cache (cache-from, cache-to flags) and got strange behavior when caching, +# # decided not to use buildkit here to get nice caches. +# # if we enable caching again, we can disable DOCKER_BUILDKIT. +# run: | +# mkdir -p output +# ./scripts/build_x86_64_executable.sh +# mv build/x86_64/gprofiler output/gprofiler_x86_64 +# cp output/gprofiler_x86_64 output/gprofiler # for backwards compatibility + +# - name: Upload the executables as job artifacts +# uses: actions/upload-artifact@v2 +# with: +# name: gprofiler +# path: output/ +# retention-days: 1 + +# test-executable-x64: +# if: ${{ !startsWith(github.ref, 'refs/tags/') }} +# runs-on: ubuntu-20.04 # the tests which use ruby/node/python etc and run at non-root privs, fail to read the files when running +# # on ubuntu-22.04/ubuntu-latest: +# # stderr: ruby: Permission denied -- /home/runner/work/gprofiler/gprofiler/tests/containers/ruby/fibonacci.rb (LoadError) +# needs: build-executable-x64 +# strategy: +# fail-fast: false +# matrix: +# containers: +# - alpine +# - ubuntu:14.04 +# - ubuntu:16.04 +# - ubuntu:18.04 +# - ubuntu:20.04 +# - ubuntu:20.10 +# - ubuntu:22.04 +# - centos:6 +# - centos:7 +# - centos:8 +# - debian:8 +# - debian:9 +# - debian:10 +# - debian:11 +# steps: +# - name: Set up Python 3.8 +# uses: actions/setup-python@v2 +# with: +# python-version: "3.8" + +# - name: Install Java +# uses: actions/setup-java@v1 +# with: +# java-version: '8.0.275' +# java-package: jdk +# architecture: x64 + +# - name: Install Node.JS +# uses: actions/setup-node@v2 +# with: +# # same version as used in tests/containers/nodejs/Dockerfile +# node-version: 10.x + +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# submodules: true + +# - name: Download the executable from previous job +# uses: actions/download-artifact@v2 +# with: +# name: gprofiler +# path: dist/ + +# - name: Run gProfiler tests +# run: | +# mv dist/gprofiler_x86_64 dist/gprofiler +# chmod +x dist/gprofiler +# NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable + +# build-executable-aarch64: +# if: startsWith(github.ref, 'refs/tags/') +# runs-on: ["self-hosted", "ARM64"] +# steps: +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# submodules: true + +# - name: Set up requirements +# run: scripts/setup_runner_requirements.sh + +# - name: Get and verify tag value +# run: | +# echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV +# ./scripts/cmp_tags.sh + +# - name: Build gProfiler executable +# run: | +# mkdir -p output +# ./scripts/build_aarch64_executable.sh +# mv build/aarch64/gprofiler output/gprofiler_aarch64 + +# - name: Upload the executables as job artifacts +# uses: actions/upload-artifact@v2 +# with: +# name: gprofiler +# path: output/ +# retention-days: 1 + +# deploy-executable: +# if: startsWith(github.ref, 'refs/tags/') +# runs-on: ubuntu-20.04 +# needs: +# - build-executable-x64 +# - build-executable-aarch64 + +# steps: +# - name: Download executables from the previous job +# uses: actions/download-artifact@v2 +# with: +# name: gprofiler +# path: output/ +# - name: Get tag value +# run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV +# - name: Release +# uses: softprops/action-gh-release@v1 +# with: +# files: | +# output/gprofiler_x86_64 +# output/gprofiler_aarch64 +# output/gprofiler +# env: +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +# build-container-x64: +# runs-on: ubuntu-latest +# needs: +# - build-executable-x64 + +# steps: +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# submodules: true +# fetch-depth: 0 + +# - name: Download executables from the previous job +# uses: actions/download-artifact@v2 +# with: +# name: gprofiler +# path: output/ + +# # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs +# # more nicely. the tests will then be able to use the built image. +# - name: Build gProfiler image +# run: ./scripts/build_x86_64_container.sh --skip-exe-build --build-arg EXE_PATH=output/gprofiler_x86_64 -t gprofiler_x86_64 + +# - name: Export gProfiler image +# run: mkdir -p output && docker image save gprofiler_x86_64 > output/gprofiler_x86_64.img + +# - name: Upload the image artifact +# uses: actions/upload-artifact@v2 +# with: +# name: gprofiler_x86_64.img +# path: output/ +# retention-days: 1 + +# test-container-x64: +# if: ${{ !startsWith(github.ref, 'refs/tags/') }} +# needs: build-container-x64 + +# runs-on: ubuntu-20.04 # for 3.10.0 python tag + +# strategy: +# fail-fast: false # helps detecting flakiness / errors specific to one Python version +# matrix: +# python-version: +# - "3.8" +# - "3.9" +# - "3.10.0" # see https://github.com/Granulate/gprofiler/issues/502, we need 3.10.0, others fail PyPerf tests. + +# steps: +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} + +# - name: Install Java +# uses: actions/setup-java@v1 +# with: +# java-version: '8.0.275' +# java-package: jdk +# architecture: x64 + +# - name: Install Node.JS +# uses: actions/setup-node@v2 +# with: +# # same version as used in tests/containers/nodejs/Dockerfile +# node-version: 10.x + +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# submodules: true + +# - name: Download the executable from previous job +# uses: actions/download-artifact@v2 +# with: +# name: gprofiler +# path: output/ + +# - name: Download the image from previous job +# uses: actions/download-artifact@v2 +# with: +# name: gprofiler_x86_64.img +# path: output/ + +# - name: Add +x to gprofiler +# run: chmod +x ./output/gprofiler + +# - name: Extract resources from gProfiler executable +# run: sudo ./output/gprofiler extract-resources --resources-dest=./gprofiler/resources + +# # used in the tests +# - name: Import gProfiler image +# run: docker image load < output/gprofiler_x86_64.img + +# # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" +# - name: Run gProfiler tests +# run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py + +# build-container-aarch64: +# if: startsWith(github.ref, 'refs/tags/') +# needs: build-executable-aarch64 +# runs-on: ["self-hosted", "ARM64"] +# steps: +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# submodules: true + +# - name: Set up runner requirements +# run: scripts/setup_runner_requirements.sh + +# - name: Get and verify tag value +# run: ./scripts/verify_tag.sh + +# - name: Login to DockerHub +# uses: docker/login-action@v1 +# with: +# username: ${{ secrets.DOCKERHUB_USERNAME }} +# password: ${{ secrets.DOCKERHUB_TOKEN }} + +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v1 + +# - name: Download executables from the previous job +# uses: actions/download-artifact@v2 +# with: +# name: gprofiler +# path: output/ + +# # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" +# - name: Build and push +# run: | +# set -x +# export DOCKER_BUILDKIT=1 + +# BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" +# AARCH64_IMAGE="$BASE_IMAGE-aarch64" + +# # build & push Aarch64 +# ./scripts/build_aarch64_container.sh --skip-exe-build --build-arg EXE_PATH=output/gprofiler_aarch64 -t "$AARCH64_IMAGE" --push + +# deploy-container: +# if: startsWith(github.ref, 'refs/tags/') +# runs-on: ubuntu-latest +# needs: +# - build-container-x64 +# - build-container-aarch64 +# steps: +# # needed to set up env.GH_REPO etc +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# submodules: true + +# # build-container-x64 has uploaded the image as an artifact, we donwload it here. +# # build-container-aarch64 has pushed the image to DockerHub, so we'll pull it later when creating +# # the manifest. +# - name: Download the x86_64 image from previous job +# uses: actions/download-artifact@v2 +# with: +# name: gprofiler_x86_64.img +# path: output/ + +# - name: Get and verify tag value +# run: ./scripts/verify_tag.sh + +# - name: Login to DockerHub +# uses: docker/login-action@v1 +# with: +# username: ${{ secrets.DOCKERHUB_USERNAME }} +# password: ${{ secrets.DOCKERHUB_TOKEN }} + +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v1 + +# - name: Push manifest +# run: | +# set -x +# export DOCKER_BUILDKIT=1 + +# BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" +# LATEST_IMAGE="${{ env.GH_REPO }}:latest" +# AARCH64_IMAGE="$BASE_IMAGE-aarch64" +# X86_64_IMAGE="$BASE_IMAGE-x86_64" + +# docker pull $AARCH64_IMAGE +# docker image load < output/gprofiler_x86_64.img +# docker tag gprofiler_x86_64 $X86_64_IMAGE +# docker push $X86_64_IMAGE + +# # create manifests for the tag + for 'latest' +# docker manifest create "$BASE_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" +# docker manifest push "$BASE_IMAGE" +# docker manifest create "$LATEST_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" +# docker manifest push "$LATEST_IMAGE" diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index dd4f9f33f..a65521c2a 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -1,49 +1,49 @@ -name: Linters - -on: pull_request - -jobs: - non-python-lint: - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Run shell linter - run: ./shell_lint.sh - - - name: Run Dockerfile linters - run: ./dockerfile_lint.sh - - python-lint: - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - runs-on: ubuntu-latest - - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r dev-requirements.txt - pip install -r requirements.txt - - - name: Run linters - run: ./lint.sh --ci +# name: Linters + +# on: pull_request + +# jobs: +# non-python-lint: +# runs-on: ubuntu-latest + +# steps: +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# submodules: true + +# - name: Run shell linter +# run: ./shell_lint.sh + +# - name: Run Dockerfile linters +# run: ./dockerfile_lint.sh + +# python-lint: +# strategy: +# matrix: +# python-version: +# - "3.8" +# - "3.9" +# - "3.10" + +# runs-on: ubuntu-latest + +# steps: +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ matrix.python-version }} + +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# submodules: true + +# - name: Install dependencies +# run: | +# python -m pip install --upgrade pip +# pip install -r dev-requirements.txt +# pip install -r requirements.txt + +# - name: Run linters +# run: ./lint.sh --ci From 217a4282be2cde7fc30ce7988213174d6d73ff3a Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 17 Aug 2023 11:11:26 +0200 Subject: [PATCH 084/141] lets extract resources --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 9c2e967e2..145baf59e 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -9,7 +9,7 @@ jobs: - public - ARM64 - steps: + steps: # - name: change pycache permissions # run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler From e6894c8bd04d86e06bcce995eedc633e34fc16d2 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 17 Aug 2023 11:28:25 +0200 Subject: [PATCH 085/141] lets extract resources --- .../build-container-and-test-aarch64.yml | 134 +++++++++--------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 145baf59e..50be79dcb 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -10,74 +10,74 @@ jobs: - ARM64 steps: -# - name: change pycache permissions -# run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler - -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# submodules: true - -# - name: Set up Docker Buildx -# uses: docker/setup-buildx-action@v2 - -# - name: Login to DockerHub -# uses: docker/login-action@v1 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} - -# # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs -# # more nicely. the tests will then be able to use the built image. -# - name: Build gProfiler image -# run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 - -# - name: Export gProfiler image -# run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img - -# - name: Upload the image artifact -# uses: actions/upload-artifact@v2 -# with: -# name: gprofiler_aarch64.img -# path: output/ -# retention-days: 1 - -# test: -# needs: build-container-aarch64 - -# runs-on: -# - self-hosted -# - public -# - ARM64 -# steps: + - name: change pycache permissions + run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler + + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # more nicely. the tests will then be able to use the built image. + - name: Build gProfiler image + run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 + + - name: Export gProfiler image + run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img + + - name: Upload the image artifact + uses: actions/upload-artifact@v2 + with: + name: gprofiler_aarch64.img + path: output/ + retention-days: 1 + + test: + needs: build-container-aarch64 + + runs-on: + - self-hosted + - public + - ARM64 + steps: -# - name: Login to DockerHub -# uses: docker/login-action@v1 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} - -# - name: change pycache permissions -# run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler - -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# submodules: true - -# - name: Download the executable from previous job -# uses: actions/download-artifact@v2 -# with: -# name: gprofiler_aarch64.img -# path: output/ - -# - name: Import gProfiler image -# run: docker image load < output/gprofiler_aarch64.img - -# - name: Extract resources from gProfiler executable -# run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - - name: Show site-packages - run: python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])' + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: change pycache permissions + run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler + + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Download the executable from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler_aarch64.img + path: output/ + + - name: Import gProfiler image + run: docker image load < output/gprofiler_aarch64.img + + - name: Extract resources from gProfiler executable + run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources + - name: Remove problematic PyYAML from site-packages + run: find $(python3 -m site --user-site) -type d -name '*PyYAML*' # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 077788f7b41de61f316887e00dab850f68be5263 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 17 Aug 2023 12:02:14 +0200 Subject: [PATCH 086/141] lets extract resources --- .github/workflows/build-on-windows.yml | 28 +- .github/workflows/build-test-deploy.yml | 722 ++++++++++++------------ .github/workflows/linters.yml | 98 ++-- 3 files changed, 424 insertions(+), 424 deletions(-) diff --git a/.github/workflows/build-on-windows.yml b/.github/workflows/build-on-windows.yml index 5544fe6c1..847befc75 100644 --- a/.github/workflows/build-on-windows.yml +++ b/.github/workflows/build-on-windows.yml @@ -1,18 +1,18 @@ -# name: Build gProfiler on Windows +name: Build gProfiler on Windows -# on: pull_request +on: pull_request -# jobs: -# build-executable: -# runs-on: windows-latest -# steps: -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# submodules: true +jobs: + build-executable: + runs-on: windows-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true -# - name: run the build script -# run: .\scripts\windows\build.bat + - name: run the build script + run: .\scripts\windows\build.bat -# - name: check gprofiler build -# run: .\app\dist\gprofiler.exe -h + - name: check gprofiler build + run: .\app\dist\gprofiler.exe -h diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 04ecbb688..39caf843b 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -1,361 +1,361 @@ -# # build-executable-{arch} job builds the gprofiler executable by running build_{arch}_executable.sh scripts, then uploads it as a job artifact -# # test-executable-{arch} job downloads gprofiler exe artifact and then runs test_executable.py tests from tests dir -# # deploy-executable job runs only on tagged commits and it deploys gprofiler executables as release assets -# # build-container-{arch} job downloads gprofiler exe artifact, produces gprofiler docker image from it, exports it into the file, and uploads image as a job artifact -# # test-container-{arch} job downloads gprofiler image artifact and runs tests/test.sh script -# # deploy-container job runs only on tagged commits and it deploys gprofiler images to dockerhub - -# name: Build, test and deploy - -# on: -# pull_request: -# push: -# tags: -# - '**' - -# jobs: -# build-executable-x64: -# runs-on: ubuntu-latest -# steps: -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# fetch-depth: 0 -# submodules: true - -# - name: Get and verify tag value -# run: | -# echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV -# ./scripts/cmp_tags.sh - -# # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" -# - name: Build gProfiler executable -# # Using DOCKER_BUILDKIT=1 although it has another cache mechanism which is not supported by satackey/action-docker-layer-caching@v0.0.11 -# # We tried to cache using buildx cache (cache-from, cache-to flags) and got strange behavior when caching, -# # decided not to use buildkit here to get nice caches. -# # if we enable caching again, we can disable DOCKER_BUILDKIT. -# run: | -# mkdir -p output -# ./scripts/build_x86_64_executable.sh -# mv build/x86_64/gprofiler output/gprofiler_x86_64 -# cp output/gprofiler_x86_64 output/gprofiler # for backwards compatibility - -# - name: Upload the executables as job artifacts -# uses: actions/upload-artifact@v2 -# with: -# name: gprofiler -# path: output/ -# retention-days: 1 - -# test-executable-x64: -# if: ${{ !startsWith(github.ref, 'refs/tags/') }} -# runs-on: ubuntu-20.04 # the tests which use ruby/node/python etc and run at non-root privs, fail to read the files when running -# # on ubuntu-22.04/ubuntu-latest: -# # stderr: ruby: Permission denied -- /home/runner/work/gprofiler/gprofiler/tests/containers/ruby/fibonacci.rb (LoadError) -# needs: build-executable-x64 -# strategy: -# fail-fast: false -# matrix: -# containers: -# - alpine -# - ubuntu:14.04 -# - ubuntu:16.04 -# - ubuntu:18.04 -# - ubuntu:20.04 -# - ubuntu:20.10 -# - ubuntu:22.04 -# - centos:6 -# - centos:7 -# - centos:8 -# - debian:8 -# - debian:9 -# - debian:10 -# - debian:11 -# steps: -# - name: Set up Python 3.8 -# uses: actions/setup-python@v2 -# with: -# python-version: "3.8" - -# - name: Install Java -# uses: actions/setup-java@v1 -# with: -# java-version: '8.0.275' -# java-package: jdk -# architecture: x64 - -# - name: Install Node.JS -# uses: actions/setup-node@v2 -# with: -# # same version as used in tests/containers/nodejs/Dockerfile -# node-version: 10.x - -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# submodules: true - -# - name: Download the executable from previous job -# uses: actions/download-artifact@v2 -# with: -# name: gprofiler -# path: dist/ - -# - name: Run gProfiler tests -# run: | -# mv dist/gprofiler_x86_64 dist/gprofiler -# chmod +x dist/gprofiler -# NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable - -# build-executable-aarch64: -# if: startsWith(github.ref, 'refs/tags/') -# runs-on: ["self-hosted", "ARM64"] -# steps: -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# fetch-depth: 0 -# submodules: true - -# - name: Set up requirements -# run: scripts/setup_runner_requirements.sh - -# - name: Get and verify tag value -# run: | -# echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV -# ./scripts/cmp_tags.sh - -# - name: Build gProfiler executable -# run: | -# mkdir -p output -# ./scripts/build_aarch64_executable.sh -# mv build/aarch64/gprofiler output/gprofiler_aarch64 - -# - name: Upload the executables as job artifacts -# uses: actions/upload-artifact@v2 -# with: -# name: gprofiler -# path: output/ -# retention-days: 1 - -# deploy-executable: -# if: startsWith(github.ref, 'refs/tags/') -# runs-on: ubuntu-20.04 -# needs: -# - build-executable-x64 -# - build-executable-aarch64 - -# steps: -# - name: Download executables from the previous job -# uses: actions/download-artifact@v2 -# with: -# name: gprofiler -# path: output/ -# - name: Get tag value -# run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV -# - name: Release -# uses: softprops/action-gh-release@v1 -# with: -# files: | -# output/gprofiler_x86_64 -# output/gprofiler_aarch64 -# output/gprofiler -# env: -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - -# build-container-x64: -# runs-on: ubuntu-latest -# needs: -# - build-executable-x64 - -# steps: -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# submodules: true -# fetch-depth: 0 - -# - name: Download executables from the previous job -# uses: actions/download-artifact@v2 -# with: -# name: gprofiler -# path: output/ - -# # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs -# # more nicely. the tests will then be able to use the built image. -# - name: Build gProfiler image -# run: ./scripts/build_x86_64_container.sh --skip-exe-build --build-arg EXE_PATH=output/gprofiler_x86_64 -t gprofiler_x86_64 - -# - name: Export gProfiler image -# run: mkdir -p output && docker image save gprofiler_x86_64 > output/gprofiler_x86_64.img - -# - name: Upload the image artifact -# uses: actions/upload-artifact@v2 -# with: -# name: gprofiler_x86_64.img -# path: output/ -# retention-days: 1 - -# test-container-x64: -# if: ${{ !startsWith(github.ref, 'refs/tags/') }} -# needs: build-container-x64 - -# runs-on: ubuntu-20.04 # for 3.10.0 python tag - -# strategy: -# fail-fast: false # helps detecting flakiness / errors specific to one Python version -# matrix: -# python-version: -# - "3.8" -# - "3.9" -# - "3.10.0" # see https://github.com/Granulate/gprofiler/issues/502, we need 3.10.0, others fail PyPerf tests. - -# steps: -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v2 -# with: -# python-version: ${{ matrix.python-version }} - -# - name: Install Java -# uses: actions/setup-java@v1 -# with: -# java-version: '8.0.275' -# java-package: jdk -# architecture: x64 - -# - name: Install Node.JS -# uses: actions/setup-node@v2 -# with: -# # same version as used in tests/containers/nodejs/Dockerfile -# node-version: 10.x - -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# submodules: true - -# - name: Download the executable from previous job -# uses: actions/download-artifact@v2 -# with: -# name: gprofiler -# path: output/ - -# - name: Download the image from previous job -# uses: actions/download-artifact@v2 -# with: -# name: gprofiler_x86_64.img -# path: output/ - -# - name: Add +x to gprofiler -# run: chmod +x ./output/gprofiler - -# - name: Extract resources from gProfiler executable -# run: sudo ./output/gprofiler extract-resources --resources-dest=./gprofiler/resources - -# # used in the tests -# - name: Import gProfiler image -# run: docker image load < output/gprofiler_x86_64.img - -# # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" -# - name: Run gProfiler tests -# run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py - -# build-container-aarch64: -# if: startsWith(github.ref, 'refs/tags/') -# needs: build-executable-aarch64 -# runs-on: ["self-hosted", "ARM64"] -# steps: -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# fetch-depth: 0 -# submodules: true - -# - name: Set up runner requirements -# run: scripts/setup_runner_requirements.sh - -# - name: Get and verify tag value -# run: ./scripts/verify_tag.sh - -# - name: Login to DockerHub -# uses: docker/login-action@v1 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} - -# - name: Set up Docker Buildx -# uses: docker/setup-buildx-action@v1 - -# - name: Download executables from the previous job -# uses: actions/download-artifact@v2 -# with: -# name: gprofiler -# path: output/ - -# # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" -# - name: Build and push -# run: | -# set -x -# export DOCKER_BUILDKIT=1 - -# BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" -# AARCH64_IMAGE="$BASE_IMAGE-aarch64" - -# # build & push Aarch64 -# ./scripts/build_aarch64_container.sh --skip-exe-build --build-arg EXE_PATH=output/gprofiler_aarch64 -t "$AARCH64_IMAGE" --push - -# deploy-container: -# if: startsWith(github.ref, 'refs/tags/') -# runs-on: ubuntu-latest -# needs: -# - build-container-x64 -# - build-container-aarch64 -# steps: -# # needed to set up env.GH_REPO etc -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# fetch-depth: 0 -# submodules: true - -# # build-container-x64 has uploaded the image as an artifact, we donwload it here. -# # build-container-aarch64 has pushed the image to DockerHub, so we'll pull it later when creating -# # the manifest. -# - name: Download the x86_64 image from previous job -# uses: actions/download-artifact@v2 -# with: -# name: gprofiler_x86_64.img -# path: output/ - -# - name: Get and verify tag value -# run: ./scripts/verify_tag.sh - -# - name: Login to DockerHub -# uses: docker/login-action@v1 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} - -# - name: Set up Docker Buildx -# uses: docker/setup-buildx-action@v1 - -# - name: Push manifest -# run: | -# set -x -# export DOCKER_BUILDKIT=1 - -# BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" -# LATEST_IMAGE="${{ env.GH_REPO }}:latest" -# AARCH64_IMAGE="$BASE_IMAGE-aarch64" -# X86_64_IMAGE="$BASE_IMAGE-x86_64" - -# docker pull $AARCH64_IMAGE -# docker image load < output/gprofiler_x86_64.img -# docker tag gprofiler_x86_64 $X86_64_IMAGE -# docker push $X86_64_IMAGE - -# # create manifests for the tag + for 'latest' -# docker manifest create "$BASE_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" -# docker manifest push "$BASE_IMAGE" -# docker manifest create "$LATEST_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" -# docker manifest push "$LATEST_IMAGE" +# build-executable-{arch} job builds the gprofiler executable by running build_{arch}_executable.sh scripts, then uploads it as a job artifact +# test-executable-{arch} job downloads gprofiler exe artifact and then runs test_executable.py tests from tests dir +# deploy-executable job runs only on tagged commits and it deploys gprofiler executables as release assets +# build-container-{arch} job downloads gprofiler exe artifact, produces gprofiler docker image from it, exports it into the file, and uploads image as a job artifact +# test-container-{arch} job downloads gprofiler image artifact and runs tests/test.sh script +# deploy-container job runs only on tagged commits and it deploys gprofiler images to dockerhub + +name: Build, test and deploy + +on: + pull_request: + push: + tags: + - '**' + +jobs: + build-executable-x64: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + - name: Get and verify tag value + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + ./scripts/cmp_tags.sh + + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" + - name: Build gProfiler executable + # Using DOCKER_BUILDKIT=1 although it has another cache mechanism which is not supported by satackey/action-docker-layer-caching@v0.0.11 + # We tried to cache using buildx cache (cache-from, cache-to flags) and got strange behavior when caching, + # decided not to use buildkit here to get nice caches. + # if we enable caching again, we can disable DOCKER_BUILDKIT. + run: | + mkdir -p output + ./scripts/build_x86_64_executable.sh + mv build/x86_64/gprofiler output/gprofiler_x86_64 + cp output/gprofiler_x86_64 output/gprofiler # for backwards compatibility + + - name: Upload the executables as job artifacts + uses: actions/upload-artifact@v2 + with: + name: gprofiler + path: output/ + retention-days: 1 + + test-executable-x64: + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + runs-on: ubuntu-20.04 # the tests which use ruby/node/python etc and run at non-root privs, fail to read the files when running + # on ubuntu-22.04/ubuntu-latest: + # stderr: ruby: Permission denied -- /home/runner/work/gprofiler/gprofiler/tests/containers/ruby/fibonacci.rb (LoadError) + needs: build-executable-x64 + strategy: + fail-fast: false + matrix: + containers: + - alpine + - ubuntu:14.04 + - ubuntu:16.04 + - ubuntu:18.04 + - ubuntu:20.04 + - ubuntu:20.10 + - ubuntu:22.04 + - centos:6 + - centos:7 + - centos:8 + - debian:8 + - debian:9 + - debian:10 + - debian:11 + steps: + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: '8.0.275' + java-package: jdk + architecture: x64 + + - name: Install Node.JS + uses: actions/setup-node@v2 + with: + # same version as used in tests/containers/nodejs/Dockerfile + node-version: 10.x + + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Download the executable from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler + path: dist/ + + - name: Run gProfiler tests + run: | + mv dist/gprofiler_x86_64 dist/gprofiler + chmod +x dist/gprofiler + NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable + + build-executable-aarch64: + if: startsWith(github.ref, 'refs/tags/') + runs-on: ["self-hosted", "ARM64"] + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + - name: Set up requirements + run: scripts/setup_runner_requirements.sh + + - name: Get and verify tag value + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + ./scripts/cmp_tags.sh + + - name: Build gProfiler executable + run: | + mkdir -p output + ./scripts/build_aarch64_executable.sh + mv build/aarch64/gprofiler output/gprofiler_aarch64 + + - name: Upload the executables as job artifacts + uses: actions/upload-artifact@v2 + with: + name: gprofiler + path: output/ + retention-days: 1 + + deploy-executable: + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-20.04 + needs: + - build-executable-x64 + - build-executable-aarch64 + + steps: + - name: Download executables from the previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler + path: output/ + - name: Get tag value + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Release + uses: softprops/action-gh-release@v1 + with: + files: | + output/gprofiler_x86_64 + output/gprofiler_aarch64 + output/gprofiler + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + build-container-x64: + runs-on: ubuntu-latest + needs: + - build-executable-x64 + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Download executables from the previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler + path: output/ + + # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # more nicely. the tests will then be able to use the built image. + - name: Build gProfiler image + run: ./scripts/build_x86_64_container.sh --skip-exe-build --build-arg EXE_PATH=output/gprofiler_x86_64 -t gprofiler_x86_64 + + - name: Export gProfiler image + run: mkdir -p output && docker image save gprofiler_x86_64 > output/gprofiler_x86_64.img + + - name: Upload the image artifact + uses: actions/upload-artifact@v2 + with: + name: gprofiler_x86_64.img + path: output/ + retention-days: 1 + + test-container-x64: + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + needs: build-container-x64 + + runs-on: ubuntu-20.04 # for 3.10.0 python tag + + strategy: + fail-fast: false # helps detecting flakiness / errors specific to one Python version + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10.0" # see https://github.com/Granulate/gprofiler/issues/502, we need 3.10.0, others fail PyPerf tests. + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: '8.0.275' + java-package: jdk + architecture: x64 + + - name: Install Node.JS + uses: actions/setup-node@v2 + with: + # same version as used in tests/containers/nodejs/Dockerfile + node-version: 10.x + + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Download the executable from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler + path: output/ + + - name: Download the image from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler_x86_64.img + path: output/ + + - name: Add +x to gprofiler + run: chmod +x ./output/gprofiler + + - name: Extract resources from gProfiler executable + run: sudo ./output/gprofiler extract-resources --resources-dest=./gprofiler/resources + + # used in the tests + - name: Import gProfiler image + run: docker image load < output/gprofiler_x86_64.img + + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" + - name: Run gProfiler tests + run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py + + build-container-aarch64: + if: startsWith(github.ref, 'refs/tags/') + needs: build-executable-aarch64 + runs-on: ["self-hosted", "ARM64"] + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + - name: Set up runner requirements + run: scripts/setup_runner_requirements.sh + + - name: Get and verify tag value + run: ./scripts/verify_tag.sh + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Download executables from the previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler + path: output/ + + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" + - name: Build and push + run: | + set -x + export DOCKER_BUILDKIT=1 + + BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" + AARCH64_IMAGE="$BASE_IMAGE-aarch64" + + # build & push Aarch64 + ./scripts/build_aarch64_container.sh --skip-exe-build --build-arg EXE_PATH=output/gprofiler_aarch64 -t "$AARCH64_IMAGE" --push + + deploy-container: + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + needs: + - build-container-x64 + - build-container-aarch64 + steps: + # needed to set up env.GH_REPO etc + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + + # build-container-x64 has uploaded the image as an artifact, we donwload it here. + # build-container-aarch64 has pushed the image to DockerHub, so we'll pull it later when creating + # the manifest. + - name: Download the x86_64 image from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler_x86_64.img + path: output/ + + - name: Get and verify tag value + run: ./scripts/verify_tag.sh + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Push manifest + run: | + set -x + export DOCKER_BUILDKIT=1 + + BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" + LATEST_IMAGE="${{ env.GH_REPO }}:latest" + AARCH64_IMAGE="$BASE_IMAGE-aarch64" + X86_64_IMAGE="$BASE_IMAGE-x86_64" + + docker pull $AARCH64_IMAGE + docker image load < output/gprofiler_x86_64.img + docker tag gprofiler_x86_64 $X86_64_IMAGE + docker push $X86_64_IMAGE + + # create manifests for the tag + for 'latest' + docker manifest create "$BASE_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" + docker manifest push "$BASE_IMAGE" + docker manifest create "$LATEST_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" + docker manifest push "$LATEST_IMAGE" diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index a65521c2a..dd4f9f33f 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -1,49 +1,49 @@ -# name: Linters - -# on: pull_request - -# jobs: -# non-python-lint: -# runs-on: ubuntu-latest - -# steps: -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# submodules: true - -# - name: Run shell linter -# run: ./shell_lint.sh - -# - name: Run Dockerfile linters -# run: ./dockerfile_lint.sh - -# python-lint: -# strategy: -# matrix: -# python-version: -# - "3.8" -# - "3.9" -# - "3.10" - -# runs-on: ubuntu-latest - -# steps: -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v2 -# with: -# python-version: ${{ matrix.python-version }} - -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# submodules: true - -# - name: Install dependencies -# run: | -# python -m pip install --upgrade pip -# pip install -r dev-requirements.txt -# pip install -r requirements.txt - -# - name: Run linters -# run: ./lint.sh --ci +name: Linters + +on: pull_request + +jobs: + non-python-lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Run shell linter + run: ./shell_lint.sh + + - name: Run Dockerfile linters + run: ./dockerfile_lint.sh + + python-lint: + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + + runs-on: ubuntu-latest + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r dev-requirements.txt + pip install -r requirements.txt + + - name: Run linters + run: ./lint.sh --ci From 3a1b3a10c51961031abdf5766a021a206689c60f Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 22 Aug 2023 08:34:16 +0200 Subject: [PATCH 087/141] wget installed manually --- scripts/prepare_machine-unknown-linux-musl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_machine-unknown-linux-musl.sh b/scripts/prepare_machine-unknown-linux-musl.sh index 29d7aefd4..85653d244 100755 --- a/scripts/prepare_machine-unknown-linux-musl.sh +++ b/scripts/prepare_machine-unknown-linux-musl.sh @@ -19,7 +19,7 @@ target="$(uname -m)"-unknown-linux-musl target_dir="/usr/local/musl/$target/" # as searched for by the remoteprocess create. rustup target add "$target" -apk add --no-cache musl-dev make git curl # git & curl used by next scripts +apk add --no-cache musl-dev make git curl wget # git & curl used by next scripts mkdir builds && cd builds From b9b98b2d3f187544d831c3ffed2d529721008a71 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 22 Aug 2023 08:39:51 +0200 Subject: [PATCH 088/141] new zlib version --- scripts/prepare_machine-unknown-linux-musl.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_machine-unknown-linux-musl.sh b/scripts/prepare_machine-unknown-linux-musl.sh index 85653d244..c6e378131 100755 --- a/scripts/prepare_machine-unknown-linux-musl.sh +++ b/scripts/prepare_machine-unknown-linux-musl.sh @@ -19,11 +19,11 @@ target="$(uname -m)"-unknown-linux-musl target_dir="/usr/local/musl/$target/" # as searched for by the remoteprocess create. rustup target add "$target" -apk add --no-cache musl-dev make git curl wget # git & curl used by next scripts +apk add --no-cache musl-dev make git curl # git & curl used by next scripts mkdir builds && cd builds -ZLIB_VERSION=1.2.13 +ZLIB_VERSION=1.3 ZLIB_FILE="zlib-$ZLIB_VERSION.tar.xz" wget "https://zlib.net/$ZLIB_FILE" tar -xf "$ZLIB_FILE" From 4bbd140bbc44ee09f624425c0cdd6954dc1600ba Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 22 Aug 2023 09:45:24 +0200 Subject: [PATCH 089/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 50be79dcb..c1724bed5 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -76,8 +76,9 @@ jobs: - name: Extract resources from gProfiler executable run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources + - name: Remove problematic PyYAML from site-packages - run: find $(python3 -m site --user-site) -type d -name '*PyYAML*' + run: find /usr/lib/python* -type d -name '*PyYAML*' # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From bd135c42752ffee28338161f815f3e31bd961954 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 22 Aug 2023 10:19:28 +0200 Subject: [PATCH 090/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index c1724bed5..6986032eb 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -78,7 +78,7 @@ jobs: run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - name: Remove problematic PyYAML from site-packages - run: find /usr/lib/python* -type d -name '*PyYAML*' + run: find /usr/lib/python* -type d -name '*PyYAML*' -delete # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 5eac8c78dfe620ff96968cafbe7a543d9cee4c7c Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 22 Aug 2023 11:38:14 +0200 Subject: [PATCH 091/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 6986032eb..c5c4c1a4c 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -77,8 +77,8 @@ jobs: - name: Extract resources from gProfiler executable run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - - name: Remove problematic PyYAML from site-packages - run: find /usr/lib/python* -type d -name '*PyYAML*' -delete + - name: Remove problematic PyYAML + run: pip3 uninstall --force PyYAML # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 47dbaa88f53a7dc6210340ed31207f5eca74bf0d Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 22 Aug 2023 11:46:03 +0200 Subject: [PATCH 092/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index c5c4c1a4c..0f3ece0a6 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -78,7 +78,7 @@ jobs: run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - name: Remove problematic PyYAML - run: pip3 uninstall --force PyYAML + run: python3 -m pip uninstall --force PyYAML # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From deed41bfb1731e46e8dfe1b6c92156bd8ae0b063 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 22 Aug 2023 13:05:41 +0200 Subject: [PATCH 093/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 0f3ece0a6..d0165399e 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -78,7 +78,7 @@ jobs: run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - name: Remove problematic PyYAML - run: python3 -m pip uninstall --force PyYAML + run: python3 -m pip uninstall -y --root-user-action PyYAML # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 1d82edfb0f33112da1b0afea346abe7801a13f5d Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 22 Aug 2023 13:39:40 +0200 Subject: [PATCH 094/141] Update build-container-and-test-aarch64.yml --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index d0165399e..8f6e48376 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -78,7 +78,7 @@ jobs: run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - name: Remove problematic PyYAML - run: python3 -m pip uninstall -y --root-user-action PyYAML + run: python -m pip uninstall -y --root-user-action PyYAML # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 2369a977f0caa0d0fc69e074307b89a32f20d38c Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 22 Aug 2023 15:14:52 +0200 Subject: [PATCH 095/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index d0165399e..833b1745a 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -76,9 +76,12 @@ jobs: - name: Extract resources from gProfiler executable run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - - - name: Remove problematic PyYAML - run: python3 -m pip uninstall -y --root-user-action PyYAML + + - name: Remove problematic PyYAML from site-packages + run: find /usr/lib/python* -type d -name '*yaml*' -delete + + - name: Remove problematic PyYAML from site-packages + run: python -m site # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 9fe633dc23d0029a036644f6cf38a1aeffc63ec3 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 22 Aug 2023 16:27:59 +0200 Subject: [PATCH 096/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 833b1745a..4d8d46acb 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -78,7 +78,7 @@ jobs: run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - name: Remove problematic PyYAML from site-packages - run: find /usr/lib/python* -type d -name '*yaml*' -delete + run: sudo find /usr/lib/python* -type d -name '*yaml*' -delete - name: Remove problematic PyYAML from site-packages run: python -m site From c88bd12c312c86f3e14be66b683e914a8607128f Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 22 Aug 2023 17:27:04 +0200 Subject: [PATCH 097/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 4d8d46acb..04ed218f5 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -78,7 +78,7 @@ jobs: run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - name: Remove problematic PyYAML from site-packages - run: sudo find /usr/lib/python* -type d -name '*yaml*' -delete + run: sudo find /usr/lib/python* -type d -name '*yaml*' -exec rm rf {} + - name: Remove problematic PyYAML from site-packages run: python -m site From e9137e198a1ec12d1085f0e0c580cb4768e107a2 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 22 Aug 2023 18:08:19 +0200 Subject: [PATCH 098/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 04ed218f5..3af364dac 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -78,7 +78,7 @@ jobs: run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - name: Remove problematic PyYAML from site-packages - run: sudo find /usr/lib/python* -type d -name '*yaml*' -exec rm rf {} + + run: sudo find /usr/lib/python* -type d -name '*yaml*' -exec rm -rf {} + - name: Remove problematic PyYAML from site-packages run: python -m site From 227edbb63883d25a8560c3b5665948168d4441e9 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 23 Aug 2023 09:04:45 +0200 Subject: [PATCH 099/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 3af364dac..9aa0fa392 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -80,9 +80,6 @@ jobs: - name: Remove problematic PyYAML from site-packages run: sudo find /usr/lib/python* -type d -name '*yaml*' -exec rm -rf {} + - - name: Remove problematic PyYAML from site-packages - run: python -m site - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: sudo ./tests/test.sh --ignore=tests/test_executable.py From 0167c9406b2e69f3aed964165aaf288f842d2456 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 23 Aug 2023 09:55:36 +0200 Subject: [PATCH 100/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 9aa0fa392..017b2e24d 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -80,6 +80,9 @@ jobs: - name: Remove problematic PyYAML from site-packages run: sudo find /usr/lib/python* -type d -name '*yaml*' -exec rm -rf {} + + - name: Remove problematic PyYAML from site-packages + run: sudo find /usr/local/lib/python* -type d -name '*yaml*' -exec rm -rf {} + + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: sudo ./tests/test.sh --ignore=tests/test_executable.py From 78b9974685e157f2a82a4941f42277ac9083015f Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 23 Aug 2023 12:21:44 +0200 Subject: [PATCH 101/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 017b2e24d..39695ea28 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -78,10 +78,10 @@ jobs: run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - name: Remove problematic PyYAML from site-packages - run: sudo find /usr/lib/python* -type d -name '*yaml*' -exec rm -rf {} + - - - name: Remove problematic PyYAML from site-packages - run: sudo find /usr/local/lib/python* -type d -name '*yaml*' -exec rm -rf {} + + run: | + sudo apt-get -qq update + sudo apt-get install -y -qq --no-install-recommends python3 python3-pip + python -m pip install --ignore-installed PyYAML==6.0 # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 3a45b062b580e3218b4de6452791d4c635c14be5 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 23 Aug 2023 13:03:49 +0200 Subject: [PATCH 102/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 39695ea28..2b5488f41 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -77,11 +77,8 @@ jobs: - name: Extract resources from gProfiler executable run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - - name: Remove problematic PyYAML from site-packages - run: | - sudo apt-get -qq update - sudo apt-get install -y -qq --no-install-recommends python3 python3-pip - python -m pip install --ignore-installed PyYAML==6.0 + - name: update PyYAML with --ignore-installed cause its setuptools + run: sudo apt-get -qq update && sudo apt-get install -y -qq --no-install-recommends python3 python3-pip && python -m pip install --ignore-installed PyYAML==6.0 # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 3d90538f44c72b4b7ca3e335510166072ac60bd5 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 23 Aug 2023 13:36:53 +0200 Subject: [PATCH 103/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- scripts/prepare_machine-unknown-linux-musl.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 2b5488f41..427e775e0 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -78,7 +78,7 @@ jobs: run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - name: update PyYAML with --ignore-installed cause its setuptools - run: sudo apt-get -qq update && sudo apt-get install -y -qq --no-install-recommends python3 python3-pip && python -m pip install --ignore-installed PyYAML==6.0 + run: sudo apt-get -qq update && sudo apt-get install -y -qq --no-install-recommends python3 python3-pip && python3 -m pip install --ignore-installed PyYAML==6.0 # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests diff --git a/scripts/prepare_machine-unknown-linux-musl.sh b/scripts/prepare_machine-unknown-linux-musl.sh index c6e378131..60cc8bbdb 100755 --- a/scripts/prepare_machine-unknown-linux-musl.sh +++ b/scripts/prepare_machine-unknown-linux-musl.sh @@ -19,7 +19,7 @@ target="$(uname -m)"-unknown-linux-musl target_dir="/usr/local/musl/$target/" # as searched for by the remoteprocess create. rustup target add "$target" -apk add --no-cache musl-dev make git curl # git & curl used by next scripts +apk add --no-cache musl-dev make git curl # git & curl used by next scripts mkdir builds && cd builds From 307441e489c5e387dfadaec1120a0cf43d7f552a Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 23 Aug 2023 14:35:06 +0200 Subject: [PATCH 104/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 427e775e0..4c2020611 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -77,8 +77,8 @@ jobs: - name: Extract resources from gProfiler executable run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - - name: update PyYAML with --ignore-installed cause its setuptools - run: sudo apt-get -qq update && sudo apt-get install -y -qq --no-install-recommends python3 python3-pip && python3 -m pip install --ignore-installed PyYAML==6.0 + - name: update PyYAML with --ignore-installed cause its distutils + run: sudo apt-get -qq update && sudo apt-get install -y -qq --no-install-recommends python3 python3-pip && python3 -m pip uninstall -y PyYAML # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 20863a8d6818a95855c899e324d6c2771b09ecca Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 23 Aug 2023 14:52:37 +0200 Subject: [PATCH 105/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 4c2020611..8a926f09b 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -78,7 +78,7 @@ jobs: run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - name: update PyYAML with --ignore-installed cause its distutils - run: sudo apt-get -qq update && sudo apt-get install -y -qq --no-install-recommends python3 python3-pip && python3 -m pip uninstall -y PyYAML + run: sudo apt-get -qq update && sudo apt-get install -y -qq --no-install-recommends python3 python3-pip && sudo python3 -m pip install --ignore-installed PyYAML==6.0 # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From ab49a8c3a2cb88a0eb3dce40b51a50747540e7c7 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 24 Aug 2023 14:04:54 +0200 Subject: [PATCH 106/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 8a926f09b..748648edc 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -78,8 +78,8 @@ jobs: run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - name: update PyYAML with --ignore-installed cause its distutils - run: sudo apt-get -qq update && sudo apt-get install -y -qq --no-install-recommends python3 python3-pip && sudo python3 -m pip install --ignore-installed PyYAML==6.0 + run: sudo apt-get -qq update && sudo apt-get install -y -qq --no-install-recommends python3 python3-pip && python3 -m pip install --ignore-installed PyYAML==6.0 # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests - run: sudo ./tests/test.sh --ignore=tests/test_executable.py + run: ./tests/test.sh --ignore=tests/test_executable.py From 8d8f71cf055ab413a94bc848a825550a3e52c555 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 24 Aug 2023 15:19:28 +0200 Subject: [PATCH 107/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 8 -------- tests/test.sh | 1 + 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 748648edc..34cb4d499 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -10,8 +10,6 @@ jobs: - ARM64 steps: - - name: change pycache permissions - run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler - name: Checkout Code uses: actions/checkout@v3 @@ -57,9 +55,6 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: change pycache permissions - run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler - - name: Checkout Code uses: actions/checkout@v3 with: @@ -77,9 +72,6 @@ jobs: - name: Extract resources from gProfiler executable run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - - name: update PyYAML with --ignore-installed cause its distutils - run: sudo apt-get -qq update && sudo apt-get install -y -qq --no-install-recommends python3 python3-pip && python3 -m pip install --ignore-installed PyYAML==6.0 - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: ./tests/test.sh --ignore=tests/test_executable.py diff --git a/tests/test.sh b/tests/test.sh index 377e7cf1e..363e120f1 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -36,5 +36,6 @@ fi python3 -m pip install -q --upgrade setuptools pip python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt +python3 -m pip show pytest # TODO: python3 -m pip install . sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 7c7af02e8110da04cb414564b922e0ad654bde46 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 24 Aug 2023 15:59:22 +0200 Subject: [PATCH 108/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 34cb4d499..df4586a0d 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -10,7 +10,8 @@ jobs: - ARM64 steps: - + - name: change pycache permissions + run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler - name: Checkout Code uses: actions/checkout@v3 with: From 346d98322d52485315a7e70a1fd728ebb15a56c3 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 24 Aug 2023 17:15:43 +0200 Subject: [PATCH 109/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index df4586a0d..889bd8a63 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -72,7 +72,8 @@ jobs: - name: Extract resources from gProfiler executable run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - + - name: install pytest + run: sudo apt install -y python3-pytest # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: ./tests/test.sh --ignore=tests/test_executable.py From f4814762d0b047c5828f13184eb761c2b853db88 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Fri, 25 Aug 2023 13:50:46 +0200 Subject: [PATCH 110/141] how correct is that --- .github/workflows/build-container-and-test-aarch64.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 889bd8a63..bc4655263 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -72,8 +72,10 @@ jobs: - name: Extract resources from gProfiler executable run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources + - name: install pytest - run: sudo apt install -y python3-pytest + run: ./scripts/setup_runner_requirements.sh + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: ./tests/test.sh --ignore=tests/test_executable.py From 5552817377ba09455557bb08123f400af8005aa4 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Sat, 26 Aug 2023 19:11:47 +0200 Subject: [PATCH 111/141] pythonpath? --- .github/workflows/build-container-and-test-aarch64.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index bc4655263..08a450361 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -73,9 +73,9 @@ jobs: - name: Extract resources from gProfiler executable run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - - name: install pytest - run: ./scripts/setup_runner_requirements.sh - + - name: expand PYTHONPATH + run: echo 'export PYTHONPATH="${PYTHONPATH}:/home/ubuntu/.local/lib/python3.8/site-packages' >> ~/.bashrc + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: ./tests/test.sh --ignore=tests/test_executable.py From 5b29cdae75b03ed3a87b7300d0c5c371a9ed8611 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Sat, 26 Aug 2023 19:48:20 +0200 Subject: [PATCH 112/141] pythonpath? --- tests/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test.sh b/tests/test.sh index 363e120f1..05a53e319 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -37,5 +37,6 @@ fi python3 -m pip install -q --upgrade setuptools pip python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt python3 -m pip show pytest +echo ~/.bashrc # TODO: python3 -m pip install . sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 83887a063bbbe2853cdb002b01742dbb226eab44 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Sat, 26 Aug 2023 20:36:30 +0200 Subject: [PATCH 113/141] pythonpath? --- tests/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.sh b/tests/test.sh index 05a53e319..68c2a476f 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -37,6 +37,6 @@ fi python3 -m pip install -q --upgrade setuptools pip python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt python3 -m pip show pytest -echo ~/.bashrc +cat ~/.bashrc # TODO: python3 -m pip install . sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From f5ecbd248cd8ea1a8f26d8be2a37b1a6dc45e861 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Sun, 27 Aug 2023 10:15:06 +0200 Subject: [PATCH 114/141] lets check the PYTHONPATH --- .github/workflows/build-container-and-test-aarch64.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 08a450361..617c44e82 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -73,8 +73,8 @@ jobs: - name: Extract resources from gProfiler executable run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - - name: expand PYTHONPATH - run: echo 'export PYTHONPATH="${PYTHONPATH}:/home/ubuntu/.local/lib/python3.8/site-packages' >> ~/.bashrc + - name: setup runner requirements + run: ./setup_runner_requirements.sh # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From db3153788579e6baa6c112dea6de76bbcd904ecf Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Sun, 27 Aug 2023 10:46:19 +0200 Subject: [PATCH 115/141] lets check the PYTHONPATH --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 617c44e82..100938ac2 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -74,7 +74,7 @@ jobs: run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - name: setup runner requirements - run: ./setup_runner_requirements.sh + run: ./scripts/setup_runner_requirements.sh # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 283dccf4e1513681ce9ebd2240b61e36995be37d Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Sun, 27 Aug 2023 11:20:24 +0200 Subject: [PATCH 116/141] lets check the PYTHONPATH --- tests/test.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test.sh b/tests/test.sh index 68c2a476f..92f639021 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -36,7 +36,8 @@ fi python3 -m pip install -q --upgrade setuptools pip python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt -python3 -m pip show pytest -cat ~/.bashrc +if [ "$(uname -m)" = "aarch64" ]; then + export PYTHONPATH="${PYTHONPATH}:/home/ubuntu/.local/lib/python3.8/site-packages" +fi # TODO: python3 -m pip install . sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 0ca35c8cda68c6aa86c88476491e303856df77e9 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Sun, 27 Aug 2023 11:52:38 +0200 Subject: [PATCH 117/141] lets check the PYTHONPATH --- .github/workflows/build-container-and-test-aarch64.yml | 5 ++++- tests/test.sh | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 100938ac2..6ae135646 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -78,4 +78,7 @@ jobs: # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests - run: ./tests/test.sh --ignore=tests/test_executable.py + run: | + virtualenv --python python3.8 aarch64-venv + source aarch64-venv/bin/activate + ./tests/test.sh --ignore=tests/test_executable.py diff --git a/tests/test.sh b/tests/test.sh index 92f639021..4f62a9b17 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -36,8 +36,6 @@ fi python3 -m pip install -q --upgrade setuptools pip python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt -if [ "$(uname -m)" = "aarch64" ]; then - export PYTHONPATH="${PYTHONPATH}:/home/ubuntu/.local/lib/python3.8/site-packages" -fi + # TODO: python3 -m pip install . sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" From 82c0681132effb9590351c26744dc55b358cff21 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Sun, 27 Aug 2023 12:40:01 +0200 Subject: [PATCH 118/141] lets check the PYTHONPATH --- .github/workflows/build-container-and-test-aarch64.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 6ae135646..c8df80708 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -79,6 +79,7 @@ jobs: # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: | + python3 -m pip install virtualenv virtualenv --python python3.8 aarch64-venv source aarch64-venv/bin/activate ./tests/test.sh --ignore=tests/test_executable.py From 487ea66f080dd3683ef020179c4bd63c0fda88f3 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Sun, 27 Aug 2023 14:05:33 +0200 Subject: [PATCH 119/141] venv approach --- .github/workflows/build-container-and-test-aarch64.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index c8df80708..c7d66062e 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -80,6 +80,8 @@ jobs: - name: Run gProfiler tests run: | python3 -m pip install virtualenv - virtualenv --python python3.8 aarch64-venv + sudo virtualenv --python python3.8 aarch64-venv + sudo chown -R ${USER} aarch64-venv + sudo chmod -R a+rX aarch64-venv source aarch64-venv/bin/activate ./tests/test.sh --ignore=tests/test_executable.py From eaaf31c2c453ed85ded2c71b28671a236a2f4fd6 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Sun, 27 Aug 2023 14:55:51 +0200 Subject: [PATCH 120/141] Update build-container-and-test-aarch64.yml --- .github/workflows/build-container-and-test-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index c7d66062e..a3b5c7cd8 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -79,7 +79,7 @@ jobs: # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: | - python3 -m pip install virtualenv + sudo python3 -m pip install virtualenv sudo virtualenv --python python3.8 aarch64-venv sudo chown -R ${USER} aarch64-venv sudo chmod -R a+rX aarch64-venv From c10ee6c3e268c520e84dcf5a74661635e422f2b8 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 29 Aug 2023 13:07:07 +0200 Subject: [PATCH 121/141] show me the error mr gprofiler --- tests/test_preconditions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_preconditions.py b/tests/test_preconditions.py index 04ef710a2..b1c22debd 100644 --- a/tests/test_preconditions.py +++ b/tests/test_preconditions.py @@ -89,7 +89,7 @@ def test_not_root( # exits without an error with pytest.raises(ContainerError) as e: wait_for_container(gprofiler) - + print(e.value.stderr) assert e.value.exit_status == 1 assert e.value.stderr == b"Must run gprofiler as root, please re-run.\n" From 363a4bf4cff00eb070b9ce59d8a29bf94d078796 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 29 Aug 2023 15:09:25 +0200 Subject: [PATCH 122/141] show me the error mr gprofiler --- .github/workflows/build-container-and-test-aarch64.yml | 4 +++- tests/test_preconditions.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index a3b5c7cd8..3301a681e 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -12,6 +12,7 @@ jobs: steps: - name: change pycache permissions run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler + - name: Checkout Code uses: actions/checkout@v3 with: @@ -75,7 +76,8 @@ jobs: - name: setup runner requirements run: ./scripts/setup_runner_requirements.sh - + - name: change pycache permissions + run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: | diff --git a/tests/test_preconditions.py b/tests/test_preconditions.py index b1c22debd..04ef710a2 100644 --- a/tests/test_preconditions.py +++ b/tests/test_preconditions.py @@ -89,7 +89,7 @@ def test_not_root( # exits without an error with pytest.raises(ContainerError) as e: wait_for_container(gprofiler) - print(e.value.stderr) + assert e.value.exit_status == 1 assert e.value.stderr == b"Must run gprofiler as root, please re-run.\n" From 00a11067aadec4b8984f9c6939d69701e5dcb2ac Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 30 Aug 2023 09:46:24 +0200 Subject: [PATCH 123/141] one more test xfailed --- .github/workflows/build-container-and-test-aarch64.yml | 3 +-- scripts/setup_venv_aarch64.sh | 9 +++++++++ tests/test.sh | 1 - tests/test_preconditions.py | 4 +++- 4 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 scripts/setup_venv_aarch64.sh diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 3301a681e..400ab259f 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -76,8 +76,7 @@ jobs: - name: setup runner requirements run: ./scripts/setup_runner_requirements.sh - - name: change pycache permissions - run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests run: | diff --git a/scripts/setup_venv_aarch64.sh b/scripts/setup_venv_aarch64.sh new file mode 100644 index 000000000..7fa0312c2 --- /dev/null +++ b/scripts/setup_venv_aarch64.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# +# Copyright (c) Granulate. All rights reserved. +# Licensed under the AGPL3 License. See LICENSE.md in the project root for license information. +# +set -euo pipefail + +virtualenv --python python3.8 aarch64-venv +echo 'source ${PWD}/aarch64-venv/bin/activate' >> ~/.bashrc \ No newline at end of file diff --git a/tests/test.sh b/tests/test.sh index 4f62a9b17..377e7cf1e 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -36,6 +36,5 @@ fi python3 -m pip install -q --upgrade setuptools pip python3 -m pip install -r ./requirements.txt -r ./exe-requirements.txt -r ./dev-requirements.txt - # TODO: python3 -m pip install . sudo env "PATH=$PATH" python3 -m pytest -v tests/ "$@" diff --git a/tests/test_preconditions.py b/tests/test_preconditions.py index 04ef710a2..083393208 100644 --- a/tests/test_preconditions.py +++ b/tests/test_preconditions.py @@ -12,7 +12,7 @@ from docker.models.containers import Container from docker.models.images import Image -from tests.utils import start_gprofiler_in_container_for_one_session, wait_for_container, wait_for_log +from tests.utils import is_aarch64, start_gprofiler_in_container_for_one_session, wait_for_container, wait_for_log def start_gprofiler( @@ -84,6 +84,8 @@ def test_not_root( """ gProfiler must run as root and should complain otherwise. """ + if is_aarch64(): + pytest.xfail("This combination fails on aarch64, see https://github.com/Granulate/gprofiler/issues/849") gprofiler = start_gprofiler(docker_client, gprofiler_docker_image, user=42) # exits without an error From f13f92b2d00a349272a8a4460b4a9aa957445fda Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 30 Aug 2023 10:01:28 +0200 Subject: [PATCH 124/141] one more test xfailed --- scripts/setup_venv_aarch64.sh | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 scripts/setup_venv_aarch64.sh diff --git a/scripts/setup_venv_aarch64.sh b/scripts/setup_venv_aarch64.sh deleted file mode 100644 index 7fa0312c2..000000000 --- a/scripts/setup_venv_aarch64.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) Granulate. All rights reserved. -# Licensed under the AGPL3 License. See LICENSE.md in the project root for license information. -# -set -euo pipefail - -virtualenv --python python3.8 aarch64-venv -echo 'source ${PWD}/aarch64-venv/bin/activate' >> ~/.bashrc \ No newline at end of file From 62a72b29c54d71fb55cfdb610bf01a01a4f10494 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Wed, 30 Aug 2023 13:57:22 +0200 Subject: [PATCH 125/141] one more test xfailed --- .github/workflows/build-container-and-test-aarch64.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index 400ab259f..b265abbeb 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -10,9 +10,6 @@ jobs: - ARM64 steps: - - name: change pycache permissions - run: sudo chown -R ${USER} /home/ubuntu/actions-runner/_work/gprofiler/gprofiler - - name: Checkout Code uses: actions/checkout@v3 with: From 39036f67d0d80d1f3366279b64bbac51e3b4c8e8 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 31 Aug 2023 13:05:44 +0200 Subject: [PATCH 126/141] one more test xfailed --- .../build-container-and-test-aarch64.yml | 2 +- .github/workflows/build-test-deploy.yml | 108 ++++++++++++------ 2 files changed, 74 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index b265abbeb..d3fa78d56 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -39,7 +39,7 @@ jobs: path: output/ retention-days: 1 - test: + test-container-aarch64: needs: build-container-aarch64 runs-on: diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 39caf843b..e1a95865e 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -108,8 +108,10 @@ jobs: NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable build-executable-aarch64: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ["self-hosted", "ARM64"] + runs-on: + - self-hosted + - public + - ARM64 steps: - name: Checkout Code uses: actions/checkout@v3 @@ -261,49 +263,85 @@ jobs: run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py build-container-aarch64: - if: startsWith(github.ref, 'refs/tags/') - needs: build-executable-aarch64 - runs-on: ["self-hosted", "ARM64"] + runs-on: + - self-hosted + - public + - ARM64 + steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true - - name: Set up runner requirements - run: scripts/setup_runner_requirements.sh + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - - name: Get and verify tag value - run: ./scripts/verify_tag.sh + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # more nicely. the tests will then be able to use the built image. + - name: Build gProfiler image + run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + - name: Export gProfiler image + run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img - - name: Download executables from the previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler - path: output/ + - name: Upload the image artifact + uses: actions/upload-artifact@v2 + with: + name: gprofiler_aarch64.img + path: output/ + retention-days: 1 - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - - name: Build and push - run: | - set -x - export DOCKER_BUILDKIT=1 + test-container-aarch64: + needs: build-container-aarch64 - BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" - AARCH64_IMAGE="$BASE_IMAGE-aarch64" + runs-on: + - self-hosted + - public + - ARM64 + steps: + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - # build & push Aarch64 - ./scripts/build_aarch64_container.sh --skip-exe-build --build-arg EXE_PATH=output/gprofiler_aarch64 -t "$AARCH64_IMAGE" --push + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Download the executable from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler_aarch64.img + path: output/ + - name: Import gProfiler image + run: docker image load < output/gprofiler_aarch64.img + + - name: Extract resources from gProfiler executable + run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources + + - name: setup runner requirements + run: ./scripts/setup_runner_requirements.sh + + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" + - name: Run gProfiler tests + run: | + sudo python3 -m pip install virtualenv + sudo virtualenv --python python3.8 aarch64-venv + sudo chown -R ${USER} aarch64-venv + sudo chmod -R a+rX aarch64-venv + source aarch64-venv/bin/activate + ./tests/test.sh --ignore=tests/test_executable.py deploy-container: if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest From 6c53d005a695990e18509b4cc256bb692f479829 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Thu, 31 Aug 2023 13:09:34 +0200 Subject: [PATCH 127/141] one more test xfailed --- .../build-container-and-test-aarch64.yml | 168 +++++++++--------- 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml index d3fa78d56..84031f179 100644 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ b/.github/workflows/build-container-and-test-aarch64.yml @@ -1,85 +1,85 @@ -name: Lint, build gProfiler container and run tests on aarch64 - -on: pull_request - -jobs: - build-container-aarch64: - runs-on: - - self-hosted - - public - - ARM64 - - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs - # more nicely. the tests will then be able to use the built image. - - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 - - - name: Export gProfiler image - run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img - - - name: Upload the image artifact - uses: actions/upload-artifact@v2 - with: - name: gprofiler_aarch64.img - path: output/ - retention-days: 1 - - test-container-aarch64: - needs: build-container-aarch64 - - runs-on: - - self-hosted - - public - - ARM64 - steps: +# name: Lint, build gProfiler container and run tests on aarch64 + +# on: pull_request + +# jobs: +# build-container-aarch64: +# runs-on: +# - self-hosted +# - public +# - ARM64 + +# steps: +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# submodules: true + +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v2 + +# - name: Login to DockerHub +# uses: docker/login-action@v1 +# with: +# username: ${{ secrets.DOCKERHUB_USERNAME }} +# password: ${{ secrets.DOCKERHUB_TOKEN }} + +# # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs +# # more nicely. the tests will then be able to use the built image. +# - name: Build gProfiler image +# run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 + +# - name: Export gProfiler image +# run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img + +# - name: Upload the image artifact +# uses: actions/upload-artifact@v2 +# with: +# name: gprofiler_aarch64.img +# path: output/ +# retention-days: 1 + +# test-container-aarch64: +# needs: build-container-aarch64 + +# runs-on: +# - self-hosted +# - public +# - ARM64 +# steps: - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true - - - name: Download the executable from previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler_aarch64.img - path: output/ - - - name: Import gProfiler image - run: docker image load < output/gprofiler_aarch64.img - - - name: Extract resources from gProfiler executable - run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - - - name: setup runner requirements - run: ./scripts/setup_runner_requirements.sh - - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - - name: Run gProfiler tests - run: | - sudo python3 -m pip install virtualenv - sudo virtualenv --python python3.8 aarch64-venv - sudo chown -R ${USER} aarch64-venv - sudo chmod -R a+rX aarch64-venv - source aarch64-venv/bin/activate - ./tests/test.sh --ignore=tests/test_executable.py +# - name: Login to DockerHub +# uses: docker/login-action@v1 +# with: +# username: ${{ secrets.DOCKERHUB_USERNAME }} +# password: ${{ secrets.DOCKERHUB_TOKEN }} + +# - name: Checkout Code +# uses: actions/checkout@v3 +# with: +# submodules: true + +# - name: Download the executable from previous job +# uses: actions/download-artifact@v2 +# with: +# name: gprofiler_aarch64.img +# path: output/ + +# - name: Import gProfiler image +# run: docker image load < output/gprofiler_aarch64.img + +# - name: Extract resources from gProfiler executable +# run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources + +# - name: setup runner requirements +# run: ./scripts/setup_runner_requirements.sh + +# # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" +# - name: Run gProfiler tests +# run: | +# sudo python3 -m pip install virtualenv +# sudo virtualenv --python python3.8 aarch64-venv +# sudo chown -R ${USER} aarch64-venv +# sudo chmod -R a+rX aarch64-venv +# source aarch64-venv/bin/activate +# ./tests/test.sh --ignore=tests/test_executable.py From c1b8a96cf1c76184c54c720733e5bab51ef9a19a Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Fri, 1 Sep 2023 10:39:31 +0200 Subject: [PATCH 128/141] scipt removed --- .../build-container-and-test-aarch64.yml | 85 ------------------- 1 file changed, 85 deletions(-) delete mode 100644 .github/workflows/build-container-and-test-aarch64.yml diff --git a/.github/workflows/build-container-and-test-aarch64.yml b/.github/workflows/build-container-and-test-aarch64.yml deleted file mode 100644 index 84031f179..000000000 --- a/.github/workflows/build-container-and-test-aarch64.yml +++ /dev/null @@ -1,85 +0,0 @@ -# name: Lint, build gProfiler container and run tests on aarch64 - -# on: pull_request - -# jobs: -# build-container-aarch64: -# runs-on: -# - self-hosted -# - public -# - ARM64 - -# steps: -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# submodules: true - -# - name: Set up Docker Buildx -# uses: docker/setup-buildx-action@v2 - -# - name: Login to DockerHub -# uses: docker/login-action@v1 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} - -# # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs -# # more nicely. the tests will then be able to use the built image. -# - name: Build gProfiler image -# run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 - -# - name: Export gProfiler image -# run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img - -# - name: Upload the image artifact -# uses: actions/upload-artifact@v2 -# with: -# name: gprofiler_aarch64.img -# path: output/ -# retention-days: 1 - -# test-container-aarch64: -# needs: build-container-aarch64 - -# runs-on: -# - self-hosted -# - public -# - ARM64 -# steps: - -# - name: Login to DockerHub -# uses: docker/login-action@v1 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} - -# - name: Checkout Code -# uses: actions/checkout@v3 -# with: -# submodules: true - -# - name: Download the executable from previous job -# uses: actions/download-artifact@v2 -# with: -# name: gprofiler_aarch64.img -# path: output/ - -# - name: Import gProfiler image -# run: docker image load < output/gprofiler_aarch64.img - -# - name: Extract resources from gProfiler executable -# run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - -# - name: setup runner requirements -# run: ./scripts/setup_runner_requirements.sh - -# # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" -# - name: Run gProfiler tests -# run: | -# sudo python3 -m pip install virtualenv -# sudo virtualenv --python python3.8 aarch64-venv -# sudo chown -R ${USER} aarch64-venv -# sudo chmod -R a+rX aarch64-venv -# source aarch64-venv/bin/activate -# ./tests/test.sh --ignore=tests/test_executable.py From a4c35ddd93265c456ff9320c102dab4e6119c32c Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Fri, 1 Sep 2023 14:12:42 +0200 Subject: [PATCH 129/141] output type --- .github/workflows/build-test-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 2c66d2fe0..105f7a3d6 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -285,7 +285,7 @@ jobs: # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs # more nicely. the tests will then be able to use the built image. - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 + run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 --output type=docker - name: Export gProfiler image run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img From 9888f76e6db2b713cafbbbf8e254e833cde72d72 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Mon, 4 Sep 2023 11:55:42 +0200 Subject: [PATCH 130/141] buildx? --- .github/workflows/build-test-deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 105f7a3d6..c979725ed 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -126,6 +126,9 @@ jobs: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV ./scripts/cmp_tags.sh + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build gProfiler executable run: | mkdir -p output From deb77153b1fae953c4a2e33e4cd7fa6846664a4d Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Mon, 4 Sep 2023 12:13:44 +0200 Subject: [PATCH 131/141] buildx? requirements? --- .github/workflows/build-test-deploy.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index c979725ed..2da924f32 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -274,8 +274,12 @@ jobs: - name: Checkout Code uses: actions/checkout@v3 with: + fetch-depth: 0 submodules: true + - name: Get and verify tag value + run: ./scripts/verify_tag.sh + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 From cd27bc86e81c77f1f43de978d466a3f64247e199 Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Mon, 4 Sep 2023 14:43:45 +0200 Subject: [PATCH 132/141] buildx? requirements? --- .github/workflows/build-test-deploy.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 2da924f32..092309c28 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -277,6 +277,9 @@ jobs: fetch-depth: 0 submodules: true + - name: setup runner requirements + run: ./scripts/setup_runner_requirements.sh + - name: Get and verify tag value run: ./scripts/verify_tag.sh @@ -336,8 +339,6 @@ jobs: - name: Extract resources from gProfiler executable run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources - - name: setup runner requirements - run: ./scripts/setup_runner_requirements.sh # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - name: Run gProfiler tests From 7fd7d14689d4cfac0b953b0c653148080220bbee Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 5 Sep 2023 16:07:44 +0200 Subject: [PATCH 133/141] executable aarch tests --- .github/workflows/build-test-deploy.yml | 38 ++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 092309c28..930cb4e16 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -138,10 +138,46 @@ jobs: - name: Upload the executables as job artifacts uses: actions/upload-artifact@v2 with: - name: gprofiler + name: gprofiler_aarch64 path: output/ retention-days: 1 + test-executable-aarch64: + needs: build-container-aarch64 + + runs-on: + - self-hosted + - public + - ARM64 + steps: + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Download the executable from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler-aarch64 + path: dist/ + + # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" + - name: Run gProfiler tests + run: | + sudo python3 -m pip install virtualenv + sudo virtualenv --python python3.8 aarch64-venv + sudo chown -R ${USER} aarch64-venv + sudo chmod -R a+rX aarch64-venv + source aarch64-venv/bin/activate + ./tests/test.sh --executable dist/gprofiler_aarch64 -k test_executable + deploy-executable: if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-20.04 From fa4ce534a172a8c25f92346be5605b9460253fcd Mon Sep 17 00:00:00 2001 From: pfilipko1 Date: Tue, 12 Sep 2023 13:15:21 +0200 Subject: [PATCH 134/141] stupid aarch64 ubuntu --- .github/workflows/build-test-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 930cb4e16..b2c8fe506 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -124,7 +124,7 @@ jobs: - name: Get and verify tag value run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - ./scripts/cmp_tags.sh + sudo ./scripts/cmp_tags.sh - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 From 933546bc4e20470e04d6bdf925072433b8cbe3d5 Mon Sep 17 00:00:00 2001 From: Roi Agai Date: Sun, 12 Nov 2023 15:01:04 +0200 Subject: [PATCH 135/141] cre --- .github/workflows/build-test-deploy.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index b2c8fe506..ff9a30e6f 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -307,11 +307,11 @@ jobs: - ARM64 steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true - name: setup runner requirements run: ./scripts/setup_runner_requirements.sh From 8bbde84b995990001fdb54b0aa783634a2fc96f6 Mon Sep 17 00:00:00 2001 From: Roi Agai Date: Sun, 12 Nov 2023 15:04:16 +0200 Subject: [PATCH 136/141] ? --- .github/workflows/build-test-deploy.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index ff9a30e6f..b2c8fe506 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -307,11 +307,11 @@ jobs: - ARM64 steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true - name: setup runner requirements run: ./scripts/setup_runner_requirements.sh From 4111253df5bbf1463870ff2d81d020a1729861bb Mon Sep 17 00:00:00 2001 From: Roi Agai Date: Sun, 12 Nov 2023 16:34:52 +0200 Subject: [PATCH 137/141] remove sudo --- .github/workflows/build-test-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index b2c8fe506..930cb4e16 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -124,7 +124,7 @@ jobs: - name: Get and verify tag value run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - sudo ./scripts/cmp_tags.sh + ./scripts/cmp_tags.sh - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 From ac7e9ab0c4b3da87f731b621e1fc04d2eb6f1e9d Mon Sep 17 00:00:00 2001 From: Roi Agai Date: Sun, 12 Nov 2023 16:44:28 +0200 Subject: [PATCH 138/141] ? --- scripts/cmp_tags.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cmp_tags.sh b/scripts/cmp_tags.sh index dab3e4978..5c541487c 100755 --- a/scripts/cmp_tags.sh +++ b/scripts/cmp_tags.sh @@ -8,7 +8,7 @@ set -euo pipefail # Used in CI and checks that last pushed tag is greater than last existing tag. # Using python package 'cmp_version' to do the compare work -pip install cmp_version +pip3 install cmp_version TAGS=$(git tag --sort=creatordate | tail -2) # shellcheck disable=SC2206 # expansion is desired here to get array values From 8115d7b38cf8446c6e66579a920bb8a8cd60eddc Mon Sep 17 00:00:00 2001 From: Roi Agai Date: Sun, 12 Nov 2023 16:52:08 +0200 Subject: [PATCH 139/141] ? --- .github/workflows/build-test-deploy.yml | 2 +- scripts/cmp_tags.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 930cb4e16..d923d072d 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -317,7 +317,7 @@ jobs: run: ./scripts/setup_runner_requirements.sh - name: Get and verify tag value - run: ./scripts/verify_tag.sh + run: sudo ./scripts/verify_tag.sh - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 diff --git a/scripts/cmp_tags.sh b/scripts/cmp_tags.sh index 5c541487c..f808c5e97 100755 --- a/scripts/cmp_tags.sh +++ b/scripts/cmp_tags.sh @@ -8,7 +8,7 @@ set -euo pipefail # Used in CI and checks that last pushed tag is greater than last existing tag. # Using python package 'cmp_version' to do the compare work -pip3 install cmp_version +sudo pip install cmp_version TAGS=$(git tag --sort=creatordate | tail -2) # shellcheck disable=SC2206 # expansion is desired here to get array values From bed61f33fec54916c724bcc1fcf4713706a2b073 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 24 Nov 2023 23:02:25 +0200 Subject: [PATCH 140/141] fixes --- .github/workflows/build-test-deploy.yml | 83 ++++++++++++++----------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 85dae88c2..592c1558d 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -48,10 +48,10 @@ jobs: test-executable-x64: if: ${{ !startsWith(github.ref, 'refs/tags/') }} + needs: build-executable-x64 runs-on: ubuntu-20.04 # the tests which use ruby/node/python etc and run at non-root privs, fail to read the files when running # on ubuntu-22.04/ubuntu-latest: # stderr: ruby: Permission denied -- /home/runner/work/gprofiler/gprofiler/tests/containers/ruby/fibonacci.rb (LoadError) - needs: build-executable-x64 strategy: fail-fast: false matrix: @@ -121,11 +121,6 @@ jobs: - name: Set up requirements run: scripts/setup_runner_requirements.sh - - name: Get and verify tag value - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - ./scripts/cmp_tags.sh - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -143,40 +138,61 @@ jobs: retention-days: 1 test-executable-aarch64: + if: !startsWith(github.ref, 'refs/tags/') needs: build-container-aarch64 - runs-on: - self-hosted - public - ARM64 + strategy: + fail-fast: false + matrix: + containers: + - alpine + - ubuntu:20.04 + - ubuntu:20.10 + - ubuntu:22.04 + - centos:7 + - centos:8 + - debian:8 + - debian:9 + - debian:10 + - debian:11 steps: - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: "3.8" - - name: Checkout Code - uses: actions/checkout@v3 - with: - submodules: true + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: '8.0.275' + java-package: jdk + architecture: x64 - - name: Download the executable from previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler-aarch64 - path: dist/ + - name: Install Node.JS + uses: actions/setup-node@v2 + with: + # same version as used in tests/containers/nodejs/Dockerfile + node-version: 10.x - # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/action-docker-layer-caching@v0.0.11" - - name: Run gProfiler tests - run: | - sudo python3 -m pip install virtualenv - sudo virtualenv --python python3.8 aarch64-venv - sudo chown -R ${USER} aarch64-venv - sudo chmod -R a+rX aarch64-venv - source aarch64-venv/bin/activate - ./tests/test.sh --executable dist/gprofiler_aarch64 -k test_executable + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Download the executable from previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler_aarch64 + path: dist/ + + - name: Run gProfiler tests + run: | + mv dist/gprofiler_aarch64 dist/gprofiler + chmod +x dist/gprofiler + NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable deploy-executable: if: startsWith(github.ref, 'refs/tags/') @@ -247,9 +263,7 @@ jobs: test-container-x64: if: ${{ !startsWith(github.ref, 'refs/tags/') }} needs: build-container-x64 - runs-on: ubuntu-20.04 # for 3.10.0 python tag - strategy: fail-fast: false # helps detecting flakiness / errors specific to one Python version matrix: @@ -358,14 +372,13 @@ jobs: retention-days: 1 test-container-aarch64: + if: ${{ !startsWith(github.ref, 'refs/tags/') }} needs: build-container-aarch64 - runs-on: - self-hosted - public - ARM64 steps: - - name: Login to DockerHub uses: docker/login-action@v1 with: From f5c86af2d413654021057e9f55260c298aaf27ea Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 24 Nov 2023 23:07:16 +0200 Subject: [PATCH 141/141] fix --- .github/workflows/build-test-deploy.yml | 71 +++++++++++++------------ 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 592c1558d..55af620e6 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -118,6 +118,11 @@ jobs: fetch-depth: 0 submodules: true + - name: Get and verify tag value + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + ./scripts/cmp_tags.sh + - name: Set up requirements run: scripts/setup_runner_requirements.sh @@ -329,47 +334,47 @@ jobs: - public - ARM64 steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true - - name: setup runner requirements - run: ./scripts/setup_runner_requirements.sh + - name: setup runner requirements + run: ./scripts/setup_runner_requirements.sh - - name: Get and verify tag value - run: sudo ./scripts/verify_tag.sh + - name: Get and verify tag value + run: sudo ./scripts/verify_tag.sh - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Download executables from the previous job - uses: actions/download-artifact@v2 - with: - name: gprofiler_aarch64 - path: output/ + - name: Download executables from the previous job + uses: actions/download-artifact@v2 + with: + name: gprofiler_aarch64 + path: output/ - # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs - # more nicely. the tests will then be able to use the built image. - - name: Build gProfiler image - run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 --output type=docker + # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs + # more nicely. the tests will then be able to use the built image. + - name: Build gProfiler image + run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 --output type=docker - - name: Export gProfiler image - run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img + - name: Export gProfiler image + run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img - - name: Upload the image artifact - uses: actions/upload-artifact@v2 - with: - name: gprofiler_aarch64.img - path: output/ - retention-days: 1 + - name: Upload the image artifact + uses: actions/upload-artifact@v2 + with: + name: gprofiler_aarch64.img + path: output/ + retention-days: 1 test-container-aarch64: if: ${{ !startsWith(github.ref, 'refs/tags/') }}