diff --git a/.github/workflows/build_on_push_pr.yml b/.github/workflows/build_on_push_pr.yml new file mode 100644 index 0000000..af15ef1 --- /dev/null +++ b/.github/workflows/build_on_push_pr.yml @@ -0,0 +1,72 @@ +name: Build package on Push or Pull Request + +on: + push + +jobs: + Prepare: + name: 'Prepare' + uses: Ultimaker/embedded-workflows/.github/workflows/prepare_env.yml@main + with: + BUILD_DOCKER_CACHE: true + secrets: inherit + + Shellcheck: + name: 'Test' + needs: Prepare + uses: Ultimaker/embedded-workflows/.github/workflows/shellcheck.yml@main + secrets: inherit + + Flake8: + name: 'Test' + needs: Prepare + uses: Ultimaker/python-quality-control/.github/workflows/flake8.yml@master + with: + PARENT_BRANCH: 'main' + secrets: inherit + + MyPy: + name: 'Test' + needs: Prepare + uses: Ultimaker/python-quality-control/.github/workflows/mypy.yml@master + with: + PARENT_BRANCH: 'main' + secrets: inherit + + PyCodeStyle: + name: 'Test' + needs: Prepare + uses: Ultimaker/python-quality-control/.github/workflows/pycodestyle.yml@master + with: + PARENT_BRANCH: 'main' + secrets: inherit + + Vulture: + name: 'Test' + needs: Prepare + uses: Ultimaker/python-quality-control/.github/workflows/vulture.yml@master + secrets: inherit + + PyTest: + name: 'Test' + needs: Prepare + uses: Ultimaker/python-quality-control/.github/workflows/pytest.yml@master + secrets: inherit + + Build: + name: 'Build' + needs: Prepare + uses: Ultimaker/embedded-workflows/.github/workflows/build.yml@main + with: + RELEASE_VERSION: ${{ needs.Prepare.outputs.RELEASE_VERSION }} + secrets: inherit + + Release_Package: + name: 'Release' + needs: [Prepare, Shellcheck, Build, Flake8, MyPy, PyCodeStyle, Vulture, PyTest] + if: ${{ (success() && needs.Prepare.outputs.RELEASE_REPO != 'none') || + (failure() && needs.Build.result == 'success' && needs.Prepare.outputs.RELEASE_REPO == 'packages-dev') }} + uses: Ultimaker/embedded-workflows/.github/workflows/release_pkg.yml@main + with: + RELEASE_REPO: ${{ needs.Prepare.outputs.RELEASE_REPO }} + secrets: inherit diff --git a/.gitignore b/.gitignore index aba19ad..0fa023f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ __pycache__/ # Build folder _build_armhf/* +# Ignore MyTest data +.pytest_cache/ +pstats/ + # VIM temp files *.swp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index c2378b8..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,46 +0,0 @@ -include: - - project: ultimaker/embedded/prime-jedi - ref: master - file: /gitlab_ci_templates/jedi-gitlab-ci-template.yml - -complexity: - extends: - - .jobs_common - - .build_test_common - stage: test - script: - - ./ci/complexity_analysis.sh - -dead_code: - extends: - - .jobs_common - - .build_test_common - stage: test - script: - - ./ci/dead_code_analysis.sh - -style: - extends: - - .jobs_common - - .build_test_common - stage: test - script: - - git fetch origin master/s-line:master/s-line - - ./ci/style_analysis.sh - -mypy: - extends: - - .jobs_common - - .build_test_common - stage: test - script: - - git fetch origin master/s-line:master/s-line - - ./ci/mypy.sh - -pytest: - extends: - - .jobs_common - - .build_test_common - stage: test - script: - - ./ci/pytest.sh diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..81d94f9 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ci"] + path = ci + url = git@github.com:Ultimaker/python-quality-control.git diff --git a/build.sh b/build.sh index 2a5a82a..61910f2 100755 --- a/build.sh +++ b/build.sh @@ -27,6 +27,7 @@ build() create_debian_package() { make package + cp ./*.deb ../ || true } cleanup() diff --git a/build_for_ultimaker.sh b/build_for_ultimaker.sh index 73f0943..ba52c99 100755 --- a/build_for_ultimaker.sh +++ b/build_for_ultimaker.sh @@ -2,7 +2,6 @@ # # Copyright (C) 2019 Ultimaker B.V. # -# SPDX-License-Identifier: LGPL-3.0+ set -eu @@ -15,11 +14,8 @@ BUILD_DIR_TEMPLATE="_build_${ARCH}" BUILD_DIR="${BUILD_DIR:-${SRC_DIR}/${BUILD_DIR_TEMPLATE}}" run_env_check="yes" -run_linters="yes" -run_tests="yes" - -# Run the make_docker.sh script here, within the context of the build_for_ultimaker.sh script -. ./make_docker.sh "" +run_verification="yes" +action="none" env_check() { @@ -31,25 +27,11 @@ run_build() run_in_docker "./build.sh" "${@}" } -deliver_pkg() -{ - run_in_docker chown -R "$(id -u):$(id -g)" "${DOCKER_WORK_DIR}" - - cp "${BUILD_DIR}/"*".deb" "./" -} - -run_tests() +run_verification() { echo "Testing!" - # These tests should never fail! See .gitlab-ci.yml - ./run_style_analysis.sh || echo "Code Style Analaysis Failed!" - ./run_mypy.sh || echo "MYPY Analysis Failed!" - ./run_pytest.sh || echo "PyTest failed!" -} - -run_linters() -{ - run_shellcheck + # These verifications should never fail! See .gitlab-ci.yml + ./ci/local/run_all.sh } run_shellcheck() @@ -67,12 +49,14 @@ usage() echo "Usage: ${0} [OPTIONS]" echo " -c Skip build environment checks" echo " -h Print usage" - echo " -l Skip code linting" - echo " -t Skip tests" + echo " -s Skip code verification" } -while getopts ":chlst" options; do +while getopts ":a:chls" options; do case "${options}" in + a) + action="${OPTARG}" + ;; c) run_env_check="no" ;; @@ -80,14 +64,8 @@ while getopts ":chlst" options; do usage exit 0 ;; - l) - run_linters="no" - ;; - t) - run_tests="no" - ;; s) - # Ignore for compatibility with other build scripts + run_verification="no" ;; :) echo "Option -${OPTARG} requires an argument." @@ -106,21 +84,40 @@ if ! command -V docker; then exit 1 fi +case "${action}" in + shellcheck) + run_shellcheck + exit 0 + ;; + build) + source ./docker_env/make_docker.sh "" + run_build + exit 0 + ;; + build_docker_cache) + DOCKER_BUILD_ONLY_CACHE="yes" + source ./docker_env/make_docker.sh "" + exit 0 + ;; + none) + ;; + ?) + echo "Invalid action: -${OPTARG}" + exit 1 + ;; +esac + +# Make sure to pass an empty argument to make_docker, else any arguments passed to build_for_ultimaker is passed to make_docker instead! +source ./docker_env/make_docker.sh "" if [ "${run_env_check}" = "yes" ]; then env_check fi -if [ "${run_linters}" = "yes" ]; then - run_linters -fi - run_build "${@}" -if [ "${run_tests}" = "yes" ]; then - run_tests +if [ "${run_verification}" = "yes" ]; then + run_verification fi -deliver_pkg - exit 0 diff --git a/charon_requirements.txt b/charon_requirements.txt index 7fc19a2..2441c82 100644 --- a/charon_requirements.txt +++ b/charon_requirements.txt @@ -1,8 +1,17 @@ -typing -pycodestyle -pytest -pytest-cov +flake8==4.0.1 +flake8-polyfill==1.0.2 +flake8-quotes==3.3.1 +mypy==0.910 +pep8-naming==0.12.1 +pytest==6.2.5 +pytest-cov==3.0.0 +pytest-mock==1.10.4 +pytest-raises==0.11 +pytest-profiling==1.7.0 +pytest-raises==0.11 +vulture==2.3 +pycodestyle==2.8.0 +pylint==2.13.9 coverage lizard -vulture -mypy +typing diff --git a/ci b/ci new file mode 160000 index 0000000..58cd8b5 --- /dev/null +++ b/ci @@ -0,0 +1 @@ +Subproject commit 58cd8b53178ace54427fddd3096703298c875c6d diff --git a/ci/complexity_analysis.sh b/ci/complexity_analysis.sh deleted file mode 100755 index ea0dafc..0000000 --- a/ci/complexity_analysis.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -set -eu - -# TODO set limits, shouldn't expect 100% -lizard -Eduplicate Charon -T cyclomatic_complexity=20 #This value shall not increase, target is <= 10 - -exit 0 diff --git a/ci/dead_code_analysis.sh b/ci/dead_code_analysis.sh deleted file mode 100755 index d85b145..0000000 --- a/ci/dead_code_analysis.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -set -eu - -vulture --min-confidence 100 "Charon" - -exit 0 diff --git a/ci/mypy.sh b/ci/mypy.sh deleted file mode 100755 index cbdba03..0000000 --- a/ci/mypy.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -set -eu - -MYPY_FILES=$(git diff --name-only --diff-filter=d origin/master/s-line | grep -i .py$ | cat) - -if [ -n "${MYPY_FILES}" ] ; then - echo "Testing ${MYPY_FILES}" - for file in ${MYPY_FILES} ; do - echo "Mypying ${file}" - mypy --config-file=mypy.ini --follow-imports=skip --cache-dir=/dev/null "${file}" - done -fi - -exit 0 diff --git a/ci/pytest.sh b/ci/pytest.sh deleted file mode 100755 index 2a6f9d1..0000000 --- a/ci/pytest.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -set -eu - -py3clean . -rm -rf ./.cache -rm -rf ./.pytest_cache -rm -rf ./.mypy_cache -rm -rf ./cov_report -pytest --cov=. --cov-report=html --cov-config=./coverage.ini -py3clean . - -echo "Getting coverage report" - -coverage report - -exit 0 diff --git a/ci/style_analysis.sh b/ci/style_analysis.sh deleted file mode 100755 index f17b5ce..0000000 --- a/ci/style_analysis.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -set -eu - -git diff origin/master/s-line | pycodestyle --config=pycodestyle.ini --diff - -exit 0 diff --git a/docker_env/make_docker.sh b/docker_env/make_docker.sh new file mode 100755 index 0000000..b4a60b9 --- /dev/null +++ b/docker_env/make_docker.sh @@ -0,0 +1,49 @@ +#!/bin/sh +# +# Copyright (C) 2019 Ultimaker B.V. +# + +set -eu + +DOCKER_BUILD_ONLY_CACHE="${DOCKER_BUILD_ONLY_CACHE:-no}" +DOCKER_IMAGE_NAME="${DOCKER_IMAGE_NAME:-libcharon}" +DOCKER_REGISTRY_NAME="ghcr.io/ultimaker/${DOCKER_IMAGE_NAME}" + +echo "Checking for image updates" + +# Creates a new docker driver named "ultimaker" if it doesnt exist yet. +docker buildx create --name ultimaker --driver=docker-container 2> /dev/null || true + +if [ "${DOCKER_BUILD_ONLY_CACHE}" = "yes" ]; then + docker buildx build --builder ultimaker --cache-to "${DOCKER_REGISTRY_NAME}" --cache-from "${DOCKER_REGISTRY_NAME}" -f docker_env/Dockerfile -t "${DOCKER_IMAGE_NAME}" . +else + docker buildx build --builder ultimaker --load --cache-from "${DOCKER_REGISTRY_NAME}" -f docker_env/Dockerfile -t "${DOCKER_IMAGE_NAME}" . + + if ! docker run --rm --privileged "${DOCKER_IMAGE_NAME}" "./buildenv_check.sh"; then + echo "Something is wrong with the build environment, please check your Dockerfile." + docker image rm "${DOCKER_IMAGE_NAME}" + exit 1 + fi +fi; + +DOCKER_WORK_DIR="${WORKDIR:-/build/libcharon}" +PREFIX="/usr" + +run_in_docker() +{ + echo "Running '${*}' in docker." + docker run \ + --rm \ + --privileged \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):${DOCKER_WORK_DIR}" \ + -v "$(pwd)/../:${DOCKER_WORK_DIR}/.." \ + -e "USE_DUMMY_DBUS=true" \ + -e "PYTHONPATH=:../dbus-interface-lib:../libpalantir:../libPalantir:../charon:../libCharon:../libsmeagol:../libSmeagol:../marvin-service/src:../libLogger:../ultiLib/libs:../mqttHandler" \ + -e "PREFIX=${PREFIX}" \ + -e "RELEASE_VERSION=${RELEASE_VERSION:-}" \ + -e "ONLY_CHECK_STAGED=${ONLY_CHECK_STAGED:-}" \ + -w "${DOCKER_WORK_DIR}" \ + "${DOCKER_IMAGE_NAME}" \ + "${@}" +} diff --git a/make_docker.sh b/make_docker.sh deleted file mode 100755 index 1910475..0000000 --- a/make_docker.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2019 Ultimaker B.V. -# -# SPDX-License-Identifier: LGPL-3.0+ - -set -eu - -GIT_HASH="$(git rev-parse HEAD)" - -DOCKER_IMAGE_NAME="${1:-charon_${GIT_HASH}}" - -DOCKER_FILE_CHANGES=$(git diff docker_env/Dockerfile) - -if ! docker inspect --type=image "${DOCKER_IMAGE_NAME}" > /dev/null || [ -n "${DOCKER_FILE_CHANGES}" ]; then - - docker build --rm -f docker_env/Dockerfile -t "${DOCKER_IMAGE_NAME}" . - - if ! docker run --rm --privileged "${DOCKER_IMAGE_NAME}" "./buildenv_check.sh"; then - echo "Something is wrong with the build environment, please check your Dockerfile." - docker image rm "${DOCKER_IMAGE_NAME}" - exit 1 - fi -fi - -DOCKER_WORK_DIR="${WORKDIR:-/build/charon}" -PREFIX="/usr" - -run_in_docker() -{ - echo "Running '${*}' in docker." - docker run \ - --rm \ - --privileged \ - -v "$(pwd):${DOCKER_WORK_DIR}" \ - -v "$(pwd)/../:${DOCKER_WORK_DIR}/.." \ - -e "PREFIX=${PREFIX}" \ - -e "RELEASE_VERSION=${RELEASE_VERSION:-}" \ - -w "${DOCKER_WORK_DIR}" \ - "${DOCKER_IMAGE_NAME}" \ - "${@}" -} diff --git a/run_all_tests.sh b/run_all_tests.sh index b0a4325..148c483 100755 --- a/run_all_tests.sh +++ b/run_all_tests.sh @@ -3,7 +3,7 @@ set -eu # Run the make_docker.sh script here, within the context of the run_all_tests.sh script -. ./make_docker.sh +. ./docker_env/make_docker.sh git fetch diff --git a/run_complexity_analysis.sh b/run_complexity_analysis.sh index 927938f..bb47480 100755 --- a/run_complexity_analysis.sh +++ b/run_complexity_analysis.sh @@ -2,7 +2,7 @@ set -eu -. ./make_docker.sh +. ./docker_env/make_docker.sh run_in_docker "ci/complexity_analysis.sh" || echo "Failed!" diff --git a/run_dead_code_analysis.sh b/run_dead_code_analysis.sh index 3a013dd..6266baf 100755 --- a/run_dead_code_analysis.sh +++ b/run_dead_code_analysis.sh @@ -2,7 +2,7 @@ set -eu -. ./make_docker.sh +. ./docker_env/make_docker.sh run_in_docker "ci/dead_code_analysis.sh" || echo "Failed!" diff --git a/run_mypy.sh b/run_mypy.sh index 25d84c1..30520ba 100755 --- a/run_mypy.sh +++ b/run_mypy.sh @@ -2,7 +2,7 @@ set -eu -. ./make_docker.sh +. ./docker_env/make_docker.sh git fetch diff --git a/run_pytest.sh b/run_pytest.sh index 8c8be68..eb27c81 100755 --- a/run_pytest.sh +++ b/run_pytest.sh @@ -2,7 +2,7 @@ set -eu -. ./make_docker.sh +. ./docker_env/make_docker.sh run_in_docker "ci/pytest.sh" || echo "Failed!" diff --git a/run_style_analysis.sh b/run_style_analysis.sh index 9e8df6d..4bb1771 100755 --- a/run_style_analysis.sh +++ b/run_style_analysis.sh @@ -2,7 +2,7 @@ set -eu -. ./make_docker.sh +. ./docker_env/make_docker.sh git fetch