From c340f4063f48425469e2ba958ed017f33d5db2cc Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Mon, 5 Jul 2021 17:01:03 -0400 Subject: [PATCH 1/4] Pipeline changes for new cluster + RJ resources --- ci/dockerfiles/mjackets-build/Dockerfile | 12 +- ci/pipelines/docker-build/pipeline.yml | 115 ++++++++++++------- ci/pipelines/mjackets-build/pipeline.yml | 138 ++++++++++++----------- ci/scripts/documentation_coverage.sh | 0 4 files changed, 151 insertions(+), 114 deletions(-) delete mode 100755 ci/scripts/documentation_coverage.sh diff --git a/ci/dockerfiles/mjackets-build/Dockerfile b/ci/dockerfiles/mjackets-build/Dockerfile index d1a510e..1bbd9a2 100644 --- a/ci/dockerfiles/mjackets-build/Dockerfile +++ b/ci/dockerfiles/mjackets-build/Dockerfile @@ -1,5 +1,7 @@ -FROM ubuntu:20.04 -LABEL maintainer="RoboJackets commit-sha/value + path: sh + task: get-commit-sha + - load_var: commit + file: commit-sha/value + format: trim + reveal: true + - get: ubuntu + params: + format: oci - do: - - inputs: - - mjackets-api + - inputs: [] put: build-check - config: image_resource: @@ -17,31 +40,32 @@ jobs: type: registry-image inputs: - name: mjackets-api + - name: ubuntu outputs: - name: image params: CONTEXT: mjackets-api/ci/dockerfiles/mjackets-build + IMAGE_ARG_base_image: ubuntu/image.tar platform: linux run: path: build + caches: + - path: cache on_abort: inputs: - build-check - - mjackets-api params: conclusion: cancelled put: build-check on_error: inputs: - build-check - - mjackets-api params: conclusion: action_required put: build-check on_failure: inputs: - build-check - - mjackets-api params: conclusion: failure summary: Review the output within Concourse. @@ -54,34 +78,29 @@ jobs: steps: - inputs: - build-check - - mjackets-api params: conclusion: success summary: The OCI image was built successfully. title: Image built put: build-check - - inputs: - - mjackets-api + - inputs: [] put: push-check - inputs: detect on_abort: inputs: - push-check - - mjackets-api params: conclusion: cancelled put: push-check on_error: inputs: - push-check - - mjackets-api params: conclusion: action_required put: push-check on_failure: inputs: - push-check - - mjackets-api params: conclusion: failure summary: Review the output within Concourse. @@ -90,7 +109,6 @@ jobs: on_success: inputs: - push-check - - mjackets-api params: conclusion: success summary: The image was successfully uploaded to the registry. @@ -103,63 +121,78 @@ jobs: resource_types: - name: github-check source: - password: ((docker-robojackets-org.password)) - repository: docker.robojackets.org/robojackets/concourse-github-check-resource - username: ((docker-robojackets-org.username)) + aws_access_key_id: ((aws/sts/ecr.access_key)) + aws_region: us-east-1 + aws_secret_access_key: ((aws/sts/ecr.secret_key)) + aws_session_token: ((aws/sts/ecr.security_token)) + repository: robojackets/concourse-github-check-resource type: registry-image + defaults: + commit: ((.:commit)) + repository_url: https://github.com/RoboJackets/mjackets-api + token: (("github.com"/token.token)) + tags: + - resources - name: github-webhook source: - password: ((docker-robojackets-org.password)) - repository: docker.robojackets.org/robojackets/concourse-github-webhook-resource - username: ((docker-robojackets-org.username)) + aws_access_key_id: ((aws/sts/ecr.access_key)) + aws_region: us-east-1 + aws_secret_access_key: ((aws/sts/ecr.secret_key)) + aws_session_token: ((aws/sts/ecr.security_token)) + repository: robojackets/concourse-github-webhook-resource type: registry-image + tags: + - resources resources: -- icon: check-bold +- icon: check name: build-check public: true source: check_name: OCI Build - pull_request: pull-request - repository: mjackets-api resource_name: build-check - token: ((github-com/token.token)) type: github-check -- icon: github - name: concourse-pipeline-library - public: true - source: - branch: main - password: ((github-com/token.token)) - uri: https://github.com/RoboJackets/concourse-pipeline-library - username: x-access-token - type: git + tags: + - resources - icon: github name: mjackets-api public: true source: branch: master - password: ((github-com/token.token)) + password: (("github.com"/token.token)) paths: - ci/dockerfiles/mjackets-build/Dockerfile uri: https://github.com/RoboJackets/mjackets-api username: x-access-token type: git webhook_token: ((webhook-token)) + tags: + - resources - icon: docker name: mjackets-build-image public: true source: - password: ((hub-docker-com.password)) + password: ((kv/"hub.docker.com".password)) repository: robojackets/mjackets-dev - username: ((hub-docker-com.username)) + username: ((kv/"hub.docker.com".username)) type: registry-image -- icon: check-bold + tags: + - resources +- icon: check name: push-check public: true source: check_name: Push to Registry - pull_request: pull-request - repository: mjackets-api resource_name: push-check - token: ((github-com/token.token)) - type: github-check \ No newline at end of file + type: github-check + tags: + - resources +- check_every: 1h0m0s + icon: ubuntu + name: ubuntu + public: true + source: + repository: ubuntu + tag: latest + tags: + - resources + type: registry-image diff --git a/ci/pipelines/mjackets-build/pipeline.yml b/ci/pipelines/mjackets-build/pipeline.yml index 62685d5..49aefee 100644 --- a/ci/pipelines/mjackets-build/pipeline.yml +++ b/ci/pipelines/mjackets-build/pipeline.yml @@ -2,60 +2,67 @@ resource_types: - name: github-check source: - password: ((docker-robojackets-org.password)) - repository: docker.robojackets.org/robojackets/concourse-github-check-resource - username: ((docker-robojackets-org.username)) + aws_access_key_id: ((aws/sts/ecr.access_key)) + aws_region: us-east-1 + aws_secret_access_key: ((aws/sts/ecr.secret_key)) + aws_session_token: ((aws/sts/ecr.security_token)) + repository: robojackets/concourse-github-check-resource type: registry-image + defaults: + commit: ((.:commit)) + repository_url: https://github.com/RoboJackets/mjackets-api + token: (("github.com"/token.token)) + tags: + - resources - name: pull-request source: repository: teliaoss/github-pr-resource - type: docker-image -- name: sonar-runner - source: - repository: cathive/concourse-sonarqube-resource - tag: latest - type: docker-image + type: registry-image + tags: + - resources resources: -- icon: check-bold +- icon: check name: build-check public: true source: check_name: mJackets Sanity Build - pull_request: pull-request - repository: mjackets-api resource_name: build-check - token: ((github-com/token.token)) type: github-check -- icon: check-bold + tags: + - resources +- icon: check name: documentation-check public: true source: - check_name: API Documenation Coverage - pull_request: pull-request - repository: mjackets-api + check_name: API Documentation Coverage resource_name: documentation-check - token: ((github-com/token.token)) type: github-check + tags: + - resources - icon: github name: mjackets-api public: true source: branch: master - password: ((github-com/token.token)) + password: (("github.com"/token)) uri: https://github.com/RoboJackets/mjackets-api username: x-access-token type: git webhook_token: ((webhook-token)) + tags: + - resources - icon: source-pull name: pull-request public: true source: - access_token: ((github-com/token.token)) + access_token: (("github.com"/token)) repository: RoboJackets/mjackets-api base_branch: master type: pull-request + tags: + - resources jobs: - name: build-master @@ -63,26 +70,46 @@ jobs: - get: mjackets-api trigger: true - - inputs: - - mjackets-api + - config: + image_resource: + source: + repository: alpine/git + type: registry-image + inputs: + - name: mjackets-api + outputs: + - name: commit-sha + platform: linux + run: + args: + - -e + - -x + - -c + - git -C mjackets-api rev-parse HEAD > commit-sha/value + path: sh + task: get-commit-sha + - load_var: commit + file: commit-sha/value + format: trim + reveal: true + + - inputs: [] put: build-check - config: image_resource: source: repository: robojackets/mjackets-dev - type: docker-image + type: registry-image inputs: - name: mjackets-api params: - SONAR_TOKEN: ((sonarcloud-io)) + SONAR_TOKEN: ((kv/"sonarcloud.io")) platform: linux run: args: - -exc - | - set -e - ls cd mjackets-api cmake configure . -D BOARD=MTRAIN build-wrapper-linux-x86-64 --out-dir bw-output make @@ -98,31 +125,27 @@ jobs: on_abort: inputs: - build-check - - mjackets-api params: conclusion: cancelled put: build-check on_error: inputs: - build-check - - mjackets-api params: conclusion: action_required put: build-check on_failure: inputs: - build-check - - mjackets-api params: conclusion: failure summary: Review the output within Concourse. title: Sanity build failed put: build-check task: build - + - inputs: - build-check - - mjackets-api params: conclusion: success summary: mJackets sanity build compiled sucessfully. @@ -134,15 +157,14 @@ jobs: - get: mjackets-api trigger: true - - inputs: - - mjackets-api + - inputs: [] put: documentation-check - + - config: image_resource: source: repository: robojackets/mjackets-dev - type: docker-image + type: registry-image inputs: - name: mjackets-api platform: linux @@ -150,8 +172,7 @@ jobs: args: - -exc - | - set -e - cd mjackets-api/docs + cd mjackets-api/docs cmake configure . make python ../utilities/doxy-coverage/doxy-coverage.py docs_build/doxygen_docs/xml/ --threshold 100 @@ -159,37 +180,33 @@ jobs: on_abort: inputs: - documentation-check - - mjackets-api params: conclusion: cancelled put: documentation-check on_error: inputs: - documentation-check - - mjackets-api params: conclusion: action_required put: documentation-check on_failure: inputs: - documentation-check - - mjackets-api params: conclusion: failure summary: Review the output within Concourse. - title: Insuffient API Documenation Coverage + title: Insufficent API Documentation Coverage put: documentation-check task: documentation-coverage - inputs: - documentation-check - - mjackets-api params: conclusion: success summary: mJackets API has 100% documentation coverage. title: 100% API Documentation Coverage put: documentation-check - + - name: build-pull-request plan: @@ -200,28 +217,24 @@ jobs: trigger: true version: every - - inputs: - - pull-request + - inputs: [] put: build-check - config: image_resource: source: repository: robojackets/mjackets-dev - type: docker-image + type: registry-image inputs: - name: pull-request params: - SONAR_TOKEN: ((sonarcloud-io)) + SONAR_TOKEN: ((kv/"sonarcloud.io")) platform: linux run: args: - -exc - | - set -e - apt install jq -y - cd pull-request - PULL_REQUEST_KEY=`jq '.pr' .git/resource/version.json` + PULL_REQUEST_KEY=`cat .git/resource/pr` PULL_REQUEST_KEY="${PULL_REQUEST_KEY%\"}" PULL_REQUEST_KEY="${PULL_REQUEST_KEY#\"}" git submodule init && git submodule update @@ -237,24 +250,22 @@ jobs: -Dsonar.scm.disabled=false \ -Dsonar.pullrequest.key=$PULL_REQUEST_KEY path: sh + dir: pull-request on_abort: inputs: - build-check - - pull-request params: conclusion: cancelled put: build-check on_error: inputs: - build-check - - pull-request params: conclusion: action_required put: build-check on_failure: inputs: - build-check - - pull-request params: conclusion: failure summary: Review the output within Concourse. @@ -264,7 +275,6 @@ jobs: - inputs: - build-check - - pull-request params: conclusion: success summary: mJackets sanity build compiled sucessfully. @@ -280,15 +290,14 @@ jobs: trigger: true version: every - - inputs: - - pull-request + - inputs: [] put: documentation-check - config: image_resource: source: repository: robojackets/mjackets-dev - type: docker-image + type: registry-image inputs: - name: pull-request platform: linux @@ -296,44 +305,37 @@ jobs: args: - -exc - | - set -e - ls - cd pull-request/docs cmake configure . make python ../utilities/doxy-coverage/doxy-coverage.py docs_build/doxygen_docs/xml/ --threshold 100 path: sh + dir: pull-request/docs on_abort: inputs: - documentation-check - - pull-request params: conclusion: cancelled put: documentation-check on_error: inputs: - documentation-check - - pull-request params: conclusion: action_required put: documentation-check on_failure: inputs: - documentation-check - - pull-request params: conclusion: failure summary: Review the output within Concourse. - title: Insuffient API Documentation Coverage + title: Insufficent API Documentation Coverage put: documentation-check task: documentation-coverage - inputs: - documentation-check - - pull-request params: conclusion: success summary: mJackets API has 100% documentation coverage. - title: 100% Documenation Coverage + title: 100% Documentation Coverage put: documentation-check - diff --git a/ci/scripts/documentation_coverage.sh b/ci/scripts/documentation_coverage.sh deleted file mode 100755 index e69de29..0000000 From 806ee6b19f09168ba0146af2d2b9ffa9ee4088ae Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Mon, 5 Jul 2021 17:07:34 -0400 Subject: [PATCH 2/4] specify ubuntu version to match dockerfile --- ci/dockerfiles/mjackets-build/Dockerfile | 2 ++ ci/pipelines/docker-build/pipeline.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/dockerfiles/mjackets-build/Dockerfile b/ci/dockerfiles/mjackets-build/Dockerfile index 1bbd9a2..cef5018 100644 --- a/ci/dockerfiles/mjackets-build/Dockerfile +++ b/ci/dockerfiles/mjackets-build/Dockerfile @@ -1,3 +1,5 @@ +# This value is overridden during Concourse jobs to use a local copy of the image. +# If you are changing the base image here, you should also change it in the Concourse pipeline configuration. ARG base_image=ubuntu:20.04 FROM ${base_image} diff --git a/ci/pipelines/docker-build/pipeline.yml b/ci/pipelines/docker-build/pipeline.yml index c8bd1a4..e1fd701 100644 --- a/ci/pipelines/docker-build/pipeline.yml +++ b/ci/pipelines/docker-build/pipeline.yml @@ -192,7 +192,7 @@ resources: public: true source: repository: ubuntu - tag: latest + tag: "20.04" tags: - resources type: registry-image From 472ad313a6782477338d0b27633bbc53fe34af35 Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Sun, 1 Aug 2021 17:22:25 -0400 Subject: [PATCH 3/4] Removing custom pipeline for Docker image build --- ci/pipelines/docker-build/pipeline.yml | 198 ------------------------- 1 file changed, 198 deletions(-) delete mode 100644 ci/pipelines/docker-build/pipeline.yml diff --git a/ci/pipelines/docker-build/pipeline.yml b/ci/pipelines/docker-build/pipeline.yml deleted file mode 100644 index e1fd701..0000000 --- a/ci/pipelines/docker-build/pipeline.yml +++ /dev/null @@ -1,198 +0,0 @@ -jobs: -- name: build-master - plan: - - in_parallel: - - do: - - get: mjackets-api - trigger: true - - config: - image_resource: - source: - repository: alpine/git - type: registry-image - inputs: - - name: mjackets-api - outputs: - - name: commit-sha - platform: linux - run: - args: - - -e - - -x - - -c - - git -C mjackets-api rev-parse HEAD > commit-sha/value - path: sh - task: get-commit-sha - - load_var: commit - file: commit-sha/value - format: trim - reveal: true - - get: ubuntu - params: - format: oci - - do: - - inputs: [] - put: build-check - - config: - image_resource: - source: - repository: vito/oci-build-task - type: registry-image - inputs: - - name: mjackets-api - - name: ubuntu - outputs: - - name: image - params: - CONTEXT: mjackets-api/ci/dockerfiles/mjackets-build - IMAGE_ARG_base_image: ubuntu/image.tar - platform: linux - run: - path: build - caches: - - path: cache - on_abort: - inputs: - - build-check - params: - conclusion: cancelled - put: build-check - on_error: - inputs: - - build-check - params: - conclusion: action_required - put: build-check - on_failure: - inputs: - - build-check - params: - conclusion: failure - summary: Review the output within Concourse. - title: Image failed to build - put: build-check - privileged: true - task: build-task-image - timeout: 30m - - in_parallel: - steps: - - inputs: - - build-check - params: - conclusion: success - summary: The OCI image was built successfully. - title: Image built - put: build-check - - inputs: [] - put: push-check - - inputs: detect - on_abort: - inputs: - - push-check - params: - conclusion: cancelled - put: push-check - on_error: - inputs: - - push-check - params: - conclusion: action_required - put: push-check - on_failure: - inputs: - - push-check - params: - conclusion: failure - summary: Review the output within Concourse. - title: Image failed to push - put: push-check - on_success: - inputs: - - push-check - params: - conclusion: success - summary: The image was successfully uploaded to the registry. - title: Image pushed to registry - put: push-check - params: - image: image/image.tar - put: mjackets-build-image - public: true -resource_types: -- name: github-check - source: - aws_access_key_id: ((aws/sts/ecr.access_key)) - aws_region: us-east-1 - aws_secret_access_key: ((aws/sts/ecr.secret_key)) - aws_session_token: ((aws/sts/ecr.security_token)) - repository: robojackets/concourse-github-check-resource - type: registry-image - defaults: - commit: ((.:commit)) - repository_url: https://github.com/RoboJackets/mjackets-api - token: (("github.com"/token.token)) - tags: - - resources -- name: github-webhook - source: - aws_access_key_id: ((aws/sts/ecr.access_key)) - aws_region: us-east-1 - aws_secret_access_key: ((aws/sts/ecr.secret_key)) - aws_session_token: ((aws/sts/ecr.security_token)) - repository: robojackets/concourse-github-webhook-resource - type: registry-image - tags: - - resources -resources: -- icon: check - name: build-check - public: true - source: - check_name: OCI Build - resource_name: build-check - type: github-check - tags: - - resources -- icon: github - name: mjackets-api - public: true - source: - branch: master - password: (("github.com"/token.token)) - paths: - - ci/dockerfiles/mjackets-build/Dockerfile - uri: https://github.com/RoboJackets/mjackets-api - username: x-access-token - type: git - webhook_token: ((webhook-token)) - tags: - - resources -- icon: docker - name: mjackets-build-image - public: true - source: - password: ((kv/"hub.docker.com".password)) - repository: robojackets/mjackets-dev - username: ((kv/"hub.docker.com".username)) - type: registry-image - tags: - - resources -- icon: check - name: push-check - public: true - source: - check_name: Push to Registry - resource_name: push-check - type: github-check - tags: - - resources -- check_every: 1h0m0s - icon: ubuntu - name: ubuntu - public: true - source: - repository: ubuntu - tag: "20.04" - tags: - - resources - type: registry-image From a46ba05e022ad8c5761c0e0c69624eb8bf901791 Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Wed, 4 Aug 2021 20:21:55 -0400 Subject: [PATCH 4/4] Fix several issues --- ci/pipelines/mjackets-build/pipeline.yml | 50 +++++++++++++++++++++--- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/ci/pipelines/mjackets-build/pipeline.yml b/ci/pipelines/mjackets-build/pipeline.yml index 49aefee..9dfd2aa 100644 --- a/ci/pipelines/mjackets-build/pipeline.yml +++ b/ci/pipelines/mjackets-build/pipeline.yml @@ -45,7 +45,7 @@ resources: public: true source: branch: master - password: (("github.com"/token)) + password: (("github.com"/token.token)) uri: https://github.com/RoboJackets/mjackets-api username: x-access-token type: git @@ -57,7 +57,7 @@ resources: name: pull-request public: true source: - access_token: (("github.com"/token)) + access_token: (("github.com"/token.token)) repository: RoboJackets/mjackets-api base_branch: master type: pull-request @@ -88,6 +88,7 @@ jobs: - git -C mjackets-api rev-parse HEAD > commit-sha/value path: sh task: get-commit-sha + - load_var: commit file: commit-sha/value format: trim @@ -157,6 +158,30 @@ jobs: - get: mjackets-api trigger: true + - config: + image_resource: + source: + repository: alpine/git + type: registry-image + inputs: + - name: mjackets-api + outputs: + - name: commit-sha + platform: linux + run: + args: + - -e + - -x + - -c + - git -C mjackets-api rev-parse HEAD > commit-sha/value + path: sh + task: get-commit-sha + + - load_var: commit + file: commit-sha/value + format: trim + reveal: true + - inputs: [] put: documentation-check @@ -214,9 +239,20 @@ jobs: params: integration_tool: checkout list_changed_files: true + submodules: true trigger: true version: every + - load_var: commit + file: pull-request/.git/resource/head_sha + format: trim + reveal: true + + - load_var: pull_request_key + file: pull-request/.git/resource/pr + format: trim + reveal: true + - inputs: [] put: build-check @@ -229,15 +265,12 @@ jobs: - name: pull-request params: SONAR_TOKEN: ((kv/"sonarcloud.io")) + PULL_REQUEST_KEY: ((.:pull_request_key)) platform: linux run: args: - -exc - | - PULL_REQUEST_KEY=`cat .git/resource/pr` - PULL_REQUEST_KEY="${PULL_REQUEST_KEY%\"}" - PULL_REQUEST_KEY="${PULL_REQUEST_KEY#\"}" - git submodule init && git submodule update cmake configure . -D BOARD=MTRAIN build-wrapper-linux-x86-64 --out-dir bw-output make sonar-scanner \ @@ -290,6 +323,11 @@ jobs: trigger: true version: every + - load_var: commit + file: pull-request/.git/resource/head_sha + format: trim + reveal: true + - inputs: [] put: documentation-check