diff --git a/.gitignore b/.gitignore index 77189f41c..ca2084e62 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ site .venv .DS_Store build/bin/awktest.sh +.venv \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5da5d84e5..761b65f29 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,27 @@ Contributing to MAS Gitops =============================================================================== + +Documentation +------------------------------------------------------------------------------- + + +Versioned documentation is published automatically here: [https://ibm-mas.github.io/gitops/](https://ibm-mas.github.io/gitops/). +Documentation source is located in the `docs` folder. + +To view your local documentation updates before pushing to git, run the following: + +``` +python3.9 -m venv .venv +source .venv/bin/activate +pip install --upgrade pip +pip install mkdocs +pip install mkdocs-redirects +pip install mkdocs-macros-plugin +pip install mkdocs-drawio-file +mkdocs serve +``` + Pre-Commit Hooks ------------------------------------------------------------------------------- diff --git a/README.md b/README.md index f2e44924f..ab087aee6 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,4 @@ Documentation [https://ibm-mas.github.io/gitops/](https://ibm-mas.github.io/gitops/) [https://github.com/ibm-mas/gitops-demo/tree/002](https://github.com/ibm-mas/gitops-demo/tree/002) + diff --git a/build/bin/verify-job-definitions.sh b/build/bin/verify-job-definitions.sh index 3dab8f691..d7ffa230d 100755 --- a/build/bin/verify-job-definitions.sh +++ b/build/bin/verify-job-definitions.sh @@ -18,8 +18,9 @@ Job name accordingly: - The \$_job_config_values constant is defined - The \$_job_version constant is defined - The \$_job_hash constant is defined and has the correct value - - The \$_job_name constant is defined and has the correct value - - The \$_job_name constant is used as the name of the Job + - The \$_job_name constant is defined, has the correct value and is used as the name of the Job + - The \$_job_cleanup_group is constant defined and assigned to the mas.ibm.com/job-cleanup-group Job label + - each template file contains only a single Job definition [PATH]... can be either: - A single directory: the script will check all files under this directory (recursive) @@ -127,7 +128,7 @@ for file in ${files}; do done <<< "$(sed -En 's/.*quay\.io\/ibmmas\/cli:(.*)/\1/p' $file)" - # Experimental: attempt to dynamically detect if we can relax job naming restrictions for this file + # Attempt to dynamically detect if we can relax job naming restrictions for this file # The following awk commands exits 0 if and only if: # - File does not contain a Job resource # Jobs are currently the only resource we use where immutability of the image field is a problem. @@ -219,14 +220,29 @@ for file in ${files}; do problems=${problems}' Missing {{- $_job_name := "..." }}\n' fi - # Check all jobs actually use $_job_name + # Check there is exactly one Job resource defined in the file + awkout=$(awk 'BEGIN { job_count=0; } + /^[[:space:]]*kind:[[:space:]]+Job/ { job_count++ } + END { + if(job_count != 1) { + printf "Exactly 1 Job should be defined in each template file, but %s were found", job_count + exit 1 + } + }' $file \ + ) + rc=$? + if [[ $rc != 0 ]]; then + problems=${problems}' '${awkout}'\n' + fi + + # Check the job actually uses $_job_name awkout=$(awk 'BEGIN { job_count=0; valid_name_count=0; } /^[[:space:]]*kind:[[:space:]]+Job/ { inJob=1; job_count++ } /^---/ { inJob=0 } inJob && /name:[[:space:]]+\{\{[[:space:]]*\$_job_name[[:space:]]*\}\}/ { valid_name_count++ } END { if(valid_name_count!=job_count) { - print "At least one Job does not have name: {{ $_job_name }}" + print "The Job does not have name: {{ $_job_name }}" exit 1 } }' $file \ @@ -235,6 +251,36 @@ for file in ${files}; do if [[ $rc != 0 ]]; then problems=${problems}' '${awkout}'\n' fi + + + + # Check $_job_cleanup_group constant is defined + grep -Eq '^[[:space:]]*\{\{-?[[:space:]]+\$_job_cleanup_group[[:space:]]*:=[^}]+\}' $file + rc=$? + if [[ $rc != 0 ]]; then + problems=${problems}' Missing {{- $_job_cleanup_group := ... }}\n' + fi + + # Check mas.ibm.com/job-cleanup_group: $_job_cleanup_group label is applied to the Job + awkout=$(awk 'BEGIN { state=0; found=0 } + /^---/ { state=0 } + /^[[:space:]]*spec:/ { state=0 } + /^[[:space:]]*kind:[[:space:]]+Job/ { state=1; } + state==1 && /^[[:space:]]*metadata:/ { state=2; } + state==2 && /^[[:space:]]+labels:/ { state=3; } + state==3 && /^[[:space:]]+mas\.ibm\.com\/job-cleanup-group[[:space:]]*:[[:space:]]+\{\{[[:space:]]*\$_job_cleanup_group[[:space:]]*\}\}/ { found=1 } + END { + if(found!=1) { + print "The Job does not have the mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} label" + exit 1 + } + }' $file \ + ) + rc=$? + if [[ $rc != 0 ]]; then + problems=${problems}' '${awkout}'\n' + fi + fi diff --git a/cluster-applications/000-job-cleaner/Chart.yaml b/cluster-applications/000-job-cleaner/Chart.yaml new file mode 100644 index 000000000..b4ba0fa1a --- /dev/null +++ b/cluster-applications/000-job-cleaner/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: job-cleaner +description: A CronJob to delete old versions of Jobs created by ArgoCD +type: application +version: 1.0.0 + +dependencies: +- name: junitreporter + version: 1.0.0 + repository: "file://../../sub-charts/junitreporter/" + condition: junitreporter.devops_mongo_uri != "" \ No newline at end of file diff --git a/cluster-applications/000-job-cleaner/README.md b/cluster-applications/000-job-cleaner/README.md new file mode 100644 index 000000000..1ee944e64 --- /dev/null +++ b/cluster-applications/000-job-cleaner/README.md @@ -0,0 +1,11 @@ +MAS SaaS Job Cleaner +=============================================================================== + +Deploys the `mas-saas-job-cleaner-cron` CronJob, responsible for cleaning up orphaned Job resources in the cluster. It works by grouping Jobs in the cluster according to the `mas.ibm.com/job-cleanup-group` label, then deleting all Jobs from each group except for the one with the latest `creationTimestamp`. + +For safety, the CronJob is assigned a ServiceAccount that can only list and delete Job resources (so it can never delete any other type of resource). Furthermore, the logic ensures that only Job resources with the `mas.ibm.com/job-cleanup-group` label can be deleted. + +The `mas-devops-saas-job-cleaner` command executed by this CronJob is defined in [python-devops](https://github.com/ibm-mas/python-devops/blob/stable/bin/mas-devops-saas-job-cleaner). + + +> In MaS SaaS, Job resources are routinely orphaned (i.e. marked for deletion by ArgoCD) since, when an update is required to an immutable Job field (e.g. its image tag), a new version of the Job resource must be created with a different name. When [auto_delete: false](https://ibm-mas.github.io/gitops/main/accountrootmanifest/#auto_delete) is set, ArgoCD will (by design) not perform this cleanup for us. Over time, Job resources will accumulate and put pressure on the K8S API server. \ No newline at end of file diff --git a/cluster-applications/000-job-cleaner/templates/04-jobcleaner_CronJob.yaml b/cluster-applications/000-job-cleaner/templates/04-jobcleaner_CronJob.yaml new file mode 100644 index 000000000..3f20acdfb --- /dev/null +++ b/cluster-applications/000-job-cleaner/templates/04-jobcleaner_CronJob.yaml @@ -0,0 +1,102 @@ +{{- /* +Use the build/bin/set-cli-image-tag.sh script to update this value across all charts. +*/}} +{{- $_cli_image_tag := "13.17.0" }} + + +{{- $ns := "job-cleaner" }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mas-saas-job-cleaner-role + annotations: + argocd.argoproj.io/sync-wave: "02" +{{- if .Values.custom_labels }} + labels: +{{ .Values.custom_labels | toYaml | indent 4 }} +{{- end }} +rules: + - apiGroups: + - batch + resources: + - jobs + verbs: + - list + - delete + +--- +# Service account that is authorized to read k8s secrets (needed by the job) +kind: ServiceAccount +apiVersion: v1 +metadata: + name: "mas-saas-job-cleaner-sa" + namespace: "{{ $ns }}" + annotations: + argocd.argoproj.io/sync-wave: "02" +{{- if .Values.custom_labels }} + labels: +{{ .Values.custom_labels | toYaml | indent 4 }} +{{- end }} + + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mas-saas-job-cleaner-rolebinding + annotations: + argocd.argoproj.io/sync-wave: "03" +{{- if .Values.custom_labels }} + labels: +{{ .Values.custom_labels | toYaml | indent 4 }} +{{- end }} +subjects: + - kind: ServiceAccount + name: mas-saas-job-cleaner-sa + namespace: {{ $ns }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mas-saas-job-cleaner-role + + + +--- +kind: CronJob +apiVersion: batch/v1 +metadata: + name: "mas-saas-job-cleaner-cron" + namespace: "{{ $ns }}" + annotations: + argocd.argoproj.io/sync-wave: "04" +{{- if .Values.custom_labels }} + labels: +{{ .Values.custom_labels | toYaml | indent 4 }} +{{- end }} +spec: + schedule: '0 0 * * *' + suspend: false + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + metadata: +{{- if .Values.custom_labels }} + labels: +{{ .Values.custom_labels | toYaml | indent 12 }} +{{- end }} + spec: + containers: + - name: "mas-saas-job-cleaner" + image: quay.io/ibmmas/cli:{{ $_cli_image_tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - | + set -e + mas-devops-saas-job-cleaner --label mas.ibm.com/job-cleanup-group --log-level INFO + restartPolicy: OnFailure + serviceAccountName: "mas-saas-job-cleaner-sa" diff --git a/cluster-applications/000-job-cleaner/values.yaml b/cluster-applications/000-job-cleaner/values.yaml new file mode 100644 index 000000000..ed97d539c --- /dev/null +++ b/cluster-applications/000-job-cleaner/values.yaml @@ -0,0 +1 @@ +--- diff --git a/cluster-applications/010-redhat-cert-manager/templates/04-postsync-update-sm_Job.yaml b/cluster-applications/010-redhat-cert-manager/templates/04-postsync-update-sm_Job.yaml index 7ed704f78..4f5617c40 100644 --- a/cluster-applications/010-redhat-cert-manager/templates/04-postsync-update-sm_Job.yaml +++ b/cluster-applications/010-redhat-cert-manager/templates/04-postsync-update-sm_Job.yaml @@ -34,7 +34,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v2" }} +{{- $_job_version := "v3" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -45,6 +45,27 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} + + {{ $ns := "cert-manager-operator"}} {{ $aws_secret := "aws"}} {{ $role_name := "postsync-rhcm-update-sm-r" }} @@ -142,8 +163,9 @@ metadata: namespace: {{ $ns }} annotations: argocd.argoproj.io/sync-wave: "015" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/cluster-applications/020-ibm-dro/templates/08-postsync-update-sm_Job.yaml b/cluster-applications/020-ibm-dro/templates/08-postsync-update-sm_Job.yaml index 11458a358..deea66c78 100644 --- a/cluster-applications/020-ibm-dro/templates/08-postsync-update-sm_Job.yaml +++ b/cluster-applications/020-ibm-dro/templates/08-postsync-update-sm_Job.yaml @@ -26,7 +26,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v2" }} +{{- $_job_version := "v3" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -37,6 +37,26 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} + {{ $ns := .Values.dro_namespace}} {{ $aws_secret := "aws"}} @@ -125,8 +145,9 @@ metadata: namespace: {{ $ns }} annotations: argocd.argoproj.io/sync-wave: "028" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/cluster-applications/060-custom-sa/templates/04-postsync-update-sm_Job.yaml b/cluster-applications/060-custom-sa/templates/04-postsync-update-sm_Job.yaml index bc8bd4471..8b4dd2985 100644 --- a/cluster-applications/060-custom-sa/templates/04-postsync-update-sm_Job.yaml +++ b/cluster-applications/060-custom-sa/templates/04-postsync-update-sm_Job.yaml @@ -38,6 +38,26 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. +NOTE: this is one of those cases; we need a separate cleanup group for each per-sa-key Job. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix $key | sha1sum }} --- apiVersion: batch/v1 @@ -47,8 +67,9 @@ metadata: namespace: {{ $.Values.custom_sa_namespace }} annotations: argocd.argoproj.io/sync-wave: "064" -{{- if $.Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if $.Values.custom_labels }} {{ $.Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: @@ -148,4 +169,4 @@ spec: defaultMode: 420 optional: false backoffLimit: 4 -{{- end }} \ No newline at end of file +{{- end }} diff --git a/cluster-applications/200-cluster-promotion/templates/02-cluster-verify_Job.yaml b/cluster-applications/200-cluster-promotion/templates/02-cluster-verify_Job.yaml index 97a27efe1..b18b430ec 100644 --- a/cluster-applications/200-cluster-promotion/templates/02-cluster-verify_Job.yaml +++ b/cluster-applications/200-cluster-promotion/templates/02-cluster-verify_Job.yaml @@ -24,7 +24,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v2" }} +{{- $_job_version := "v3" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -35,6 +35,25 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} --- @@ -78,8 +97,9 @@ metadata: namespace: mas-syncres annotations: argocd.argoproj.io/sync-wave: "202" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/cluster-applications/200-cluster-promotion/templates/03-cluster-promoter_Job.yaml b/cluster-applications/200-cluster-promotion/templates/03-cluster-promoter_Job.yaml index 6885a5946..2d9b4e91b 100644 --- a/cluster-applications/200-cluster-promotion/templates/03-cluster-promoter_Job.yaml +++ b/cluster-applications/200-cluster-promotion/templates/03-cluster-promoter_Job.yaml @@ -24,7 +24,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v1" }} +{{- $_job_version := "v2" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -35,6 +35,26 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} + --- apiVersion: batch/v1 @@ -44,8 +64,9 @@ metadata: namespace: mas-syncres annotations: argocd.argoproj.io/sync-wave: "205" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/docs/accountrootmanifest.md b/docs/accountrootmanifest.md index 595c4233b..51ded78ac 100644 --- a/docs/accountrootmanifest.md +++ b/docs/accountrootmanifest.md @@ -99,6 +99,9 @@ If `true`, ArgoCD will be permitted to automatically delete ArgoCD Applications If `false`, ArgoCD will never delete a resource automatically. Instead, when ArgoCD deems that a resource should be removed, it will simply flag it as "pending removal" (with a small yellow trashcan icon). In order for an ArgoCD Application or resource on the Target Cluster to actually be deleted, a manual sync with the `Prune` optional enabled must be issued for the ArgoCD Application that owns the resource. +!!! note + **For gitops versions >= 3.11.0 only**, when `auto_delete: false` is set, the [job-cleaner](https://github.com/ibm-mas/gitops/tree/mascore5637/cluster-applications/000-job-cleaner) cluster Application is enabled. This deploys a CronJob that periodically removes specific Job resources according to their `mas.ibm.com/job-cleanup-group` label. This is to prevent the accumulation of old versions of Job resources without the need to manually run `Prune` sync operations. Note that any Job resources created in existing environments by prior versions of Gitops will lack the `mas.ibm.com/job-cleanup-group` label and so will still need to be cleaned up manually + Since the **Cluster** and **Instance** root applications are generated by an Application Set, they behave slightly differently when `auto_delete` is `false`. They will be marked for removal if the `ibm-mas-cluster-base.yaml` or `ibm-mas-instance-base.yaml` files are removed from the {{ config_repo() }}. No indication will be given in the ArgoCD that this has occurred, but ArgoCD will stop synchronizing configuration changes to the Application. The user must manually issue a **delete** operation against the Application for it to be deleted. !!! warning diff --git a/docs/drawio/appstructure.drawio b/docs/drawio/appstructure.drawio index 8a5ce4680..272ad701d 100644 --- a/docs/drawio/appstructure.drawio +++ b/docs/drawio/appstructure.drawio @@ -1,432 +1,450 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/helmcharts.md b/docs/helmcharts.md index df7129440..c209a5251 100644 --- a/docs/helmcharts.md +++ b/docs/helmcharts.md @@ -20,10 +20,11 @@ The following figure shows a tree of ArgoCD applications and Application Sets ge ## {{ cluster_root_chart() }} -**Cluster Root Applications** render the {{ cluster_root_chart() }}, which contains templates to generate ArgoCD Applications for configuring various dependencies shared by MAS instances on {{ target_clusters() }}. +**Cluster Root Applications** render the {{ cluster_root_chart() }}, which contains templates to generate ArgoCD Applications for configuring various dependencies and supporting services shared by MAS instances on {{ target_clusters() }}. | Application | Template | Helm Chart | |-|-|-| +|Job Cleaner | {{ gitops_repo_file_link("root-applications/ibm-mas-cluster-root/templates/000-job-cleaner.yaml", "000-job-cleaner.yaml") }} | {{ gitops_repo_dir_link("cluster-applications/000-job-cleaner", "000-job-cleaner") }} | |IBM Operator Catalog | {{ gitops_repo_file_link("root-applications/ibm-mas-cluster-root/templates/000-ibm-operator-catalog-app.yaml", "000-ibm-operator-catalog-app.yaml") }} | {{ gitops_repo_dir_link("cluster-applications/000-ibm-operator-catalog", "000-ibm-operator-catalog") }} | |Redhat Certificate Manager | {{ gitops_repo_file_link("root-applications/ibm-mas-cluster-root/templates/010-ibm-redhat-cert-manager-app.yaml", "010-ibm-redhat-cert-manager-app.yaml") }} | {{ gitops_repo_dir_link("cluster-applications/010-redhat-cert-manager", "010-redhat-cert-manager") }} | |Data Reporter Operator (DRO) | {{ gitops_repo_file_link("root-applications/ibm-mas-cluster-root/templates/020-ibm-dro-app.yaml", "020-ibm-dro-app.yaml") }} | {{ gitops_repo_dir_link("cluster-applications/020-ibm-dro", "020-ibm-dro") }} | @@ -31,6 +32,7 @@ The following figure shows a tree of ArgoCD applications and Application Sets ge |NFD Operator | {{ gitops_repo_file_link("root-applications/ibm-mas-cluster-root/templates/050-nfd-operator-app.yaml", "050-nfd-operator-app.yaml") }} | {{ gitops_repo_dir_link("cluster-applications/050-nfd-operator", "050-nfd-operator") }} | |Nvidia GPU Operator | {{ gitops_repo_file_link("root-applications/ibm-mas-cluster-root/templates/051-nvidia-gpu-operator-app.yaml", "051-nvidia-gpu-operator-app.yaml") }} | {{ gitops_repo_dir_link("cluster-applications/051-nvidia-gpu-operator", "051-nvidia-gpu-operator") }} | |Custom Cluster Service Accounts | {{ gitops_repo_file_link("root-applications/ibm-mas-cluster-root/templates/060-custom-sa.yaml", "060-custom-sa.yaml") }} | {{ gitops_repo_dir_link("cluster-applications/060-custom-sa", "060-custom-sa") }} | + The {{ cluster_root_chart() }} also installs the {{ instance_root_app_set() }}. This generates a set of **Instance Root Applications** based on the configuration in the {{ config_repo() }}. ## {{ instance_root_chart() }} @@ -52,7 +54,6 @@ The {{ cluster_root_chart() }} also installs the {{ instance_root_app_set() }}. |Analytics Engine powered by Apache Spark (Spark) | {{ gitops_repo_file_link("root-applications/ibm-mas-instance-root/templates/120-ibm-spark-app.yaml", "120-ibm-spark-app.yaml") }} | {{ gitops_repo_dir_link("instance-applications/120-ibm-spark", "120-ibm-spark") }} | |SPSS Modeler | {{ gitops_repo_file_link("root-applications/ibm-mas-instance-root/templates/120-ibm-spss-app.yaml", "120-ibm-spss-app.yaml") }} | {{ gitops_repo_dir_link("instance-applications/120-ibm-spss", "120-ibm-spss") }} | -gitops/root-applications/ibm-mas-instance-root/templates/120-ibm-spss-app.yaml ##### MAS Core Platform This includes the suite, suite configurations and core workspaces diff --git a/docs/orchestration.md b/docs/orchestration.md index d5ef83ea4..60324bbf8 100644 --- a/docs/orchestration.md +++ b/docs/orchestration.md @@ -63,25 +63,19 @@ The set of Custom Resource Healthchecks required by MAS GitOps can be found in t Resource Hooks ------------------------------------------------------------------------------- -Configuration tasks have to be performed at various points during the MAS synchronization procedure. We achieve this via the use of ArgoCD [Resource Hooks](https://argo-cd.readthedocs.io/en/stable/user-guide/resource_hooks/). +Configuration tasks have to be performed at various points during the MAS synchronization procedure. We achieve this via the use of Kubernetes Jobs (and supporting resources) annotated as ArgoCD [Resource Hooks](https://argo-cd.readthedocs.io/en/stable/user-guide/resource_hooks/). -#### PreSync Hooks -Tasks that must be performed **before** an Application begins syncing are defined as `PreSync` hooks. These are used, for example, to verify that cluster CRDs are present before proceeding with an installation (e.g. {{ gitops_repo_file_link("instance-applications/120-ibm-db2u-database/templates/00-presync-await-crd_Job.yaml", "00-presync-await-crd_Job") }}). +### PreSync Hooks +Jobs that must execute **before** an Application begins syncing are defined as `PreSync` hooks. These are used, for example, to verify that cluster CRDs are present before proceeding with an installation (e.g. {{ gitops_repo_file_link("instance-applications/120-ibm-db2u-database/templates/00-presync-await-crd_Job.yaml", "00-presync-await-crd_Job") }}). -### "PostSync" Hooks -Tasks that must be performed **after** an Application finishes syncing (before **before** it can report `Healthy`) are performed by Kubernetes Jobs in the final sync wave of the Application. - -Jobs of this kind typically perform some post-install configuration (e.g. {{ gitops_repo_file_link("instance-applications/120-ibm-db2u-database/templates/05-postsync-setup-db2_Job.yaml", "05-postsync-setup-db2_Job") }}) and/or register some runtime-generated information as a secret in the {{ secrets_vault() }} for use by downstream applications (e.g. {{ gitops_repo_file_link("cluster-applications/020-ibm-dro/templates/08-postsync-update-sm_Job.yaml", "08-postsync-update-sm_Job") }}). - - -!!! info - - You may notice that we do not actually use the `PostSync` ArgoCD annotation on many of these Jobs. This is because the completion status of Jobs annotated as `PostSync` is not taken into account when computing the overall health status of an application. Since the tasks we perform are typically required steps that must be performed before downstream applications in later sync waves are allowed to sync, we instead use "ordinary" Kuberenetes Jobs. Since the health status of "ordinary" Kubernetes Jobs **is** taken into account, subsequent sync waves will not be allowed to start until the Job has completed successfully. - +### PostSync Hooks +Jobs that must be execute **after** an Application finishes syncing are defined as `PostSync` hooks. We make use of `PostSync` hooks only in limited cases where the Job is not performing a critical task that another Application in a later syncwave depends upon. This is because ArgoCD deliberately excludes hooks resources -- including `PostSync` hook Jobs -- from the health evaluation of the Application that owns it, meaning that sibling Applications in later syncwaves will be permitted to begin syncing regardless of the `PostSync` hook's completion status. +Where the task being performed *must* occur before Applications in later sync waves are permitted to begin syncing, we use "ordinary" Kubernetes Jobs (i.e. no `argocd.argoproj.io/hook` annotation) assigned to the final sync wave of the Application. When configured in this manner, the completion status of the Job *is* taken into account when evaluating the health of the Application it belongs to; and so will block synchronization of Applications in later sync waves until it completes successfully. Jobs of this kind typically perform post-install configuration (e.g. {{ gitops_repo_file_link("instance-applications/120-ibm-db2u-database/templates/05-postsync-setup-db2_Job.yaml", "05-postsync-setup-db2_Job") }}) and/or register some runtime-generated information as a secret in the {{ secrets_vault() }} for use by Applications in later sync waves (e.g. {{ gitops_repo_file_link("cluster-applications/020-ibm-dro/templates/08-postsync-update-sm_Job.yaml", "08-postsync-update-sm_Job") }}). ### PostDelete Hooks -Tasks that must be performed to ensure an orderly teardown of resources when configuration files are deleted from the {{ config_repo() }}. For example, Suite Config CRs (e.g. `MongoCfg`) cannot be pruned by ArgoCD since they are assigned the `Suite` as an owner during reconciliation. To work around this, we use PostDelete hooks to issue `oc delete` commands (e.g. {{ gitops_repo_file_link("instance-applications/130-ibm-mas-mongo-config/templates/postdelete-delete-cr.yaml", "postdelete-delete-cr") }}). \ No newline at end of file +Tasks that must be performed to ensure an orderly teardown of resources when configuration files are deleted from the {{ config_repo() }}. For example, Suite Config CRs (e.g. `MongoCfg`) cannot be pruned by ArgoCD since they are assigned the `Suite` as an owner during reconciliation. To work around this, we use PostDelete hooks to issue `oc delete` commands (e.g. {{ gitops_repo_file_link("instance-applications/130-ibm-mas-mongo-config/templates/postdelete-delete-cr.yaml", "postdelete-delete-cr") }}). + diff --git a/instance-applications/010-ibm-sync-jobs/templates/00-aws-docdb-add-user_Job.yaml b/instance-applications/010-ibm-sync-jobs/templates/00-aws-docdb-add-user_Job.yaml index b2124141e..c9f6319c4 100644 --- a/instance-applications/010-ibm-sync-jobs/templates/00-aws-docdb-add-user_Job.yaml +++ b/instance-applications/010-ibm-sync-jobs/templates/00-aws-docdb-add-user_Job.yaml @@ -26,7 +26,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v2" }} +{{- $_job_version := "v3" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -37,6 +37,26 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} + --- @@ -53,8 +73,9 @@ metadata: namespace: mas-{{ .Values.instance_id }}-syncres annotations: argocd.argoproj.io/sync-wave: "00" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/010-ibm-sync-jobs/templates/00-ibm-cp4d-presync.yaml b/instance-applications/010-ibm-sync-jobs/templates/00-ibm-cp4d-presync.yaml index cc8ea0ce3..9cef29442 100644 --- a/instance-applications/010-ibm-sync-jobs/templates/00-ibm-cp4d-presync.yaml +++ b/instance-applications/010-ibm-sync-jobs/templates/00-ibm-cp4d-presync.yaml @@ -38,6 +38,26 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} + --- kind: ServiceAccount @@ -108,8 +128,9 @@ metadata: namespace: mas-{{ .Values.instance_id }}-syncres annotations: argocd.argoproj.io/sync-wave: "003" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/010-ibm-sync-jobs/templates/01-ibm-mas_suite_certs_Job.yaml b/instance-applications/010-ibm-sync-jobs/templates/01-ibm-mas_suite_certs_Job.yaml index 8ea4bc342..acf480945 100644 --- a/instance-applications/010-ibm-sync-jobs/templates/01-ibm-mas_suite_certs_Job.yaml +++ b/instance-applications/010-ibm-sync-jobs/templates/01-ibm-mas_suite_certs_Job.yaml @@ -27,7 +27,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v1" }} +{{- $_job_version := "v2" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -38,6 +38,25 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} --- apiVersion: batch/v1 @@ -47,8 +66,9 @@ metadata: namespace: mas-{{ .Values.instance_id }}-syncres annotations: argocd.argoproj.io/sync-wave: "003" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/010-ibm-sync-jobs/templates/01-ibm-mas_suite_dns_Job.yaml b/instance-applications/010-ibm-sync-jobs/templates/01-ibm-mas_suite_dns_Job.yaml index 8c4182b80..a375ca448 100644 --- a/instance-applications/010-ibm-sync-jobs/templates/01-ibm-mas_suite_dns_Job.yaml +++ b/instance-applications/010-ibm-sync-jobs/templates/01-ibm-mas_suite_dns_Job.yaml @@ -26,7 +26,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v1" }} +{{- $_job_version := "v2" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -37,6 +37,26 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} + --- apiVersion: batch/v1 @@ -46,8 +66,9 @@ metadata: namespace: mas-{{ .Values.instance_id }}-syncres annotations: argocd.argoproj.io/sync-wave: "003" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/100-ibm-sls/templates/07-postsync-update-sm_Job.yaml b/instance-applications/100-ibm-sls/templates/07-postsync-update-sm_Job.yaml index 3eb160202..9ac081968 100644 --- a/instance-applications/100-ibm-sls/templates/07-postsync-update-sm_Job.yaml +++ b/instance-applications/100-ibm-sls/templates/07-postsync-update-sm_Job.yaml @@ -26,7 +26,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v2" }} +{{- $_job_version := "v3" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -37,6 +37,26 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} + {{ $ns := printf "mas-%s-sls" .Values.instance_id}} @@ -155,8 +175,9 @@ metadata: annotations: argocd.argoproj.io/sync-wave: "112" argocd.argoproj.io/sync-options: Prune=true -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/110-ibm-cp4d-operators/templates/04-ibm-cp4d_prereqs_ops.yaml b/instance-applications/110-ibm-cp4d-operators/templates/04-ibm-cp4d_prereqs_ops.yaml index 27871a043..f89be5fb7 100644 --- a/instance-applications/110-ibm-cp4d-operators/templates/04-ibm-cp4d_prereqs_ops.yaml +++ b/instance-applications/110-ibm-cp4d-operators/templates/04-ibm-cp4d_prereqs_ops.yaml @@ -24,7 +24,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v1" }} +{{- $_job_version := "v2" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -35,6 +35,25 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} --- @@ -112,8 +131,9 @@ metadata: namespace: "{{ .Values.cpd_operators_namespace }}" annotations: argocd.argoproj.io/sync-wave: "088" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/110-ibm-cp4d-operators/templates/04-ibm-cp4d_upg_cleanup.yaml b/instance-applications/110-ibm-cp4d-operators/templates/04-ibm-cp4d_upg_cleanup.yaml index 549a04090..dfe046571 100644 --- a/instance-applications/110-ibm-cp4d-operators/templates/04-ibm-cp4d_upg_cleanup.yaml +++ b/instance-applications/110-ibm-cp4d-operators/templates/04-ibm-cp4d_upg_cleanup.yaml @@ -35,7 +35,25 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} --- # this job cleans up existing operators that might cause the cp4d upgrade to fail in the case of an upgrade. @@ -51,8 +69,9 @@ metadata: namespace: "{{ .Values.cpd_operators_namespace }}" annotations: argocd.argoproj.io/sync-wave: "086" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/110-ibm-cp4d/templates/03-ibm-cp4d-mcs_patch_sa.yaml b/instance-applications/110-ibm-cp4d/templates/03-ibm-cp4d-mcs_patch_sa.yaml index eefba2b30..3c44c96a2 100644 --- a/instance-applications/110-ibm-cp4d/templates/03-ibm-cp4d-mcs_patch_sa.yaml +++ b/instance-applications/110-ibm-cp4d/templates/03-ibm-cp4d-mcs_patch_sa.yaml @@ -31,7 +31,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v1" }} +{{- $_job_version := "v2" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -42,7 +42,25 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} --- apiVersion: batch/v1 @@ -52,8 +70,9 @@ metadata: namespace: "{{ .Values.cpd_operators_namespace }}" annotations: argocd.argoproj.io/sync-wave: "084" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/110-ibm-cp4d/templates/07-ibm-cp4d_patch_zenservice.yaml b/instance-applications/110-ibm-cp4d/templates/07-ibm-cp4d_patch_zenservice.yaml index 155f1e2e6..a414db137 100644 --- a/instance-applications/110-ibm-cp4d/templates/07-ibm-cp4d_patch_zenservice.yaml +++ b/instance-applications/110-ibm-cp4d/templates/07-ibm-cp4d_patch_zenservice.yaml @@ -25,7 +25,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v4" }} +{{- $_job_version := "v5" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -37,6 +37,26 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} + --- apiVersion: batch/v1 kind: Job @@ -45,8 +65,9 @@ metadata: namespace: "{{ .Values.cpd_operators_namespace }}" annotations: argocd.argoproj.io/sync-wave: "088" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/110-ibm-cp4d/templates/08-ibm-cp4d-post-verify.yaml b/instance-applications/110-ibm-cp4d/templates/08-ibm-cp4d-post-verify.yaml index f48c86c92..b87049f86 100644 --- a/instance-applications/110-ibm-cp4d/templates/08-ibm-cp4d-post-verify.yaml +++ b/instance-applications/110-ibm-cp4d/templates/08-ibm-cp4d-post-verify.yaml @@ -24,7 +24,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v1" }} +{{- $_job_version := "v2" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -35,7 +35,25 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} --- apiVersion: batch/v1 @@ -45,8 +63,9 @@ metadata: namespace: "{{ .Values.cpd_operators_namespace }}" annotations: argocd.argoproj.io/sync-wave: "089" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/110-ibm-cp4d/templates/09-ibm-cp4d_services_base.yaml b/instance-applications/110-ibm-cp4d/templates/09-ibm-cp4d_services_base.yaml index ebd09344a..9e9f61a73 100644 --- a/instance-applications/110-ibm-cp4d/templates/09-ibm-cp4d_services_base.yaml +++ b/instance-applications/110-ibm-cp4d/templates/09-ibm-cp4d_services_base.yaml @@ -24,7 +24,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v1" }} +{{- $_job_version := "v2" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -35,7 +35,25 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} --- @@ -46,8 +64,9 @@ metadata: namespace: "{{ .Values.cpd_operators_namespace }}" annotations: argocd.argoproj.io/sync-wave: "089" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/120-ibm-db2u-database/templates/07-postsync-setup-db2_Job.yaml b/instance-applications/120-ibm-db2u-database/templates/07-postsync-setup-db2_Job.yaml index 9f51ba88b..08781c13f 100644 --- a/instance-applications/120-ibm-db2u-database/templates/07-postsync-setup-db2_Job.yaml +++ b/instance-applications/120-ibm-db2u-database/templates/07-postsync-setup-db2_Job.yaml @@ -26,7 +26,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v7" }} +{{- $_job_version := "v9" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -37,7 +37,26 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. +NOTE: this is one of those cases; we need a separate cleanup group for each DB2 database + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix .Values.db2_instance_name | sha1sum }} --- kind: Secret @@ -152,8 +171,9 @@ metadata: namespace: "{{ .Values.db2_namespace }}" annotations: argocd.argoproj.io/sync-wave: "129" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: @@ -845,4 +865,4 @@ spec: name: db2u-scripts-facilities-{{ .Values.db2_instance_name }}-configmap {{- end }} backoffLimit: 4 -{{- end }} \ No newline at end of file +{{- end }} diff --git a/instance-applications/120-ibm-db2u-database/templates/10-postsync-setup-hadr.yaml b/instance-applications/120-ibm-db2u-database/templates/10-postsync-setup-hadr.yaml index a95559652..7fba4710f 100644 --- a/instance-applications/120-ibm-db2u-database/templates/10-postsync-setup-hadr.yaml +++ b/instance-applications/120-ibm-db2u-database/templates/10-postsync-setup-hadr.yaml @@ -38,6 +38,27 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. +NOTE: this is one of those cases; we need a separate cleanup group for each DB2 database + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix .Values.db2_instance_name | sha1sum }} + --- kind: ServiceAccount apiVersion: v1 @@ -123,8 +144,9 @@ metadata: namespace: "{{ .Values.db2_namespace }}" annotations: argocd.argoproj.io/sync-wave: "130" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/120-ibm-spark/templates/02-ibm-spark-control-plane.yaml b/instance-applications/120-ibm-spark/templates/02-ibm-spark-control-plane.yaml index 882527960..d696ab390 100644 --- a/instance-applications/120-ibm-spark/templates/02-ibm-spark-control-plane.yaml +++ b/instance-applications/120-ibm-spark/templates/02-ibm-spark-control-plane.yaml @@ -35,6 +35,25 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} --- kind: ServiceAccount @@ -97,8 +116,9 @@ metadata: namespace: "{{ .Values.cpd_operators_namespace }}" annotations: argocd.argoproj.io/sync-wave: "095" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/120-ibm-spss/templates/02-ibm-spss-post-verify.yaml b/instance-applications/120-ibm-spss/templates/02-ibm-spss-post-verify.yaml index 54c6cc4ae..854f50591 100644 --- a/instance-applications/120-ibm-spss/templates/02-ibm-spss-post-verify.yaml +++ b/instance-applications/120-ibm-spss/templates/02-ibm-spss-post-verify.yaml @@ -24,7 +24,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v1" }} +{{- $_job_version := "v2" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -35,7 +35,25 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} --- kind: ServiceAccount @@ -134,8 +152,9 @@ metadata: namespace: "{{ .Values.cpd_operators_namespace }}" annotations: argocd.argoproj.io/sync-wave: "096" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/120-ibm-wsl/templates/02-ibm-wsl-post-verify.yaml b/instance-applications/120-ibm-wsl/templates/02-ibm-wsl-post-verify.yaml index caafcc0b8..a047e02e7 100644 --- a/instance-applications/120-ibm-wsl/templates/02-ibm-wsl-post-verify.yaml +++ b/instance-applications/120-ibm-wsl/templates/02-ibm-wsl-post-verify.yaml @@ -24,7 +24,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v2" }} +{{- $_job_version := "v3" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -36,6 +36,25 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} --- kind: ServiceAccount @@ -117,8 +136,9 @@ metadata: namespace: "{{ .Values.cpd_operators_namespace }}" annotations: argocd.argoproj.io/sync-wave: "094" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/130-ibm-mas-suite/templates/05-postsync-add-label_Job.yaml b/instance-applications/130-ibm-mas-suite/templates/05-postsync-add-label_Job.yaml index 6baf0af96..21993b028 100644 --- a/instance-applications/130-ibm-mas-suite/templates/05-postsync-add-label_Job.yaml +++ b/instance-applications/130-ibm-mas-suite/templates/05-postsync-add-label_Job.yaml @@ -31,7 +31,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v2" }} +{{- $_job_version := "v3" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -42,7 +42,25 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} {{ $job_label := "mas-route-patch" }} --- @@ -140,8 +158,9 @@ metadata: namespace: mas-{{ .Values.instance_id }}-core annotations: argocd.argoproj.io/sync-wave: "142" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: diff --git a/instance-applications/130-ibm-mas-suite/templates/06-postsync-configtool-oidc.yaml b/instance-applications/130-ibm-mas-suite/templates/06-postsync-configtool-oidc.yaml index ce42ac580..65711166e 100644 --- a/instance-applications/130-ibm-mas-suite/templates/06-postsync-configtool-oidc.yaml +++ b/instance-applications/130-ibm-mas-suite/templates/06-postsync-configtool-oidc.yaml @@ -35,7 +35,25 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. + +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }} {{ $ns := printf "mas-%s-core" .Values.instance_id }} {{ $np_name := "postsync-configtool-oidc-np" }} @@ -45,6 +63,7 @@ immutable field of any existing Job resource. {{ $job_label := "postsync-configtool-oidc-job" }} {{ $oauth_admin_secret := printf "%s-credentials-oauth-admin" .Values.instance_id }} + --- # Permit outbound communication by the Job pods # (Needed to communicate with the K8S HTTP API and AWS SM) @@ -126,8 +145,9 @@ metadata: namespace: {{ $ns }} annotations: argocd.argoproj.io/sync-wave: "145" -{{- if .Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: @@ -135,7 +155,7 @@ spec: metadata: labels: app: {{ $job_label }} -{{- if .Values.custom_labels }} +{{- if .Values.custom_labels }} {{ .Values.custom_labels | toYaml | indent 8 }} {{- end }} spec: @@ -205,4 +225,3 @@ spec: defaultMode: 420 optional: false backoffLimit: 4 - diff --git a/instance-applications/510-550-ibm-mas-suite-app-config/templates/700-702-postsync-db2-manage.yaml b/instance-applications/510-550-ibm-mas-suite-app-config/templates/700-702-postsync-db2-manage.yaml index c0d4f654c..132813b1c 100644 --- a/instance-applications/510-550-ibm-mas-suite-app-config/templates/700-702-postsync-db2-manage.yaml +++ b/instance-applications/510-550-ibm-mas-suite-app-config/templates/700-702-postsync-db2-manage.yaml @@ -103,7 +103,7 @@ Increment this value whenever you make a change to an immutable field of the Job E.g. passing in a new environment variable. Included in $_job_hash (see below). */}} -{{- $_job_version := "v2" }} +{{- $_job_version := "v3" }} {{- /* 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag @@ -114,8 +114,27 @@ immutable field of any existing Job resource. {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash) }} +{{- /* +Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource. + +When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label +to identify old Job resources that should be pruned on behalf of ArgoCD. + +Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value +will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp") +Jobs will be automatically deleted. + +$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases +where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators +must be added to $_job_cleanup_group. +NOTE: this is one of those cases; we need a separate cleanup group for each per-db2 Job. + +By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators +are required here. +*/}} +{{- $_job_cleanup_group := cat $_job_name_prefix $db.db2_instance_name | sha1sum }} {{ $db2_namespace := $db.db2_namespace }} @@ -187,15 +206,13 @@ roleRef: apiVersion: batch/v1 kind: Job metadata: - # Suffix the Job name with a hash of all chart values - # This is to ensure that ArgoCD will delete and recreate the job if anything changes in the application config - # The job is idempotent name: {{ $_job_name }} namespace: "{{ $manage_ns }}" annotations: argocd.argoproj.io/sync-wave: "702" -{{- if $.Values.custom_labels }} labels: + mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} +{{- if $.Values.custom_labels }} {{ $.Values.custom_labels | toYaml | indent 4 }} {{- end }} spec: @@ -304,7 +321,6 @@ spec: serviceAccountName: "{{ $sa_name }}" volumes: [] backoffLimit: 4 - {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/root-applications/ibm-mas-cluster-root/templates/000-job-cleaner.yaml b/root-applications/ibm-mas-cluster-root/templates/000-job-cleaner.yaml new file mode 100644 index 000000000..e6b041f88 --- /dev/null +++ b/root-applications/ibm-mas-cluster-root/templates/000-job-cleaner.yaml @@ -0,0 +1,66 @@ +{{- if not .Values.auto_delete }} +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: job-cleaner.{{ .Values.cluster.id }} + namespace: {{ .Values.argo.namespace }} + labels: + environment: '{{ .Values.account.id }}' + region: '{{ .Values.region.id }}' + cluster: '{{ .Values.cluster.id }}' + annotations: + argocd.argoproj.io/sync-wave: "000" + healthCheckTimeout: "1800" + {{- if and .Values.notifications .Values.notifications.slack_channel_id }} + notifications.argoproj.io/subscribe.on-sync-failed.workspace1: {{ .Values.notifications.slack_channel_id }} + notifications.argoproj.io/subscribe.on-sync-succeeded.workspace1: {{ .Values.notifications.slack_channel_id }} + {{- end }} + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + ignoreDifferences: + - group: '*' + kind: ServiceAccount + jsonPointers: + - /imagePullSecrets + project: "{{ .Values.argo.projects.apps }}" + destination: + server: {{ .Values.cluster.url }} + namespace: job-cleaner + source: + repoURL: "{{ .Values.source.repo_url }}" + path: cluster-applications/000-job-cleaner + targetRevision: "{{ .Values.source.revision }}" + plugin: + name: {{ .Values.avp.name }} + env: + - name: {{ .Values.avp.values_varname }} + value: | + junitreporter: + reporter_name: "job-cleaner" + cluster_id: "{{ .Values.cluster.id }}" + devops_mongo_uri: "{{ .Values.devops.mongo_uri }}" + devops_build_number: "{{ .Values.devops.build_number }}" + gitops_version: "{{ .Values.source.revision }}" + {{- if .Values.custom_labels }} + custom_labels: {{ .Values.custom_labels | toYaml | nindent 14 }} + {{- end }} + - name: ARGOCD_APP_NAME + value: jobcleanerapp + {{- if not (empty .Values.avp.secret) }} + - name: AVP_SECRET + value: {{ .Values.avp.secret }} + {{- end }} + syncPolicy: + automated: + {{- if .Values.auto_delete }} + prune: true + {{- end }} + selfHeal: true + retry: + limit: 20 + syncOptions: + - CreateNamespace=true + - RespectIgnoreDifferences=true +{{- end }} \ No newline at end of file