[minor] Automated Job cleanup when auto_delete: false is set

.gitignore

@@ -7,3 +7,4 @@ site
 .venv
 .DS_Store
 build/bin/awktest.sh
+.venv

CONTRIBUTING.md

@@ -1,6 +1,27 @@
 Contributing to MAS Gitops
 ===============================================================================
 
+
+Documentation
+-------------------------------------------------------------------------------
+
+
+Versioned documentation is published automatically here: [https://ibm-mas.github.io/gitops/](https://ibm-mas.github.io/gitops/).
+Documentation source is located in the `docs` folder.
+
+To view your local documentation updates before pushing to git, run the following:
+
+```
+python3.9 -m venv .venv
+source .venv/bin/activate
+pip install --upgrade pip
+pip install mkdocs
+pip install mkdocs-redirects
+pip install mkdocs-macros-plugin
+pip install mkdocs-drawio-file
+mkdocs serve
+```
+
 Pre-Commit Hooks
 -------------------------------------------------------------------------------
 

README.md

@@ -8,3 +8,4 @@ Documentation
 [https://ibm-mas.github.io/gitops/](https://ibm-mas.github.io/gitops/)
 
 [https://github.com/ibm-mas/gitops-demo/tree/002](https://github.com/ibm-mas/gitops-demo/tree/002)
+

build/bin/verify-job-definitions.sh

@@ -18,8 +18,9 @@ Job name accordingly:
     - The \$_job_config_values constant is defined
     - The \$_job_version constant is defined
     - The \$_job_hash constant is defined and has the correct value
-    - The \$_job_name constant is defined and has the correct value
-    - The \$_job_name constant is used as the name of the Job
+    - The \$_job_name constant is defined, has the correct value and is used as the name of the Job
+    - The \$_job_cleanup_group is constant defined and assigned to the mas.ibm.com/job-cleanup-group Job label
+    - each template file contains only a single Job definition
 
 [PATH]... can be either:
     - A single directory: the script will check all files under this directory (recursive)
@@ -127,7 +128,7 @@ for file in ${files}; do
     done <<< "$(sed -En 's/.*quay\.io\/ibmmas\/cli:(.*)/\1/p' $file)"
 
 
-    # Experimental: attempt to dynamically detect if we can relax job naming restrictions for this file
+    # Attempt to dynamically detect if we can relax job naming restrictions for this file
     # The following awk commands exits 0 if and only if:
     #   - File does not contain a Job resource 
     #       Jobs are currently the only resource we use where immutability of the image field is a problem.
@@ -219,14 +220,29 @@ for file in ${files}; do
             problems=${problems}'    Missing {{- $_job_name := "..." }}\n'
         fi
 
-        # Check all jobs actually use $_job_name
+        # Check there is exactly one Job resource defined in the file
+        awkout=$(awk 'BEGIN { job_count=0; }
+            /^[[:space:]]*kind:[[:space:]]+Job/ { job_count++ }
+            END {
+                if(job_count != 1) {
+                    printf "Exactly 1 Job should be defined in each template file, but %s were found", job_count
+                    exit 1
+                }
+            }' $file \
+        )
+        rc=$?
+        if [[ $rc != 0 ]]; then
+            problems=${problems}'    '${awkout}'\n'
+        fi
+
+        # Check the job actually uses $_job_name
         awkout=$(awk 'BEGIN { job_count=0; valid_name_count=0; }
             /^[[:space:]]*kind:[[:space:]]+Job/ { inJob=1; job_count++ }
             /^---/ { inJob=0 }
             inJob && /name:[[:space:]]+\{\{[[:space:]]*\$_job_name[[:space:]]*\}\}/ { valid_name_count++ }
             END {
                 if(valid_name_count!=job_count) {
-                    print "At least one Job does not have name: {{ $_job_name }}"
+                    print "The Job does not have name: {{ $_job_name }}"
                     exit 1
                 }
             }' $file \
@@ -235,6 +251,36 @@ for file in ${files}; do
         if [[ $rc != 0 ]]; then
             problems=${problems}'    '${awkout}'\n'
         fi
+
+
+
+        # Check $_job_cleanup_group constant is defined
+        grep -Eq '^[[:space:]]*\{\{-?[[:space:]]+\$_job_cleanup_group[[:space:]]*:=[^}]+\}' $file
+        rc=$?
+        if [[ $rc != 0 ]]; then
+            problems=${problems}'    Missing {{- $_job_cleanup_group := ... }}\n'
+        fi
+
+        # Check mas.ibm.com/job-cleanup_group: $_job_cleanup_group label is applied to the Job
+        awkout=$(awk 'BEGIN { state=0; found=0 }
+            /^---/ { state=0 }
+            /^[[:space:]]*spec:/ { state=0 }
+            /^[[:space:]]*kind:[[:space:]]+Job/ { state=1; }
+            state==1 && /^[[:space:]]*metadata:/ { state=2; }
+            state==2 && /^[[:space:]]+labels:/ { state=3; }
+            state==3 && /^[[:space:]]+mas\.ibm\.com\/job-cleanup-group[[:space:]]*:[[:space:]]+\{\{[[:space:]]*\$_job_cleanup_group[[:space:]]*\}\}/ { found=1 }
+            END {
+                if(found!=1) {
+                    print "The Job does not have the mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }} label"
+                    exit 1
+                }
+            }' $file \
+        )
+        rc=$?
+        if [[ $rc != 0 ]]; then
+            problems=${problems}'    '${awkout}'\n'
+        fi
+
     fi
 
 

cluster-applications/000-job-cleaner/Chart.yaml

@@ -0,0 +1,11 @@
+apiVersion: v2
+name: job-cleaner
+description: A CronJob to delete old versions of Jobs created by ArgoCD
+type: application
+version: 1.0.0
+
+dependencies:
+- name: junitreporter
+  version: 1.0.0
+  repository: "file://../../sub-charts/junitreporter/"
+  condition: junitreporter.devops_mongo_uri != ""

cluster-applications/000-job-cleaner/README.md

@@ -0,0 +1,11 @@
+MAS SaaS Job Cleaner
+===============================================================================
+
+Deploys the `mas-saas-job-cleaner-cron` CronJob, responsible for cleaning up orphaned Job resources in the cluster. It works by grouping Jobs in the cluster according to the `mas.ibm.com/job-cleanup-group` label, then deleting all Jobs from each group except for the one with the latest `creationTimestamp`.
+
+For safety, the CronJob is assigned a ServiceAccount that can only list and delete Job resources (so it can never delete any other type of resource). Furthermore, the logic ensures that only Job resources with the `mas.ibm.com/job-cleanup-group` label can be deleted.
+
+The `mas-devops-saas-job-cleaner` command executed by this CronJob is defined in [python-devops](https://github.com/ibm-mas/python-devops/blob/stable/bin/mas-devops-saas-job-cleaner).
+
+
+> In MaS SaaS, Job resources are routinely orphaned (i.e. marked for deletion by ArgoCD) since, when an update is required to an immutable Job field (e.g. its image tag), a new version of the Job resource must be created with a different name. When [auto_delete: false](https://ibm-mas.github.io/gitops/main/accountrootmanifest/#auto_delete) is set, ArgoCD will (by design) not perform this cleanup for us. Over time, Job resources will accumulate and put pressure on the K8S API server.

cluster-applications/000-job-cleaner/templates/04-jobcleaner_CronJob.yaml

@@ -0,0 +1,102 @@
+{{- /*
+Use the build/bin/set-cli-image-tag.sh script to update this value across all charts.
+*/}}
+{{- $_cli_image_tag := "13.17.0" }}
+
+
+{{- $ns := "job-cleaner" }}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: mas-saas-job-cleaner-role
+  annotations:
+    argocd.argoproj.io/sync-wave: "02"
+{{- if .Values.custom_labels }}
+  labels:
+{{ .Values.custom_labels | toYaml | indent 4 }}
+{{- end }}
+rules:
+  - apiGroups:
+      - batch
+    resources: 
+      - jobs
+    verbs: 
+      - list
+      - delete
+
+---
+# Service account that is authorized to read k8s secrets (needed by the job)
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: "mas-saas-job-cleaner-sa"
+  namespace: "{{ $ns }}"
+  annotations:
+    argocd.argoproj.io/sync-wave: "02"
+{{- if .Values.custom_labels }}
+  labels:
+{{ .Values.custom_labels | toYaml | indent 4 }}
+{{- end }}
+
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mas-saas-job-cleaner-rolebinding
+  annotations:
+    argocd.argoproj.io/sync-wave: "03"
+{{- if .Values.custom_labels }}
+  labels:
+{{ .Values.custom_labels | toYaml | indent 4 }}
+{{- end }}
+subjects:
+  - kind: ServiceAccount
+    name: mas-saas-job-cleaner-sa
+    namespace: {{ $ns }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: mas-saas-job-cleaner-role
+
+
+
+---
+kind: CronJob
+apiVersion: batch/v1
+metadata:
+  name: "mas-saas-job-cleaner-cron"
+  namespace: "{{ $ns }}"
+  annotations:
+    argocd.argoproj.io/sync-wave: "04"
+{{- if .Values.custom_labels }}
+  labels:
+{{ .Values.custom_labels | toYaml | indent 4 }}
+{{- end }}
+spec:
+  schedule: '0 0 * * *'
+  suspend: false
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+{{- if .Values.custom_labels }}
+          labels:
+{{ .Values.custom_labels | toYaml | indent 12 }}
+{{- end }}
+        spec:
+          containers:
+            - name: "mas-saas-job-cleaner"
+              image: quay.io/ibmmas/cli:{{ $_cli_image_tag }}
+              imagePullPolicy: IfNotPresent
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  set -e
+                  mas-devops-saas-job-cleaner --label mas.ibm.com/job-cleanup-group --log-level INFO
+          restartPolicy: OnFailure
+          serviceAccountName: "mas-saas-job-cleaner-sa"

cluster-applications/000-job-cleaner/values.yaml

@@ -0,0 +1 @@
+---

cluster-applications/010-redhat-cert-manager/templates/04-postsync-update-sm_Job.yaml

@@ -34,7 +34,7 @@ Increment this value whenever you make a change to an immutable field of the Job
 E.g. passing in a new environment variable.
 Included in $_job_hash (see below).
 */}}
-{{- $_job_version := "v2" }}
+{{- $_job_version := "v3" }}
 
 {{- /*
 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag
@@ -45,6 +45,27 @@ immutable field of any existing Job resource.
 
 {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}}
 
+{{- /*
+Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource.
+
+When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label 
+to identify old Job resources that should be pruned on behalf of ArgoCD.
+
+Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value
+will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp")
+Jobs will be automatically deleted.
+
+$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases
+where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators
+must be added to $_job_cleanup_group.
+
+By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators
+are required here.
+
+*/}}
+{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }}
+
+
 {{ $ns := "cert-manager-operator"}}
 {{ $aws_secret := "aws"}}
 {{ $role_name  :=  "postsync-rhcm-update-sm-r" }}
@@ -142,8 +163,9 @@ metadata:
   namespace: {{ $ns }}
   annotations:
     argocd.argoproj.io/sync-wave: "015"
-{{- if .Values.custom_labels }}
   labels:
+    mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }}
+{{- if .Values.custom_labels }}
 {{ .Values.custom_labels | toYaml | indent 4 }}
 {{- end }}
 spec:

cluster-applications/020-ibm-dro/templates/08-postsync-update-sm_Job.yaml

@@ -26,7 +26,7 @@ Increment this value whenever you make a change to an immutable field of the Job
 E.g. passing in a new environment variable.
 Included in $_job_hash (see below).
 */}}
-{{- $_job_version := "v2" }}
+{{- $_job_version := "v3" }}
 
 {{- /*
 10 char hash appended to the job name taking into account $_job_config_values, $_job_version and $_cli_image_tag
@@ -37,6 +37,26 @@ immutable field of any existing Job resource.
 
 {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}}
 
+{{- /*
+Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource.
+
+When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label 
+to identify old Job resources that should be pruned on behalf of ArgoCD.
+
+Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value
+will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp")
+Jobs will be automatically deleted.
+
+$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases
+where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators
+must be added to $_job_cleanup_group.
+
+By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators
+are required here.
+
+*/}}
+{{- $_job_cleanup_group := cat $_job_name_prefix | sha1sum }}
+
 
 {{ $ns := .Values.dro_namespace}}
 {{ $aws_secret := "aws"}}
@@ -125,8 +145,9 @@ metadata:
   namespace: {{ $ns }}
   annotations:
     argocd.argoproj.io/sync-wave: "028"
-{{- if .Values.custom_labels }}
   labels:
+    mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }}
+{{- if .Values.custom_labels }}
 {{ .Values.custom_labels | toYaml | indent 4 }}
 {{- end }}
 spec:

cluster-applications/060-custom-sa/templates/04-postsync-update-sm_Job.yaml

@@ -38,6 +38,26 @@ immutable field of any existing Job resource.
 {{- $_job_name := join "-" (list $_job_name_prefix $_job_hash )}}
 
 
+{{- /*
+Set as the value for the mas.ibm.com/job-cleanup-group label on the Job resource.
+
+When the auto_delete flag is not set on the root application, a CronJob in the cluster uses this label 
+to identify old Job resources that should be pruned on behalf of ArgoCD.
+
+Any Job resources in the same namespace that have the mas.ibm.com/job-cleanup-group with this value
+will be considered to belong to the same cleanup group. All but the most recent (i.e. with the latest "creation_timestamp")
+Jobs will be automatically deleted.
+
+$_job_cleanup_group can usually just be based on $_job_name_prefix. There are some special cases
+where multiple Jobs are created in our templates using a Helm loop. In those cases, additional descriminators
+must be added to $_job_cleanup_group.
+NOTE: this is one of those cases; we need a separate cleanup group for each per-sa-key Job.
+
+By convention, we sha1sum this value to guarantee we never exceed the 63 char limit regardless of which discriminators
+are required here.
+
+*/}}
+{{- $_job_cleanup_group := cat $_job_name_prefix $key | sha1sum }}
 
 ---
 apiVersion: batch/v1
@@ -47,8 +67,9 @@ metadata:
   namespace: {{ $.Values.custom_sa_namespace }}
   annotations:
     argocd.argoproj.io/sync-wave: "064"
-{{- if $.Values.custom_labels }}
   labels:
+    mas.ibm.com/job-cleanup-group: {{ $_job_cleanup_group }}
+{{- if $.Values.custom_labels }}
 {{ $.Values.custom_labels | toYaml | indent 4 }}
 {{- end }}
 spec:
@@ -148,4 +169,4 @@ spec:
             defaultMode: 420
             optional: false
   backoffLimit: 4
-{{- end }}
+{{- end }}