Collect coredumps on all nodes

Configure all nodes to save coredumps, and collect any coredumps that
were saved during the gather-extra step.

Author: rfredette

ci-operator/config/openshift/release/openshift-release-master__ci-4.20-upgrade-from-stable-4.19.yaml

@@ -89,6 +89,8 @@ tests:
   interval: 168h
   steps:
     cluster_profile: azure-2
+    pre:
+    - ref: collect-coredumps
     env:
       BASE_DOMAIN: ci2.azure.devcluster.openshift.com
       CONTROL_PLANE_INSTANCE_TYPE: Standard_D8as_v5

ci-operator/config/openshift/router/openshift-router-master.yaml

@@ -142,6 +142,8 @@ tests:
 - as: e2e-upgrade
   steps:
     cluster_profile: azure4
+    pre:
+    - ref: collect-coredumps
     workflow: openshift-upgrade-azure
 - always_run: false
   as: perfscale-aws-ingress-perf

ci-operator/step-registry/collect-coredumps/OWNERS

@@ -0,0 +1,23 @@
+approvers:
+  - knobunc
+  - Miciah
+  - candita
+  - rfredette
+  - alebedev87
+  - gcs278
+  - Thealisyed
+  - grzpiotrowski
+  - rikatz
+  - bentito
+options: {}
+reviewers:
+  - knobunc
+  - Miciah
+  - candita
+  - rfredette
+  - alebedev87
+  - gcs278
+  - Thealisyed
+  - grzpiotrowski
+  - rikatz
+  - bentito

ci-operator/step-registry/collect-coredumps/collect-coredumps-commands.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Create a machine config that installs a systemd unit on nodes. The systemd unit configures the nodes to save any
+# coredumps that are generated, which will be collected during the gather-extra step.
+
+echo "Creating manifests to enable coredump collection"
+
+for role in master worker; do
+cat > "${SHARED_DIR}/manifest_collect_coredumps_machineconfig_${role}.yaml" <<EOF
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: $role
+  name: collect-coredumps-${role}
+spec:
+  config:
+    ignition:
+      version: 3.2.0
+    systemd:
+      units:
+      - contents: |
+          [Unit]
+          After=multi-user.target
+
+          [Service]
+		  Type=oneshot
+		  ExecStart=sysctl -w fs.suid_dumpable=1
+
+          [Install]
+          WantedBy=multi-user.target
+        enabled: true
+        name: collect-coredumps.service
+EOF
+echo "manifest_collect_coredumps_machineconfig_${role}.yaml"
+echo "---------------------------------------------"
+cat ${SHARED_DIR}/manifest_collect_coredumps_machineconfig_${role}.yaml
+done

ci-operator/step-registry/collect-coredumps/collect-coredumps-ref.metadata.json

@@ -0,0 +1,29 @@
+{
+	"path": "collect-coredumps/collect-coredumps-ref.yaml",
+	"owners": {
+		"approvers": [
+			"knobunc",
+			"Miciah",
+			"candita",
+			"rfredette",
+			"alebedev87",
+			"gcs278",
+			"Thealisyed",
+			"grzpiotrowski",
+			"rikatz",
+			"bentito"
+		],
+		"reviewers": [
+			"knobunc",
+			"Miciah",
+			"candita",
+			"rfredette",
+			"alebedev87",
+			"gcs278",
+			"Thealisyed",
+			"grzpiotrowski",
+			"rikatz",
+			"bentito"
+		]
+	}
+}

ci-operator/step-registry/collect-coredumps/collect-coredumps-ref.yaml

@@ -0,0 +1,17 @@
+ref:
+  as: collect-coredumps
+  from_image:
+   namespace: origin
+   name: centos
+   tag: '8'
+  commands: collect-coredumps-commands.sh
+  resources:
+    requests:
+      cpu: 10m
+      memory: 100Mi
+  documentation: |-
+    The coredump service configures nodes to save all generated coredumps. This is useful for debugging failures of
+    components running in privileged pods, such as router pods.
+    The service is deployed by injecting an installer manifest containing a MachineConfig. This contains one systemd
+    unit, which sets the sysctl variable fs.suid_dumpable to 1, instructing systemd-coredump to save any coredumps it
+    encounters into /var/lib/systemd/coredump/ . Those coredump files are then collected during the gather-extra step.

ci-operator/step-registry/gather/extra/gather-extra-commands.sh

@@ -214,6 +214,18 @@ for pqn in $(oc get pods -n openshift-etcd -l app=etcd --no-headers -o=name); do
 done
 echo "INFO: done attempting to fetch etcd debug info"
 
+echo "INFO: gathering coredumps if present"
+output_dir="${ARTIFACT_DIR}/coredumps"
+mkdir -p "$output_dir"
+oc adm node-logs -l kubernetes.io/os=linux --path="/var/lib/systemd/coredump/" | \
+  tee ${output_dir}/.coredumps_listing
+while IFS= read -r item; do
+  node=$(echo $item |cut -d ' ' -f 1)
+  fname=$(echo $item |cut -d ' ' -f 2)
+  echo "INFO: Queueing download/gzip of /var/lib/systemd/coredump/${fname} from ${node}";
+  echo "INFO: gziping to ${output_dir}/${node}-${fname}.gz";
+  FILTER=gzip queue ${output_dir}/${node}-${fname}.gz oc --insecure-skip-tls-verify adm node-logs ${node} --path=/var/lib/systemd/coredump/${fname}
+done < ${output_dir}/.coredumps_listing
 
 function gather_network() {
   local namespace=$1