Update DNF packages & upgrade to Rocky Linux 9.7

.github/workflows/fatimage.yml

@@ -35,10 +35,10 @@ jobs:
       matrix: # build RL8, RL9
         build:
           - image_name: openhpc-RL8
-            source_image_name: Rocky-8-GenericCloud-Base-8.10-20240528.0.x86_64.raw
+            source_image_name: Rocky-8-GenericCloud-Base-8.10-20240528.0.x86_64.qcow2
             inventory_groups: fatimage
           - image_name: openhpc-RL9
-            source_image_name: Rocky-9-GenericCloud-Base-9.6-20250531.0.x86_64.qcow2
+            source_image_name: Rocky-9-GenericCloud-Base-9.7-20251123.2.x86_64.qcow2
             inventory_groups: fatimage
     env:
       ANSIBLE_FORCE_COLOR: True

ansible/ci/update_timestamps.yml

@@ -1,4 +1,5 @@
 ---
+# run manually from environments/.stackhpc: ansible-playbook ../../ansible/ci/update_timestamps.yml -e appliances_repository_root=../..
 - hosts: localhost
   tasks:
     - name: Get latest timestamps from sources # noqa: syntax-check[unknown-module] # ansible/library/latest_timestamps.py

ansible/fatimage.yml

@@ -10,6 +10,16 @@
     - name: Report inventory groups
       ansible.builtin.debug:
         var: group_names
+    - name: Report ssh command to connect
+      ansible.builtin.debug:
+        msg: >
+          connect to builder machine using:
+          ssh {% if ansible_ssh_port | default(false) %}-p {{ ansible_ssh_port }}{% endif %}
+          {% if ansible_ssh_private_key_file | default(false) %} -i {{ ansible_ssh_private_key_file }}{% endif %}
+          {{ ansible_ssh_common_args | default('') }}
+          {{ ansible_ssh_extra_args | default('') }}
+          -o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null
+          {{ ansible_user }}@{{ ansible_host }}
 
 - name: Run pre.yml hook
   vars:

ansible/library/latest_timestamps.py

@@ -61,14 +61,21 @@ def run_module():  # pylint: disable=missing-function-docstring
     timestamps = dict(module.params["repos_dict"])
     for repo in timestamps:
         for version in timestamps[repo]:
-
-            html_txt = requests.get(
-                url=module.params["content_url"]
+            url = (
+                module.params["content_url"]
                 + "/"
                 + timestamps[repo][version]["pulp_path"]
-            ).text
+            )
+            html = requests.get(url=url)
+            if not html.ok:
+                module.fail_json(
+                    msg=(
+                        f"Couldn't retrieve timestamps for {repo}[{version}]"
+                        f"url {url}: {html.status_code} {html.text}"
+                    )
+                )
             timestamp_link_list = (
-                BeautifulSoup(html_txt, features="html.parser")
+                BeautifulSoup(html.text, features="html.parser")
                 .body.find("pre")
                 .find_all()
             )  # getting raw list of timestamps from html

ansible/roles/doca/defaults/main.yml

@@ -1,5 +1,11 @@
 ---
 
-doca_version: "2.9.3" # 2.9 is LTS, last to support ConnectX-4, 3 years for bug fixes and CVE updates
+# 2.9 is LTS, last to support ConnectX-4, 3 years for bug fixes and CVE updates:
+# https://docs.nvidia.com/doca/archive/2-9-3/changes+and+new+features/index.html
+# DOCA 2.9.3 has not been released for rhel 9.7 yet (as of 2025-12-18).
+# DOCA 3.2.0 is compatible with rhel 9 but lists Connectx-4 lx as supported but not ConnectX-4:
+# https://docs.nvidia.com/doca/sdk/general-support/index.html#src-4413883949_id-.GeneralSupportv3.2.0LC-SupportedPlatformsandNICFirmwareVersions
+doca_version: "{{ ansible_distribution_version is version('9.7', '>=') | ternary('3.2.1', '2.9.3') }}"
 doca_profile: doca-ofed
-doca_repo_url: "https://linux.mellanox.com/public/repo/doca/{{ doca_version }}/rhel{{ ansible_distribution_version }}/{{ ansible_architecture }}/"
+doca_distribution: "{{ doca_version is version('3.2.0', '>=') | ternary(ansible_distribution_major_version, ansible_distribution_version) }}"
+doca_repo_url: "https://linux.mellanox.com/public/repo/doca/{{ doca_version }}/rhel{{ doca_distribution }}/{{ ansible_architecture }}/"

environments/.stackhpc/tofu/cluster_image.auto.tfvars.json

@@ -1,6 +1,6 @@
 {
   "cluster_image": {
-    "RL8": "openhpc-RL8-251213-1133-31273766",
-    "RL9": "openhpc-RL9-251213-1133-31273766"
+    "RL8": "openhpc-RL8-251218-1351-610949fc",
+    "RL9": "openhpc-RL9-251218-1351-610949fc"
   }
 }

environments/common/inventory/group_vars/all/dnf_repo_timestamps.yml

@@ -24,12 +24,30 @@ dnf_repos_default:
       repo_file: OpenHPC
     '9':
       pulp_path: OpenHPC/3/updates/EL_9
-      pulp_timestamp: 20250510T003301
+      pulp_timestamp: 20251104T202642
       repo_file: OpenHPC
+  TurboVNC:
+    '8':
+      pulp_path: turbovnc/x86_64
+      pulp_timestamp: 20251009T091906
+      repo_file: TurboVNC
+    '9':
+      pulp_path: turbovnc/x86_64
+      pulp_timestamp: 20251009T091906
+      repo_file: TurboVNC
+  TurboVNC-source:
+    '8':
+      pulp_path: turbovnc/srpms
+      pulp_timestamp: 20251009T091906
+      repo_file: TurboVNC
+    '9':
+      pulp_path: turbovnc/srpms
+      pulp_timestamp: 20251009T091906
+      repo_file: TurboVNC
   appstream:
     '8.10':
       pulp_path: rocky/8.10/AppStream/x86_64/os
-      pulp_timestamp: 20250923T022841
+      pulp_timestamp: 20251214T211108
       repo_file: Rocky-AppStream
     '9.4':
       pulp_path: rocky/9.4/AppStream/x86_64/os
@@ -41,21 +59,29 @@ dnf_repos_default:
       repo_file: rocky
     '9.6':
       pulp_path: rocky/9.6/AppStream/x86_64/os
-      pulp_timestamp: 20250923T031638
+      pulp_timestamp: 20251106T205505
+      repo_file: rocky
+    '9.7':
+      pulp_path: rocky/9.7/AppStream/x86_64/os
+      pulp_timestamp: 20251215T205523
       repo_file: rocky
   appstream-source:
     '8.10':
-      pulp_path: rocky/8.10/AppStream/source/os
-      pulp_timestamp: 20250923T024945
+      pulp_path: rocky/8.10/AppStream/source/tree
+      pulp_timestamp: 20251214T204840
       repo_file: Rocky-Sources
     '9.6':
-      pulp_path: rocky/9.6/AppStream/source/os
-      pulp_timestamp: 20250923T043546
+      pulp_path: rocky/9.6/AppStream/source/tree
+      pulp_timestamp: 20251106T210718
+      repo_file: rocky
+    '9.7':
+      pulp_path: rocky/9.7/AppStream/source/tree
+      pulp_timestamp: 20251214T212049
       repo_file: rocky
   baseos:
     '8.10':
       pulp_path: rocky/8.10/BaseOS/x86_64/os
-      pulp_timestamp: 20250918T034501
+      pulp_timestamp: 20251214T211108
       repo_file: Rocky-BaseOS
     '9.4':
       pulp_path: rocky/9.4/BaseOS/x86_64/os
@@ -67,21 +93,47 @@ dnf_repos_default:
       repo_file: rocky
     '9.6':
       pulp_path: rocky/9.6/BaseOS/x86_64/os
-      pulp_timestamp: 20250923T045903
+      pulp_timestamp: 20251106T211545
+      repo_file: rocky
+    '9.7':
+      pulp_path: rocky/9.7/BaseOS/x86_64/os
+      pulp_timestamp: 20251215T212006
       repo_file: rocky
   baseos-source:
     '8.10':
-      pulp_path: rocky/8.10/BaseOS/source/os
-      pulp_timestamp: 20250918T040529
+      pulp_path: rocky/8.10/BaseOS/source/tree
+      pulp_timestamp: 20251214T204840
       repo_file: Rocky-Sources
     '9.6':
-      pulp_path: rocky/9.6/BaseOS/source/os
-      pulp_timestamp: 20250923T043546
+      pulp_path: rocky/9.6/BaseOS/source/tree
+      pulp_timestamp: 20251107T211059
       repo_file: rocky
+    '9.7':
+      pulp_path: rocky/9.7/BaseOS/source/tree
+      pulp_timestamp: 20251214T212049
+      repo_file: rocky
+  cernvmfs_cfg:
+    '8':
+      pulp_path: cvmfs-config/EL/8/x86_64
+      pulp_timestamp: 20250805T130249
+      repo_file: cvmfs
+    '9':
+      pulp_path: cvmfs-config/EL/9/x86_64
+      pulp_timestamp: 20250805T130249
+      repo_file: cvmfs
+  cernvmfs_pkgs:
+    '8':
+      pulp_path: cvmfs/EL/8/x86_64
+      pulp_timestamp: 20251113T202054
+      repo_file: cvmfs
+    '9':
+      pulp_path: cvmfs/EL/9/x86_64
+      pulp_timestamp: 20251113T202054
+      repo_file: cvmfs
   crb:
     '8.10':
       pulp_path: rocky/8.10/PowerTools/x86_64/os
-      pulp_timestamp: 20250918T034501
+      pulp_timestamp: 20251214T211108
       repo_file: Rocky-PowerTools
       repo_name: powertools
     '9.4':
@@ -94,45 +146,53 @@ dnf_repos_default:
       repo_file: rocky
     '9.6':
       pulp_path: rocky/9.6/CRB/x86_64/os
-      pulp_timestamp: 20250923T031638
+      pulp_timestamp: 20251107T205232
+      repo_file: rocky
+    '9.7':
+      pulp_path: rocky/9.7/CRB/x86_64/os
+      pulp_timestamp: 20251214T205707
       repo_file: rocky
   crb-source:
     '8.10':
       pulp_path: rocky/8.10/PowerTools/source/tree
-      pulp_timestamp: 20250923T125600
+      pulp_timestamp: 20251126T230532
       repo_file: Rocky-Sources
       repo_name: powertools-source
     '9.6':
-      pulp_path: rocky/9.6/CRB/source/os
-      pulp_timestamp: 20250923T043546
+      pulp_path: rocky/9.6/CRB/source/tree
+      pulp_timestamp: 20251106T210718
+      repo_file: rocky
+    '9.7':
+      pulp_path: rocky/9.7/CRB/source/tree
+      pulp_timestamp: 20251203T103807
       repo_file: rocky
   epel:
     '8':
       pulp_path: epel/8/Everything/x86_64
-      pulp_timestamp: 20250923T001717
+      pulp_timestamp: 20251215T205559
       repo_file: epel
     '9':
       pulp_path: epel/9/Everything/x86_64
-      pulp_timestamp: 20250923T001717
-      repo_file: epel
-  epel-source:
-    '8':
-      pulp_path: epel/8/Everything/source
-      pulp_timestamp: 20250923T001717
-      repo_file: epel
-    '9':
-      pulp_path: epel/9/Everything/source
-      pulp_timestamp: 20250923T001717
+      pulp_timestamp: 20251217T204105
       repo_file: epel
   epel-cisco-openh264:
     '9':
       pulp_path: openh264/epel/9/x86_64/os
       pulp_timestamp: 20250925T130153
       repo_file: epel-cisco-openh264
+  epel-source:
+    '8':
+      pulp_path: epel/8/Everything/source/tree
+      pulp_timestamp: 20251215T205559
+      repo_file: epel
+    '9':
+      pulp_path: epel/9/Everything/source/tree
+      pulp_timestamp: 20251217T204105
+      repo_file: epel
   extras:
     '8.10':
       pulp_path: rocky/8.10/extras/x86_64/os
-      pulp_timestamp: 20250510T032327
+      pulp_timestamp: 20251129T221326
       repo_file: Rocky-Extras
     '9.4':
       pulp_path: rocky/9.4/extras/x86_64/os
@@ -146,67 +206,39 @@ dnf_repos_default:
       pulp_path: rocky/9.6/extras/x86_64/os
       pulp_timestamp: 20250726T040613
       repo_file: rocky-extras
+    '9.7':
+      pulp_path: rocky/9.7/extras/x86_64/os
+      pulp_timestamp: 20251203T103807
+      repo_file: rocky-extras
   extras-source:
     '8.10':
-      pulp_path: rocky/8.10/extras/source/os
-      pulp_timestamp: 20250828T161842
+      pulp_path: rocky/8.10/extras/source/tree
+      pulp_timestamp: 20251129T214913
       repo_file: Rocky-Sources
     '9.6':
       pulp_path: rocky/9.6/extras/source/os
       pulp_timestamp: 20250828T161842
       repo_file: rocky-extras
+    '9.7':
+      pulp_path: rocky/9.7/extras/source/tree
+      pulp_timestamp: 20251203T103807
+      repo_file: rocky-extras
   grafana:
     '8':
       pulp_path: grafana/oss/rpm
-      pulp_timestamp: 20250917T024714
+      pulp_timestamp: 20251216T213511
       repo_file: grafana
       timestamp: 20250615T005738
     '9':
       pulp_path: grafana/oss/rpm
-      pulp_timestamp: 20250917T024714
+      pulp_timestamp: 20251216T213511
       repo_file: grafana
   ondemand-web:
     '8':
       pulp_path: ondemand/4.0/web/el8/x86_64
-      pulp_timestamp: 20250925T130153
+      pulp_timestamp: 20251120T213358
       repo_file: ondemand-web
     '9':
       pulp_path: ondemand/4.0/web/el9/x86_64
-      pulp_timestamp: 20250925T130153
+      pulp_timestamp: 20251120T213358
       repo_file: ondemand-web
-  TurboVNC:
-    '8':
-      pulp_path: turbovnc/x86_64
-      pulp_timestamp: 20251009T091906
-      repo_file: TurboVNC
-    '9':
-      pulp_path: turbovnc/x86_64
-      pulp_timestamp: 20251009T091906
-      repo_file: TurboVNC
-  TurboVNC-source:
-    '8':
-      pulp_path: turbovnc/srpms
-      pulp_timestamp: 20251009T091906
-      repo_file: TurboVNC
-    '9':
-      pulp_path: turbovnc/srpms
-      pulp_timestamp: 20251009T091906
-      repo_file: TurboVNC
-  cernvmfs_pkgs:
-    '8':
-      pulp_path: cvmfs/EL/8/x86_64
-      pulp_timestamp: 20250816T005446
-      repo_file: cvmfs
-    '9':
-      pulp_path: cvmfs/EL/9/x86_64
-      pulp_timestamp: 20250816T005446
-      repo_file: cvmfs
-  cernvmfs_cfg:
-    '8':
-      pulp_path: cvmfs-config/EL/8/x86_64
-      pulp_timestamp: 20250805T130249
-      repo_file: cvmfs
-    '9':
-      pulp_path: cvmfs-config/EL/9/x86_64
-      pulp_timestamp: 20250805T130249
-      repo_file: cvmfs