From 192fa69d931ccb33dc8a028e5fd589e32f3289e7 Mon Sep 17 00:00:00 2001 From: Justin Cinkelj Date: Fri, 17 Jan 2025 15:52:12 +0100 Subject: [PATCH] CI playbook to recreate CI test clusters Signed-off-by: Justin Cinkelj --- DEVELOPMENT.md | 11 ++++ ci-infra/helpers/ci_hosts_recreate.yml | 64 ++++++++++++++++++++++ ci-infra/helpers/ci_hosts_recreate_one.yml | 51 +++++++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 ci-infra/helpers/ci_hosts_recreate.yml create mode 100644 ci-infra/helpers/ci_hosts_recreate_one.yml diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 160f4440..024f261a 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -200,3 +200,14 @@ Steps: - ensure ip address is added to entraAD (azure) app registration for OIDC integration (ask Dave if needed) - "app_display_name": "Scale Computing HC3", - "app_id": "d2298ec0-0596-49d2-9554-840a2fe20603", + +## Reset VSNS test cluster + +After enough testing VSNS test clusters get slow. +CI integration testing takes say 8 hours instead of 1 hour. +At that time we recreate VSNS from template VM. +Run: + +```bash +ansible-playbook ci-infra/helpers/ci_hosts_recreate.yml -v +``` diff --git a/ci-infra/helpers/ci_hosts_recreate.yml b/ci-infra/helpers/ci_hosts_recreate.yml new file mode 100644 index 00000000..bb2f7344 --- /dev/null +++ b/ci-infra/helpers/ci_hosts_recreate.yml @@ -0,0 +1,64 @@ +--- +# On CI physical HyperCore host run multiple VSNS CI HyperCore hosts. +# The VSNS needs a periodic "refresh". Seems like after many many API calls, +# the VSNS API gets slow. Daily CI integration test last for 8 hours instead of 1..2 hours. +# Refresh means remove old VSNS, and create new from VSNS template VM. +# VSNS template VMs were prepared manually + +- name: Recreate CI VSNS test hosts/VMs + hosts: localhost + connection: local + # gather_facts: false + vars: + vsns_spec_all: + vsns914-ci: + source: vsns914-template + ip: 10.5.11.200 + vsns9213-ci: + source: vsns9213-template + ip: 10.5.11.201 + # vsns931-ci: # not used + # source: vsns931-template + # ip: 10.5.11.202 + # vsns935-ci: # no template VM + # source: ? + # ip: 10.5.11.203 + vsns940-ci: + # Is not used in CI tests + source: vsns940-template + ip: 10.5.11.204 + vsns9417-ci: + source: vsns9417-template + ip: 10.5.11.205 + + tasks: + # ------------------------------------------------------ + - name: Show info + ansible.builtin.debug: + msg: source_vm={{ source_vm }} dest_vm={{ dest_vm }} ip={{ vm_ip }} url=https://{{ vm_ip }} + loop: "{{ vsns_spec_all.keys() }}" + loop_control: + loop_var: dest_vm + vars: + source_vm: "{{ vsns_spec_all[dest_vm].source }}" + vm_ip: "{{ vsns_spec_all[dest_vm].ip }}" + + - name: Recreate CI VSNSs + loop: "{{ vsns_spec_all.keys() }}" + loop_control: + loop_var: dest_vm + vars: + source_vm: "{{ vsns_spec_all[dest_vm].source }}" + vm_ip: "{{ vsns_spec_all[dest_vm].ip }}" + ansible.builtin.include_tasks: + file: ci_hosts_recreate_one.yml + + - name: Show URLs + ansible.builtin.debug: + msg: source_vm={{ source_vm }} dest_vm={{ dest_vm }} ip={{ vm_ip }} url=https://{{ vm_ip }} + loop: "{{ vsns_spec_all.keys() }}" + loop_control: + loop_var: dest_vm + vars: + source_vm: "{{ vsns_spec_all[dest_vm].source }}" + vm_ip: "{{ vsns_spec_all[dest_vm].ip }}" diff --git a/ci-infra/helpers/ci_hosts_recreate_one.yml b/ci-infra/helpers/ci_hosts_recreate_one.yml new file mode 100644 index 00000000..6d5af280 --- /dev/null +++ b/ci-infra/helpers/ci_hosts_recreate_one.yml @@ -0,0 +1,51 @@ +--- +- name: Show input params + ansible.builtin.debug: + msg: source_vm={{ source_vm }} dest_vm={{ dest_vm }} vm_ip={{ vm_ip }} url=https://{{ vm_ip }} + +- name: Source VM info + scale_computing.hypercore.vm_info: + vm_name: "{{ source_vm }}" + register: source_vm_result + +- name: Fail if source VM is not present + ansible.builtin.assert: + that: + - source_vm_result.records | length == 1 + fail_msg: "Source VM {{ source_vm }} does not exist." + +- name: Old VM info + scale_computing.hypercore.vm_info: + vm_name: "{{ dest_vm }}" + register: vm_result + +- name: Force-shutdown old VM if exists + scale_computing.hypercore.vm_params: + vm_name: "{{ dest_vm }}" + power_state: stop + force_reboot: true + shutdown_timeout: 10 + when: vm_result.records | length + +- name: Delete old VM + scale_computing.hypercore.vm: + vm_name: "{{ dest_vm }}" + state: absent + +- name: Clone template VM + scale_computing.hypercore.vm_clone: + vm_name: "{{ dest_vm }}" + source_vm_name: "{{ source_vm }}" + preserve_mac_address: true + +- name: Set VM resources + scale_computing.hypercore.vm_params: + vm_name: "{{ dest_vm }}" + description: "https://{{ vm_ip }} created={{ ansible_date_time.date }}" + vcpu: 8 + memory: "{{ '16 GB' | human_to_bytes }}" + +- name: Start VM + scale_computing.hypercore.vm_params: + vm_name: "{{ dest_vm }}" + power_state: start