From 4284ca67f5838e4782a096167f96d56f52582cb0 Mon Sep 17 00:00:00 2001 From: Jakub Darmach Date: Thu, 13 Jul 2023 14:24:12 +0200 Subject: [PATCH 1/7] Overcloud database backup S3 upload Added playbook uploading overcloud MariaDB backups to S3 compatible storage. Can be linked as overcloud database backup post hook. --- .../ansible/upload-database-backup-s3.yml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 etc/kayobe/ansible/upload-database-backup-s3.yml diff --git a/etc/kayobe/ansible/upload-database-backup-s3.yml b/etc/kayobe/ansible/upload-database-backup-s3.yml new file mode 100644 index 0000000000..8a9691a30b --- /dev/null +++ b/etc/kayobe/ansible/upload-database-backup-s3.yml @@ -0,0 +1,35 @@ +--- + # This playbook uploads MariaDB backups to an AWS S3 object store. + # Can be linked as a post hook for overcloud-database-backup. + + - hosts: controllers[0] + vars: + backup_directory: "/var/lib/docker/volumes/mariadb_backup/_data" + kayobe_venv: "/opt/kayobe/venvs/kayobe" + tasks: + - name: Ensure AWS S3 module prerequisities are available + pip: + name: + - boto3 + - botocore + virtualenv: "{{ kayobe_venv }}" + + - name: Build backup file list + find: + paths: "{{ backup_directory }}" + become: True + register: backups + + - name: Upload backup files to S3 + amazon.aws.aws_s3: + s3_url: "{{ s3_url }}" + access_key: "{{ secrets_s3_access_key }}" + secret_key: "{{ secrets_s3_secret_key }}" + bucket: "{{ s3_mariadb_mysql_backup_bucket }}" + object: "{{ item.path | basename }}" + src: "{{ item.path }}" + mode: put + overwrite: different + validate_certs: False + become: True + with_items: "{{ backups.files }}" \ No newline at end of file From 26dcf3ebf75097a8d585c37e501b651790ddab4e Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 17 Dec 2025 15:34:19 +0000 Subject: [PATCH 2/7] Use new amazon.aws.s3_object module + fix indentation --- .../tools/upload-database-backup-s3.yml | 35 +++++++++++++++++++ .../ansible/upload-database-backup-s3.yml | 35 ------------------- 2 files changed, 35 insertions(+), 35 deletions(-) create mode 100644 etc/kayobe/ansible/tools/upload-database-backup-s3.yml delete mode 100644 etc/kayobe/ansible/upload-database-backup-s3.yml diff --git a/etc/kayobe/ansible/tools/upload-database-backup-s3.yml b/etc/kayobe/ansible/tools/upload-database-backup-s3.yml new file mode 100644 index 0000000000..91ca2b6005 --- /dev/null +++ b/etc/kayobe/ansible/tools/upload-database-backup-s3.yml @@ -0,0 +1,35 @@ +--- +# This playbook uploads MariaDB backups to an AWS S3 object store. +# Can be linked as a post hook for overcloud-database-backup. + +- hosts: controllers[0] + vars: + backup_directory: "/var/lib/docker/volumes/mariadb_backup/_data" + kayobe_venv: "{{ virtualenv_path }}/kayobe" + tasks: + - name: Ensure AWS S3 module prerequisites are available + pip: + name: + - boto3 + - botocore + virtualenv: "{{ kayobe_venv }}" + + - name: Build backup file list + find: + paths: "{{ backup_directory }}" + become: True + register: backups + + - name: Upload backup files to S3 + amazon.aws.s3_object: + endpoint_url: "{{ s3_mariadb_backup_url }}" + access_key: "{{ secrets_s3_mariadb_backup_access_key }}" + secret_key: "{{ secrets_s3_mariadb_backup_secret_key }}" + bucket: "{{ s3_mariadb_backup_bucket }}" + object: "{{ item.path | basename }}" + src: "{{ item.path }}" + mode: put + overwrite: different + validate_certs: False + become: True + with_items: "{{ backups.files }}" \ No newline at end of file diff --git a/etc/kayobe/ansible/upload-database-backup-s3.yml b/etc/kayobe/ansible/upload-database-backup-s3.yml deleted file mode 100644 index 8a9691a30b..0000000000 --- a/etc/kayobe/ansible/upload-database-backup-s3.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- - # This playbook uploads MariaDB backups to an AWS S3 object store. - # Can be linked as a post hook for overcloud-database-backup. - - - hosts: controllers[0] - vars: - backup_directory: "/var/lib/docker/volumes/mariadb_backup/_data" - kayobe_venv: "/opt/kayobe/venvs/kayobe" - tasks: - - name: Ensure AWS S3 module prerequisities are available - pip: - name: - - boto3 - - botocore - virtualenv: "{{ kayobe_venv }}" - - - name: Build backup file list - find: - paths: "{{ backup_directory }}" - become: True - register: backups - - - name: Upload backup files to S3 - amazon.aws.aws_s3: - s3_url: "{{ s3_url }}" - access_key: "{{ secrets_s3_access_key }}" - secret_key: "{{ secrets_s3_secret_key }}" - bucket: "{{ s3_mariadb_mysql_backup_bucket }}" - object: "{{ item.path | basename }}" - src: "{{ item.path }}" - mode: put - overwrite: different - validate_certs: False - become: True - with_items: "{{ backups.files }}" \ No newline at end of file From e568ece41be410c7f10d1668ba1f387b0938ec52 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 17 Dec 2025 15:36:05 +0000 Subject: [PATCH 3/7] end of file newline --- etc/kayobe/ansible/tools/upload-database-backup-s3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/tools/upload-database-backup-s3.yml b/etc/kayobe/ansible/tools/upload-database-backup-s3.yml index 91ca2b6005..ff6400d3ac 100644 --- a/etc/kayobe/ansible/tools/upload-database-backup-s3.yml +++ b/etc/kayobe/ansible/tools/upload-database-backup-s3.yml @@ -32,4 +32,4 @@ overwrite: different validate_certs: False become: True - with_items: "{{ backups.files }}" \ No newline at end of file + with_items: "{{ backups.files }}" From f7be4572d6ece5242731254657aba205781fe85f Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 17 Dec 2025 15:49:45 +0000 Subject: [PATCH 4/7] Documents db backups --- doc/source/operations/database-backups.rst | 40 ++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 doc/source/operations/database-backups.rst diff --git a/doc/source/operations/database-backups.rst b/doc/source/operations/database-backups.rst new file mode 100644 index 0000000000..22c21279b1 --- /dev/null +++ b/doc/source/operations/database-backups.rst @@ -0,0 +1,40 @@ +================ +Database Backups +================ + +An OpenStack deployment includes MariaDB to be used as a database by the +OpenStack services. Kayobe has `built-in support +`__ +for backing up this database, but these backups are just stored on one of the +OpenStack controller hosts. + +We have a playbook ``tools/upload-database-backup-s3.yml`` which can be used to +upload these backups to an S3 object store. To use this, you will need: + +* The endpoint of the S3 object store. + +* EC2 access and secret keys to authenticate to the S3 object store. + +* The name of a pre-existing bucket in the S3 object store. + +These should be set as follows: + +.. code-block:: yaml + :caption: ``$KAYOBE_CONFIG_PATH/inventory/group_vars/all/mariadb-backup`` + + s3_mariadb_backup_url: "" + s3_mariadb_backup_bucket: "" + +.. code-block:: yaml + :caption: ``$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml`` + + secrets_s3_mariadb_backup_access_key: "" + secrets_s3_mariadb_backup_secret_key: " Date: Wed, 17 Dec 2025 15:51:00 +0000 Subject: [PATCH 5/7] Add index for db backups docs --- doc/source/operations/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/operations/index.rst b/doc/source/operations/index.rst index f8f7333944..2ef0929a5d 100644 --- a/doc/source/operations/index.rst +++ b/doc/source/operations/index.rst @@ -10,6 +10,7 @@ This guide is for operators of the StackHPC Kayobe configuration project. ceph-management control-plane-operation customising-horizon + database-backups gpu-in-openstack bifrost-hardware-inventory-management hotfix-playbook From f13a760b5967193d9207c20934fc3d38e1234ace Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 17 Dec 2025 15:54:47 +0000 Subject: [PATCH 6/7] Linter fixes --- etc/kayobe/ansible/tools/upload-database-backup-s3.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/ansible/tools/upload-database-backup-s3.yml b/etc/kayobe/ansible/tools/upload-database-backup-s3.yml index ff6400d3ac..5efd4c72a5 100644 --- a/etc/kayobe/ansible/tools/upload-database-backup-s3.yml +++ b/etc/kayobe/ansible/tools/upload-database-backup-s3.yml @@ -2,20 +2,21 @@ # This playbook uploads MariaDB backups to an AWS S3 object store. # Can be linked as a post hook for overcloud-database-backup. -- hosts: controllers[0] +- name: Upload MariaDB backups to S3 + hosts: controllers[0] vars: backup_directory: "/var/lib/docker/volumes/mariadb_backup/_data" kayobe_venv: "{{ virtualenv_path }}/kayobe" tasks: - name: Ensure AWS S3 module prerequisites are available - pip: + ansible.builtin.pip: name: - boto3 - botocore virtualenv: "{{ kayobe_venv }}" - name: Build backup file list - find: + ansible.builtin.find: paths: "{{ backup_directory }}" become: True register: backups From b526fc0c4a9e1282658b745844960d882d9e68d1 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Thu, 18 Dec 2025 09:26:26 +0000 Subject: [PATCH 7/7] Update doc/source/operations/database-backups.rst Co-authored-by: Alex Welsh <112560678+Alex-Welsh@users.noreply.github.com> --- doc/source/operations/database-backups.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/operations/database-backups.rst b/doc/source/operations/database-backups.rst index 22c21279b1..cff58a3654 100644 --- a/doc/source/operations/database-backups.rst +++ b/doc/source/operations/database-backups.rst @@ -37,4 +37,4 @@ backup``: .. code-block:: bash mkdir -p $KAYOBE_CONFIG_PATH/hooks/overcloud-database-backup/post.d/ - ln -s ../../../ansible/upload-database-backup-s3.yml $KAYOBE_CONFIG_PATH/hooks/overcloud-database-backup/post.d/10-upload-database-backup-s3.yml + ln -s ../../../ansible/tools/upload-database-backup-s3.yml $KAYOBE_CONFIG_PATH/hooks/overcloud-database-backup/post.d/10-upload-database-backup-s3.yml