From fee29bfcf82725c27a063d78d72fdcf9b31b807c Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Thu, 15 Jan 2026 09:40:03 +0000 Subject: [PATCH] MAINT: Remove upload_all.py As we are not using the QEMU builder we do not need to upload images from the local machine to OpenStack. The OpenStack provisioner does it for us. --- .github/workflows/python_checks.yaml | 33 ----- scripts/requirements.txt | 4 - scripts/test/test_upload_all.py | 191 --------------------------- scripts/upload_all.py | 121 ----------------- 4 files changed, 349 deletions(-) delete mode 100644 .github/workflows/python_checks.yaml delete mode 100644 scripts/requirements.txt delete mode 100644 scripts/test/test_upload_all.py delete mode 100755 scripts/upload_all.py diff --git a/.github/workflows/python_checks.yaml b/.github/workflows/python_checks.yaml deleted file mode 100644 index 0a1faa10..00000000 --- a/.github/workflows/python_checks.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Black formatter - -on: [push, pull_request] - -jobs: - format: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: psf/black@stable - - test_and_lint: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11"] - steps: - - uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements.txt - - - name: Run tests - run: cd scripts && python3 -m pytest . - - - name: Analyse with pylint - run: pylint --recursive yes scripts \ No newline at end of file diff --git a/scripts/requirements.txt b/scripts/requirements.txt deleted file mode 100644 index 300a3f85..00000000 --- a/scripts/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -gitpython -openstacksdk -pytest -pylint \ No newline at end of file diff --git a/scripts/test/test_upload_all.py b/scripts/test/test_upload_all.py deleted file mode 100644 index 746da538..00000000 --- a/scripts/test/test_upload_all.py +++ /dev/null @@ -1,191 +0,0 @@ -""" -Associated tests for the upload_all.py script -""" - -from datetime import datetime -from pathlib import Path -from typing import Dict -from unittest import mock -from unittest.mock import Mock, call - -import pytest - -from upload_all import ( - glob_output_files, - Args, - parse_args, - upload_images_to_openstack, - upload_single_image, - main, - get_upload_name, -) - - -def test_glob_output_files(tmp_path): - """ - Test that the glob_output_files function returns the expected files - and does not include directories - :param tmp_path: A temp path setup by pytest - """ - expected_output_path = tmp_path / "k8s-image-builder" / "images" / "capi" / "output" - expected_output_path.mkdir(parents=True) - - expected = ["ubuntu-kube-1.23.4", "ubuntu-kube-1.40.0"] - for out_name in expected: - # Make parent dir as this is a named dir with a single file - (expected_output_path / out_name).mkdir(parents=True) - (expected_output_path / out_name / out_name).touch() - # And a fake file in each - (expected_output_path / out_name / "should_not_appear").touch() - - with mock.patch("git.Repo") as mocked_repo: - mocked_repo.return_value.working_tree_dir = str(tmp_path) - returned = list(glob_output_files()) # Avoid exhausting the generator - - expected = sorted(str(expected_output_path / file / file) for file in expected) - assert sorted(str(file) for file in returned) == expected - assert len(returned) == len(expected) - - -def test_arg_parse_all_true(): - """ - Tests that the parse_args function returns the expected Args object - if the user sets all optional arguments - """ - user_input = ["--dry-run", "--public", "default"] - expected = Args(dry_run=True, public=True, os_cloud="default") - assert parse_args(user_input) == expected - - -def test_arg_parse_no_optional_args(): - """ - Tests that the parse_args function returns the expected Args object - if the user does not set any optional arguments - """ - user_input = ["bar"] - expected = Args(dry_run=False, public=False, os_cloud="bar") - assert parse_args(user_input) == expected - - -def test_arg_parse_no_args(): - """ - Tests that the parse_args function exits with a SystemExit - if the user does not set the OS cloud argument - """ - user_input = [] - with pytest.raises(SystemExit): - parse_args(user_input) - - -def test_upload_images_to_openstack_dry_run(): - """ - Tests that the upload_images_to_openstack function does not call - anything uploading to OpenStack if the dry_run flag is set - """ - expected_files = [Path("file1"), Path("file2")] - args = Args(dry_run=True, public=True, os_cloud="default") - - with mock.patch("upload_all.upload_single_image") as mocked_upload_single_image: - upload_images_to_openstack(expected_files, args) - - mocked_upload_single_image.assert_not_called() - - -@pytest.fixture(name="fake_upload_args") -def fake_upload_args_fixture(): - """ - Fixture to generate the expected arguments for upload_single_image - """ - - def _expected_args_helper(file: Path, visibility: str) -> Dict: - """ - Helper function to generate the expected arguments for upload_single_image - :param file: The Path of a file to upload - :param visibility: A string of either "public" or "private" - :return: A dictionary of expected arguments for an assert call list - """ - # pylint: disable=duplicate-code - return { - "name": get_upload_name(file), - "filename": file.as_posix(), - "disk_format": "qcow2", - "container_format": "bare", - "wait": True, - "visibility": visibility, - # Metadata fields - "hw_scsi_model": "virtio-scsi", - "hw_disk_bus": "scsi", - "hw_vif_multiqueue_enable": "true", - } - - return _expected_args_helper - - -def test_upload_images_to_openstack_real_run(fake_upload_args): - """ - Tests that the upload_images_to_openstack function calls - upload_single_image with the expected arguments - """ - expected_files = [Path("file1"), Path("file2")] - - args = Args(dry_run=False, public=True, os_cloud="default") - with mock.patch("upload_all.upload_single_image") as mocked_upload_single_image: - upload_images_to_openstack(expected_files, args) - - expected = [ - call(args.os_cloud, fake_upload_args(file, "public")) for file in expected_files - ] - mocked_upload_single_image.assert_has_calls(expected, any_order=True) - - -def test_upload_images_with_private_annotation(fake_upload_args): - """ - Tests that the upload_images_to_openstack function calls - upload_single_image with the expected arguments if the public flag is not set - """ - args = Args(dry_run=False, public=False, os_cloud="foo") - with mock.patch("upload_all.upload_single_image") as mocked_upload_single_image: - upload_images_to_openstack([Path("file1")], args) - - assert mocked_upload_single_image.call_count == 1 - expected = [call(args.os_cloud, fake_upload_args(Path("file1"), "private"))] - assert mocked_upload_single_image.call_args_list == expected - - -def test_upload_single_image(): - """ - Tests the connection and image upload forwards the args as-is - """ - fake_args = {"foo": "bar", "baz": "qux"} - with mock.patch("openstack.connect") as mocked_openstack_connect: - mocked_openstack_connect.return_value.create_image = Mock() - upload_single_image("fake_cloud", fake_args) - - mocked_openstack_connect.assert_called_once_with(cloud="fake_cloud") - mocked_openstack_connect.return_value.create_image.assert_called_once_with( - **fake_args - ) - - -def test_get_upload_name(): - """ - Tests that the name generator returns the expected name - with the format capi-{file.name}-{today's date} - """ - input_name = "ubuntu-kube-1.23.4" - expected = f"capi-{input_name}-{datetime.today().strftime('%Y-%m-%d')}" - print(f"Expected: {expected}") - assert get_upload_name(Path(input_name)) == expected - - -@mock.patch("upload_all.glob_output_files") -@mock.patch("upload_all.upload_images_to_openstack") -def test_main_steps(mock_upload, mock_glob): - """ - Tests the main flow ordering of the script - """ - args = Args(dry_run=True, public=True, os_cloud="default") - main(args) - - mock_glob.assert_called_once() - mock_upload.assert_called_once_with(mock_glob.return_value, args) diff --git a/scripts/upload_all.py b/scripts/upload_all.py deleted file mode 100755 index 5b6f4a20..00000000 --- a/scripts/upload_all.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python3 -""" -Script to upload all CAPI images in k8s-image-builder to OpenStack -""" - -import argparse -import dataclasses -import logging -import sys -from datetime import datetime -from pathlib import Path -from typing import Dict, List - -import git -import openstack - - -@dataclasses.dataclass -class Args: - """ - Holds user arguments for the script - """ - - dry_run: bool - public: bool - os_cloud: str - - -# Find the root of the git repository -def glob_output_files() -> List[Path]: - """ - Globs the output files from the CAPI image builder - :return: A list of files to upload - """ - repo = git.Repo(".", search_parent_directories=True) - root_dir = Path(repo.working_tree_dir) - - # Append the full path to the CAPI output directory - capi_output_dir = root_dir / "k8s-image-builder" / "images" / "capi" / "output" - output_files = capi_output_dir.glob("**/*kube*") - - return [i for i in output_files if i.is_file()] - - -def parse_args(args) -> Args: - """ - Parse the user arguments - :param args: sys.argv[1:] arguments - :return: Parsed arguments as a dataclass - """ - parser = argparse.ArgumentParser(description="Upload CAPI images to OpenStack") - parser.add_argument( - "--dry-run", action="store_true", help="Do not actually upload the images" - ) - parser.add_argument("--public", action="store_true", help="Make the images public") - parser.add_argument( - "os_cloud", default="default", help="The OpenStack cloud to use" - ) - parsed = parser.parse_args(args) - return Args(**vars(parsed)) - - -def get_upload_name(file: Path) -> str: - """ - Generates the standard name for the uploaded image - :param file: The Path of a file to upload - :return: The name containing the date and original name - """ - return f"capi-{file.name}-{datetime.today().strftime('%Y-%m-%d')}" - - -def upload_images_to_openstack(files: List[Path], args: Args): - """ - Uploads the given files to OpenStack using the given arguments - :param files: A list of files to upload - :param args: The user args to drive the behaviour - """ - for file in files: - upload_args = { - "name": get_upload_name(file), - "filename": file.as_posix(), - "disk_format": "qcow2", - "container_format": "bare", - "wait": True, - "visibility": "public" if args.public else "private", - # Metadata fields - setting through kwargs invokes automatic type conversion - "hw_scsi_model": "virtio-scsi", - "hw_disk_bus": "scsi", - "hw_vif_multiqueue_enable": "true", - } - logging.debug("Upload args: %s", upload_args) - if args.dry_run: - logging.warning("Dry run, skipping upload") - else: - logging.info("Uploading %s...", file) - upload_single_image(args.os_cloud, upload_args) - logging.info("Uploaded %s", file) - - -def upload_single_image(os_cloud: str, upload_args: Dict): - """ - Uploads a single image to OpenStack - :param os_cloud: The OpenStack cloud to use - :param upload_args: The parsed OpenStack SDK Image args - """ - conn = openstack.connect(cloud=os_cloud) - conn.create_image(**upload_args) - - -def main(args: Args): - """ - Runs the main steps of the script to upload all images - :param args: Pre-parsed args from the user - """ - to_upload = glob_output_files() - upload_images_to_openstack(to_upload, args) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main(parse_args(sys.argv[1:]))