diff --git a/.github/workflows/build-base-container.yml b/.github/workflows/build-base-container.yml new file mode 100644 index 000000000..1cd96351e --- /dev/null +++ b/.github/workflows/build-base-container.yml @@ -0,0 +1,41 @@ +name: Build base container +# This builds the base container upon a push event in noqdev/iambic repository +on: + # Testing this is burdensome + # One way to test is to enable event type pull_request; however, our setup + # is restrictive on who can assume the container building role. You can consider + # temporarily opening the role to allow the pull_request or via specific branch to + # do the image building in the aws assume role side. + # pull_request: + workflow_dispatch + +jobs: + build_base_container: + if: ${{ github.repository == 'noqdev/iambic' && github.ref == 'refs/heads/main' }} + runs-on: ubuntu-latest + name: Build Base Container + permissions: + id-token: write + contents: write + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + # - linux/arm64 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::242345320040:role/iambic_image_builder + aws-region: us-east-1 + - name: build container + id: build-container + run: | + docker logout ghcr.io + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/iambic + make build_docker_base_image upload_docker_base_image + docker logout public.ecr.aws/iambic + docker buildx prune --filter=until=96h -f \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index df1aacdc7..b4a00155e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -120,7 +120,8 @@ "module": "iambic.main", "console": "integratedTerminal", "args": [ - "git-plan", + "plan", + "--git-aware", ], "envFile": "${workspaceFolder}/.env", "justMyCode": true diff --git a/Dockerfile.base_image b/Dockerfile.base_image index 2914d897e..5d1b71ce2 100644 --- a/Dockerfile.base_image +++ b/Dockerfile.base_image @@ -1,7 +1,6 @@ ARG FUNCTION_DIR="/app" -ARG AWS_LINUX_VERSION="2022" -ARG PYTHON_VERSION="3.10.8" -ARG NPROC=8 +ARG AWS_LINUX_VERSION="2023" +ARG PYTHON_VERSION="3.10.12" ARG ARCH= FROM ${ARCH}amazonlinux:${AWS_LINUX_VERSION} as python-layer @@ -11,14 +10,14 @@ ARG PYTHON_VERSION # install python RUN yum update -y \ && yum groupinstall "Development Tools" -y \ - && yum install libffi-devel bzip2-devel wget openssl openssl-devel sqlite-devel coreutils -y + && yum install libffi-devel bzip2-devel wget openssl openssl-devel sqlite-devel coreutils -y --allowerasing # breaking up this line to test building python using all the cores RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \ && tar -xf Python-${PYTHON_VERSION}.tgz \ && cd Python-${PYTHON_VERSION}/ \ && ./configure --enable-optimizations --enable-loadable-sqlite-extensions \ - && make install && ln -s /Python-${PYTHON_VERSION}/python /usr/bin/python \ + && make -j install && ln -s /Python-${PYTHON_VERSION}/python /usr/bin/python \ && python -m ensurepip --upgrade \ && python -m pip install --upgrade pip diff --git a/iambic/request_handler/git_apply.py b/iambic/request_handler/git_apply.py index 630819b57..0fe69ce0a 100644 --- a/iambic/request_handler/git_apply.py +++ b/iambic/request_handler/git_apply.py @@ -3,7 +3,9 @@ import itertools import os.path import uuid +from io import StringIO +from deepdiff import DeepDiff from git import Repo from iambic.config.dynamic_config import load_config @@ -14,8 +16,15 @@ ) from iambic.core.iambic_enum import Command from iambic.core.logger import log -from iambic.core.models import BaseTemplate, ExecutionMessage, TemplateChangeDetails +from iambic.core.models import ( + BaseTemplate, + ExecutionMessage, + ProposedChange, + ProposedChangeType, + TemplateChangeDetails, +) from iambic.core.parser import load_templates +from iambic.core.utils import yaml from iambic.request_handler.expire_resources import flag_expired_resources @@ -92,6 +101,10 @@ async def apply_git_changes( itertools.chain(new_templates, deleted_templates, modified_templates_doubles), ) + # Crutch to compute metadata changes + metadata_changes = compute_metadata_changes(config, file_changes["modified_files"]) + template_changes.extend(metadata_changes) + # note modified_templates_exist_in_repo has different entries from create_templates_for_modified_files because # create_templates_for_modified_files actually has two template instance per a single modified file modified_templates_exist_in_repo = load_templates( @@ -105,6 +118,55 @@ async def apply_git_changes( return template_changes +def compute_metadata_changes(config, modified_files): + changes = [] + for git_diff in modified_files: + old_template_dict = yaml.load(StringIO(git_diff.content)) + template_type_string = old_template_dict["template_type"] + template_cls = config.template_map.get(template_type_string, None) + + if template_cls is None: + # well the case is the previous version is an unknown config type now. + # this is typically the config file + log_params = {"template_type": template_type_string} + log.warning( + "template_type is not registered among template_map", **log_params + ) + continue + + old_template = template_cls(file_path=git_diff.path, **old_template_dict) + new_template = load_templates([git_diff.path], config.template_map)[0] + + if template_type_string != "NOQ::AWS::IAM::Role": + continue + + new_access_rules = getattr(new_template, "access_rules", []) + old_access_rules = getattr(old_template, "access_rules", []) + diff = DeepDiff( + old_access_rules, + new_access_rules, + ignore_order=True, + report_repetition=True, + ) + if diff: + propose_change = ProposedChange( + change_type=ProposedChangeType.UPDATE, + attribute="access_rules", + resource_id=new_template.resource_id, + resource_type=new_template.resource_type, + change_summary=diff, + ) + template_change_details = TemplateChangeDetails( + resource_id=new_template.resource_id, + resource_type=new_template.template_type, + template_path=new_template.file_path, + proposed_changes=[propose_change], + ) + changes.append(template_change_details) + + return changes + + def commit_deleted_templates( repo_dir: str, templates: list[BaseTemplate], details: list[TemplateChangeDetails] ):