From 4d05a10c37fee8caba7484ec8464c549745238c4 Mon Sep 17 00:00:00 2001 From: Steven Moy Date: Wed, 19 Jul 2023 20:05:41 -0700 Subject: [PATCH 1/2] Prototype access_rules diff for roles. access_rules for AWS IAM Roles is pure metadata changes. This metadata change is import for third party support. This is an experimental feature to calculate such metadata changes. This is still considered experimental. --- .vscode/launch.json | 3 +- iambic/request_handler/git_apply.py | 64 ++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index df1aacdc7..b4a00155e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -120,7 +120,8 @@ "module": "iambic.main", "console": "integratedTerminal", "args": [ - "git-plan", + "plan", + "--git-aware", ], "envFile": "${workspaceFolder}/.env", "justMyCode": true diff --git a/iambic/request_handler/git_apply.py b/iambic/request_handler/git_apply.py index 630819b57..0fe69ce0a 100644 --- a/iambic/request_handler/git_apply.py +++ b/iambic/request_handler/git_apply.py @@ -3,7 +3,9 @@ import itertools import os.path import uuid +from io import StringIO +from deepdiff import DeepDiff from git import Repo from iambic.config.dynamic_config import load_config @@ -14,8 +16,15 @@ ) from iambic.core.iambic_enum import Command from iambic.core.logger import log -from iambic.core.models import BaseTemplate, ExecutionMessage, TemplateChangeDetails +from iambic.core.models import ( + BaseTemplate, + ExecutionMessage, + ProposedChange, + ProposedChangeType, + TemplateChangeDetails, +) from iambic.core.parser import load_templates +from iambic.core.utils import yaml from iambic.request_handler.expire_resources import flag_expired_resources @@ -92,6 +101,10 @@ async def apply_git_changes( itertools.chain(new_templates, deleted_templates, modified_templates_doubles), ) + # Crutch to compute metadata changes + metadata_changes = compute_metadata_changes(config, file_changes["modified_files"]) + template_changes.extend(metadata_changes) + # note modified_templates_exist_in_repo has different entries from create_templates_for_modified_files because # create_templates_for_modified_files actually has two template instance per a single modified file modified_templates_exist_in_repo = load_templates( @@ -105,6 +118,55 @@ async def apply_git_changes( return template_changes +def compute_metadata_changes(config, modified_files): + changes = [] + for git_diff in modified_files: + old_template_dict = yaml.load(StringIO(git_diff.content)) + template_type_string = old_template_dict["template_type"] + template_cls = config.template_map.get(template_type_string, None) + + if template_cls is None: + # well the case is the previous version is an unknown config type now. + # this is typically the config file + log_params = {"template_type": template_type_string} + log.warning( + "template_type is not registered among template_map", **log_params + ) + continue + + old_template = template_cls(file_path=git_diff.path, **old_template_dict) + new_template = load_templates([git_diff.path], config.template_map)[0] + + if template_type_string != "NOQ::AWS::IAM::Role": + continue + + new_access_rules = getattr(new_template, "access_rules", []) + old_access_rules = getattr(old_template, "access_rules", []) + diff = DeepDiff( + old_access_rules, + new_access_rules, + ignore_order=True, + report_repetition=True, + ) + if diff: + propose_change = ProposedChange( + change_type=ProposedChangeType.UPDATE, + attribute="access_rules", + resource_id=new_template.resource_id, + resource_type=new_template.resource_type, + change_summary=diff, + ) + template_change_details = TemplateChangeDetails( + resource_id=new_template.resource_id, + resource_type=new_template.template_type, + template_path=new_template.file_path, + proposed_changes=[propose_change], + ) + changes.append(template_change_details) + + return changes + + def commit_deleted_templates( repo_dir: str, templates: list[BaseTemplate], details: list[TemplateChangeDetails] ): From 43ebdcc86f90ac2cb65300aefb91fc81e2121b91 Mon Sep 17 00:00:00 2001 From: Steven Moy Date: Mon, 24 Jul 2023 13:10:01 -0700 Subject: [PATCH 2/2] Update iambic_base_container to 2.0.0 * Upgrade base linux distro to Amazon Linux 2023 * Upgrade Python to 3.10.12 --- .github/workflows/build-base-container.yml | 41 ++++++++++++++++++++++ Dockerfile.base_image | 9 +++-- 2 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/build-base-container.yml diff --git a/.github/workflows/build-base-container.yml b/.github/workflows/build-base-container.yml new file mode 100644 index 000000000..1cd96351e --- /dev/null +++ b/.github/workflows/build-base-container.yml @@ -0,0 +1,41 @@ +name: Build base container +# This builds the base container upon a push event in noqdev/iambic repository +on: + # Testing this is burdensome + # One way to test is to enable event type pull_request; however, our setup + # is restrictive on who can assume the container building role. You can consider + # temporarily opening the role to allow the pull_request or via specific branch to + # do the image building in the aws assume role side. + # pull_request: + workflow_dispatch + +jobs: + build_base_container: + if: ${{ github.repository == 'noqdev/iambic' && github.ref == 'refs/heads/main' }} + runs-on: ubuntu-latest + name: Build Base Container + permissions: + id-token: write + contents: write + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + # - linux/arm64 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::242345320040:role/iambic_image_builder + aws-region: us-east-1 + - name: build container + id: build-container + run: | + docker logout ghcr.io + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/iambic + make build_docker_base_image upload_docker_base_image + docker logout public.ecr.aws/iambic + docker buildx prune --filter=until=96h -f \ No newline at end of file diff --git a/Dockerfile.base_image b/Dockerfile.base_image index 2914d897e..5d1b71ce2 100644 --- a/Dockerfile.base_image +++ b/Dockerfile.base_image @@ -1,7 +1,6 @@ ARG FUNCTION_DIR="/app" -ARG AWS_LINUX_VERSION="2022" -ARG PYTHON_VERSION="3.10.8" -ARG NPROC=8 +ARG AWS_LINUX_VERSION="2023" +ARG PYTHON_VERSION="3.10.12" ARG ARCH= FROM ${ARCH}amazonlinux:${AWS_LINUX_VERSION} as python-layer @@ -11,14 +10,14 @@ ARG PYTHON_VERSION # install python RUN yum update -y \ && yum groupinstall "Development Tools" -y \ - && yum install libffi-devel bzip2-devel wget openssl openssl-devel sqlite-devel coreutils -y + && yum install libffi-devel bzip2-devel wget openssl openssl-devel sqlite-devel coreutils -y --allowerasing # breaking up this line to test building python using all the cores RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \ && tar -xf Python-${PYTHON_VERSION}.tgz \ && cd Python-${PYTHON_VERSION}/ \ && ./configure --enable-optimizations --enable-loadable-sqlite-extensions \ - && make install && ln -s /Python-${PYTHON_VERSION}/python /usr/bin/python \ + && make -j install && ln -s /Python-${PYTHON_VERSION}/python /usr/bin/python \ && python -m ensurepip --upgrade \ && python -m pip install --upgrade pip