From f3f5d51907991b68c4694f8056d0997a3bef6680 Mon Sep 17 00:00:00 2001 From: Nivesh Mittapally Date: Wed, 31 May 2023 14:13:42 +0530 Subject: [PATCH 1/5] feat: added action for linter check --- .github/workflows/linter.yml | 14 +++++ Dockerfile | 47 ++++++++------- Makefile | 55 ++++++++++++------ config.json.sample | 108 +++++++++++++++++++++++++++++++++++ docker-compose.yml | 1 - 5 files changed, 184 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/linter.yml create mode 100644 config.json.sample diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..ee85131 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,14 @@ +name: Lint +on: [push] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup docker-compose + uses: KengoTODA/actions-setup-docker-compose@v1.0.9 + - name: Setup make + run: | + sudo apt-get install make + - name: Run linter + run: make lint diff --git a/Dockerfile b/Dockerfile index 652d066..d60cfb9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,30 +13,33 @@ RUN DEBIAN_FRONTEND=noninteractive \ && apt-get autoremove -y # Set env variables used in this Dockerfile (add a unique prefix, such as DEV) -RUN apt update && apt install -y netcat dnsutils - -RUN useradd -rm -d /home/app -s /bin/bash -g root -G sudo -u 1001 app +RUN apt update && apt install -y netcat dnsutils libmariadbclient-dev git + +RUN mkdir -p /ebs/logs && touch /ebs/logs/engima.log && chmod 777 /ebs/logs/engima.log + +ARG APPUID=1001 +RUN useradd -rm -d /home/app -s /bin/bash -g root -G sudo -u "$APPUID" app +WORKDIR /srv/code/dev +RUN git clone https://github.com/browserstack/enigma.git . +RUN mkdir -p Access/access_modules +COPY config.json.sample config.json +RUN cp requirements.txt /tmp/ +RUN mkdir -p logs +RUN mkdir -p db +RUN chown -R app /srv/code/dev /ebs USER app -# Directory in container for all project files -ENV DEV_SRVHOME=/srv - -# Local directory with project source -ENV DEV_SRC=code/dev - -# Directory in container for project source files -ENV DEV_SRVPROJ=$DEV_SRVHOME/$DEV_SRC - -# Create application subdirectories -WORKDIR $DEV_SRVPROJ - -# Copy just requirements.txt -COPY requirements.txt /tmp/requirements.txt - -# Install Python dependencies -RUN pip install -r /tmp/requirements.txt --no-cache-dir - -COPY . . +COPY requirements.txt /tmp/access-module-requirements.txt +RUN pip install -r /tmp/requirements.txt --no-cache-dir --ignore-installed +RUN pip install -r /tmp/access-module-requirements.txt --no-cache-dir --ignore-installed +COPY --chown=app:root aws_access ./Access/access_modules/aws_access +COPY --chown=app:root confluence ./Access/access_modules/confluence +COPY --chown=app:root gcp ./Access/access_modules/gcp +COPY --chown=app:root github_access ./Access/access_modules/github_access +COPY --chown=app:root opsgenie_access ./Access/access_modules/opsgenie_access +COPY --chown=app:root slack_access ./Access/access_modules/slack_access +COPY --chown=app:root ssh ./Access/access_modules/ssh +COPY --chown=app:root zoom_access ./Access/access_modules/zoom_access # Starts Docker Container and keeps it running for Debugging FROM base as test diff --git a/Makefile b/Makefile index b112cf6..1c3ba66 100644 --- a/Makefile +++ b/Makefile @@ -1,39 +1,58 @@ +APP_UID := $(shell id -u) + ## make all : Run service, test and linter .PHONY: all all: test lint .PHONY: build +build: export APPUID = $(APP_UID) build: - @docker-compose up -d + @docker-compose up --build -d .PHONY: down down: @docker-compose -f docker-compose.yml down -## Run tests with coverage -.PHONY: test -test: - @if [ $$(docker ps -f name=test | wc -l) -eq 2 ]; then \ - docker exec test python -m pytest --version; \ +ensure_container_for_test: + @if [ $$(docker ps -a -f name=test | wc -l) -eq 2 ]; then \ + docker exec test python -m pytest --version; \ else \ - echo "No containers running.. Starting runserver:"; \ + echo "No containers running.. "; \ make build; \ - echo "Running Tests"; \ fi - @docker exec test python -m pytest -v --cov --disable-warnings;\ - echo "Tests finished. Stopping runserver:" && make down +## Run tests with coverage +.PHONY: test +test: export APPUID = $(APP_UID) +test: ensure_container_for_test -## Create lint issues file -.PHONY: lint_issues -lint_issues: - @touch $@ + @docker exec test python -m pytest -v --cov --disable-warnings Access/access_modules;\ + echo "Tests finished. Stopping runserver:" -## Lint code using pylama skipping files in env (if pyenv created) .PHONY: lint -lint: lint_issues - @python3 -m pylama --version - @pylama --skip "./env/*" -r lint_issues || echo "Linter run returned errors. Check lint_issues file for details." && false +lint: export APPUID = $(APP_UID) +lint: ensure_container_for_test + @docker exec test python -m pylama --version + @docker exec test python -m pylama Access/access_modules + @if [ "$$?" -ne 0 ]; then \ + echo "Linter checks failed"; \ + exit 1; \ + else \ + echo "Linter checks passed"; \ + fi + + +## Lint code using pylama skipping files in env (if pyenv created) +# .PHONY: lint +# lint: +# @python3 -m pylama --version +# @python3 -m pylama +# @if [ "$$?" -ne 0 ]; then \ +# echo "Linter checks failed"; \ +# exit 1; \ +# else \ +# echo "Linter checks passed"; \ +# fi run_semgrep: $(shell semgrep --error --config "p/cwe-top-25" --config "p/owasp-top-ten" --config "p/r2c-security-audit") diff --git a/config.json.sample b/config.json.sample new file mode 100644 index 0000000..e15444b --- /dev/null +++ b/config.json.sample @@ -0,0 +1,108 @@ +{ + "django_setup": { + "SECRET_KEY": "random_secret_that_you_should_change_on_production", + "DEBUG": false, + "ALLOWED_HOSTS": [ + "localhost" + ], + "CSRF_TRUSTED_ORIGINS": [ + "http://localhost" + ] + }, + "sso": { + "googleapi": { + "SOCIAL_AUTH_GOOGLE_OAUTH2_KEY": "", + "SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET": "" + } + }, + "database": { + "engine": "sqlite3", + "dbname": "", + "username": "", + "password": "", + "host": "", + "port": 3306 + }, + "access_modules": { + "git_urls": [ + "https://github.com/browserstack/enigma-access-modules.git" + ], + "confluence_module": { + "CONFLUENCE_BASE_URL": "https://test.atlassian.net", + "ADMIN_EMAIL": "test@browserstack.com", + "API_TOKEN": "xyz" + }, + "aws_access": { + "aws_accounts": [ + { + "account": "Dev", + "access_key_id": "id", + "secret_access_key": "key" + } + ] + }, + "github_access": { + "GITHUB_TOKEN": "test-token", + "GITHUB_BASE_URL": "https://api.github.com", + "GITHUB_ORG": "browserstack" + }, + "gcp_access": { + "domains": [ + { + "domain_id": "browserstack.com", + "admin_id": "test@browserstack.com", + "service_account_path": "./gcp.json" + } + ] + }, + "zoom_access": { + "ZOOM_BASE_URL": "https://api.zoom.us/v2/", + "ZOOM_CLIENT_SECRET": "test-secret", + "ZOOM_API_KEY": "test-api" + }, + "ssh": { + "engima_root_user": "enigma-user", + "app_user": "app", + "inventory_file_path": "Access/access_modules/ssh/inventory.csv", + "common_sudo_group": "sudoppl", + "private_key_path": "/Users/username/.ssh/id_rsa" + }, + "opsgenie_access": { + "OPSGENIE_TOKEN": "test-token", + "IGNORE_TEAMS": [ + "team_1", + "team_2" + ] + }, + "slack_access": { + "enigma-slack": { + "AUTH_TOKEN": "abc", + "DEFAULT_CHANNELS": [ + "general" + ] + } + } + }, + "enigmaGroup": { + "MAIL_APPROVER_GROUPS": [] + }, + "emails": { + "access-approve": "", + "EMAIL_HOST": "", + "EMAIL_PORT": "", + "EMAIL_HOST_USER": "", + "EMAIL_HOST_PASSWORD": "", + "EMAIL_USE_TLS": true, + "EMAIL_USE_SSL": false, + "DEFAULT_FROM_EMAIL": "" + }, + "background_task_manager": { + "type": "celery", + "config": { + "broker": "", + "backend": "", + "need_monitoring": true, + "monitoring_apps": "django_celery_results" + } + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 97b1faa..1883688 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,5 +10,4 @@ services: env_file: - ./secrets/ops_app_test.env volumes: - - .:/srv/code/dev - ./mounts/logs/:/ebs/logs/ From c6dcb9d1a39ef69b9d91bc184704f8fa331c55c8 Mon Sep 17 00:00:00 2001 From: Nivesh Mittapally Date: Wed, 31 May 2023 14:25:13 +0530 Subject: [PATCH 2/5] refactor: linter github action --- .github/workflows/linter.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index ee85131..d7b4db8 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -5,8 +5,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Setup docker-compose - uses: KengoTODA/actions-setup-docker-compose@v1.0.9 - name: Setup make run: | sudo apt-get install make From 29344b7be869b9e2d267d816e209b5bf560423bf Mon Sep 17 00:00:00 2001 From: Nivesh Mittapally Date: Wed, 31 May 2023 15:57:46 +0530 Subject: [PATCH 3/5] feat: added unit test action --- .github/workflows/linter.yml | 2 ++ .github/workflows/unit-tests.yml | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index d7b4db8..f4b017f 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -8,5 +8,7 @@ jobs: - name: Setup make run: | sudo apt-get install make + - name: Build docker container + run: make build - name: Run linter run: make lint diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000..e57aae6 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,14 @@ +name: Unit Tests +on: [push] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup make + run: | + sudo apt-get install make + - name: Build docker container + run: make build + - name: Run unit tests + run: make test From c6ab895ed1f7c4169331309927faf2bd6427626a Mon Sep 17 00:00:00 2001 From: Nivesh Mittapally Date: Wed, 31 May 2023 16:12:16 +0530 Subject: [PATCH 4/5] fix: make test execution --- Makefile | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 1c3ba66..d40f04e 100644 --- a/Makefile +++ b/Makefile @@ -26,8 +26,13 @@ ensure_container_for_test: test: export APPUID = $(APP_UID) test: ensure_container_for_test - @docker exec test python -m pytest -v --cov --disable-warnings Access/access_modules;\ - echo "Tests finished. Stopping runserver:" + @docker exec test python -m pytest -v --cov --disable-warnings Access/access_modules + @if [ "$$?" -ne 0 ]; then \ + echo "Unit Tests failed"; \ + exit 1; \ + else \ + echo "Unit Tests passed"; \ + fi .PHONY: lint lint: export APPUID = $(APP_UID) @@ -41,18 +46,5 @@ lint: ensure_container_for_test echo "Linter checks passed"; \ fi - -## Lint code using pylama skipping files in env (if pyenv created) -# .PHONY: lint -# lint: -# @python3 -m pylama --version -# @python3 -m pylama -# @if [ "$$?" -ne 0 ]; then \ -# echo "Linter checks failed"; \ -# exit 1; \ -# else \ -# echo "Linter checks passed"; \ -# fi - run_semgrep: $(shell semgrep --error --config "p/cwe-top-25" --config "p/owasp-top-ten" --config "p/r2c-security-audit") From 3418c25be6b539c5f369cfad3360be9cfcdc5458 Mon Sep 17 00:00:00 2001 From: Nivesh Mittapally Date: Wed, 31 May 2023 18:58:40 +0530 Subject: [PATCH 5/5] fix: docker volume to syck code --- Dockerfile | 9 +-------- docker-compose.yml | 1 + 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index d60cfb9..73b1bfd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,14 +32,7 @@ USER app COPY requirements.txt /tmp/access-module-requirements.txt RUN pip install -r /tmp/requirements.txt --no-cache-dir --ignore-installed RUN pip install -r /tmp/access-module-requirements.txt --no-cache-dir --ignore-installed -COPY --chown=app:root aws_access ./Access/access_modules/aws_access -COPY --chown=app:root confluence ./Access/access_modules/confluence -COPY --chown=app:root gcp ./Access/access_modules/gcp -COPY --chown=app:root github_access ./Access/access_modules/github_access -COPY --chown=app:root opsgenie_access ./Access/access_modules/opsgenie_access -COPY --chown=app:root slack_access ./Access/access_modules/slack_access -COPY --chown=app:root ssh ./Access/access_modules/ssh -COPY --chown=app:root zoom_access ./Access/access_modules/zoom_access +COPY --chown=app:root . ./Access/access_modules # Starts Docker Container and keeps it running for Debugging FROM base as test diff --git a/docker-compose.yml b/docker-compose.yml index 1883688..20e91d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,3 +11,4 @@ services: - ./secrets/ops_app_test.env volumes: - ./mounts/logs/:/ebs/logs/ + - ./:/srv/code/dev/Access/access_modules/