diff --git a/.github/workflows/build-push-api.yml b/.github/workflows/build-push-api.yml new file mode 100644 index 00000000..8579dd3d --- /dev/null +++ b/.github/workflows/build-push-api.yml @@ -0,0 +1,71 @@ +name: Build and push + +on: + workflow_call: + inputs: + commit_sha: + description: Commit SHA to deploy (leave blank for latest) + required: false + type: string + dockerfile: + description: Path to Dockerfile + required: false + type: string + image_name: + description: Image name + required: false + type: string + port: + description: Container port + required: false + type: string + target: + description: Multistage build target + required: false + type: string + +permissions: + id-token: write + +env: + GCP_PROJECT_ID: arxiv-development + +jobs: + build-push: + runs-on: ubuntu-latest + environment: development + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.commit_sha || github.sha }} + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v3 + with: + workload_identity_provider: ${{ vars.PROVIDER_NAME }} + service_account: ${{ vars.SERVICE_ACCOUNT }} + + - name: Set up Google Cloud SDK + uses: google-github-actions/setup-gcloud@v3 + + - name: Configure Docker + run: gcloud auth configure-docker + + - name: Build image, caching from latest + run: | + docker pull gcr.io/${{ env.GCP_PROJECT_ID }}/${{ inputs.image_name }}:latest + + BRANCH_TAG="$(printf "%.128s" "${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}" | sed 's/^[.-]*//')" + + docker build --target ${{ inputs.target }} --build-arg PORT=${{ inputs.port }} -f ${{ inputs.dockerfile }} \ + --cache-from gcr.io/${{ env.GCP_PROJECT_ID }}/${{ inputs.image_name }}:latest \ + -t gcr.io/${{ env.GCP_PROJECT_ID }}/${{ inputs.image_name }}:${{ inputs.commit_sha || github.sha }} \ + -t gcr.io/${{ env.GCP_PROJECT_ID }}/${{ inputs.image_name }}:${BRANCH_TAG} \ + ${{ github.ref_name == 'main' && format('-t gcr.io/{0}/{1}:latest', env.GCP_PROJECT_ID, inputs.image_name) || '' }} . + + - name: Push image + # only push if merged to main + # if: github.ref == "refs/heads/main" + run: docker push --all-tags gcr.io/${{ env.GCP_PROJECT_ID }}/${{ inputs.image_name }} diff --git a/.github/workflows/build-stats-api.yml b/.github/workflows/build-stats-api.yml deleted file mode 100644 index 4b97c7fe..00000000 --- a/.github/workflows/build-stats-api.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: Build and push stats-api - -run-name: "Build stats-api ${{ github.ref_name }} ${{ inputs.custom_sha || github.sha }}" - -on: - push: - branches: - - main - paths: - - "stats-api/**" - pull_request: - branches: - - main - paths: - - "stats-api/**" - workflow_dispatch: - inputs: - custom_sha: - description: 'Custom SHA (image rebuilds only)' - required: false - type: string - -permissions: - id-token: write - actions: write - -env: - DEPLOY_WORKFLOW: "deploy-stats-api.yml" - DOCKERFILE: "Dockerfile.api" - GCP_PROJECT_ID: "arxiv-development" - IMAGE_NAME: "stats-api" - PORT: "8080" - TARGET: "runtime" - -jobs: - format-check: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up terraform - uses: hashicorp/setup-terraform@v3 - - - name: Check version - run: terraform --version - - - name: Format check - run: terraform fmt -check - - build-push: - needs: format-check - runs-on: ubuntu-latest - - concurrency: - group: build-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.custom_sha || github.sha }} - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v3 - with: - workload_identity_provider: ${{ vars.ARXIV_DEVELOPMENT_PROVIDER_NAME }} - service_account: ${{ vars.ARXIV_DEVELOPMENT_SERVICE_ACCOUNT }} - - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v3 - - - name: Configure Docker - run: gcloud auth configure-docker - - - name: Build image, caching from latest - run: | - docker pull gcr.io/${{ env.GCP_PROJECT_ID }}/${{ env.IMAGE_NAME }}:latest - - BRANCH_TAG="$(printf "%.128s" "${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}" | sed 's/^[.-]*//')" - - docker build --target ${TARGET} --build-arg PORT=${PORT} -f ${DOCKERFILE} \ - --cache-from gcr.io/${{ env.GCP_PROJECT_ID }}/${{ env.IMAGE_NAME }}:latest \ - -t gcr.io/${{ env.GCP_PROJECT_ID }}/${{ env.IMAGE_NAME }}:${{ inputs.custom_sha || github.sha }} \ - -t gcr.io/${{ env.GCP_PROJECT_ID }}/${{ env.IMAGE_NAME }}:${BRANCH_TAG} \ - ${{ github.ref_name == 'main' && format('-t gcr.io/{0}/{1}:latest', env.GCP_PROJECT_ID, env.IMAGE_NAME) || '' }} . - - - name: Push image - run: docker push --all-tags gcr.io/${GCP_PROJECT_ID}/${IMAGE_NAME} - - # CI Deployment Routing ... manual builds of a commit are not CD candidates - - name: Deployment Routing - if: ${{ !inputs.custom_sha }} - uses: arXiv/iac-public/.github/actions/service-deployment-routing@main - with: - image_tag: ${{ github.sha }} - deploy_workflow: ${{ env.DEPLOY_WORKFLOW }} - workload_identity_provider: ${{ vars.ARXIV_DEVELOPMENT_PROVIDER_NAME }} - service_account: ${{ vars.ARXIV_DEVELOPMENT_SERVICE_ACCOUNT }} \ No newline at end of file diff --git a/.github/workflows/deploy-function.yml b/.github/workflows/deploy-function.yml index 01097ad7..64833d50 100644 --- a/.github/workflows/deploy-function.yml +++ b/.github/workflows/deploy-function.yml @@ -8,7 +8,7 @@ on: description: Name of the function directory (e.g. aggregate_hourly_downloads) required: true commit_sha: - description: "Commit SHA to deploy (leave blank for latest)" + description: Commit SHA to deploy (leave blank for latest) required: false type: string workflow_dispatch: @@ -18,7 +18,7 @@ on: description: Name of the function directory (e.g. aggregate_hourly_downloads) required: true commit_sha: - description: "Commit SHA to deploy (leave blank for latest)" + description: Commit SHA to deploy (leave blank for latest) required: false type: string environment: diff --git a/.github/workflows/deploy-stats-api.yml b/.github/workflows/deploy-stats-api.yml deleted file mode 100644 index d78ca611..00000000 --- a/.github/workflows/deploy-stats-api.yml +++ /dev/null @@ -1,139 +0,0 @@ -name: Deploy stats-api - -# Set custom run name to show project name in workflow runs list -run-name: "Deploy stats-api ${{ inputs.env }} ${{ inputs.image_tag }}" - -on: - workflow_dispatch: - inputs: - env: - description: "Environment" - required: true - type: string - default: "" - image_tag: - description: "Docker image tag (latest or the commit SHA)" - required: false - type: string - default: "latest" - -permissions: - contents: read - pull-requests: write - id-token: write - -env: - DOCKERFILE: "Dockerfile.api" - IMAGE_NAME: "stats-api" - PORT: "8080" - TARGET: "runtime" - -jobs: - format-check: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up terraform - uses: hashicorp/setup-terraform@v3 - - - name: Check version - run: terraform --version - - - name: Format check - run: terraform fmt -check - - deploy: - runs-on: ubuntu-latest - needs: format-check - environment: ${{ inputs.env }} - if: > - (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') && - ((inputs.env != 'production' && inputs.env != 'prod' && inputs.env != 'staging' && inputs.env != 'stage') || inputs.image_tag != 'latest') - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set environment variables - run: | - # Determine environment from inputs or GitHub environment - if [ -n "${{ inputs.env }}" ]; then - ENV_NAME="${{ inputs.env }}" - else - echo "Error: No environment specified" - exit 1 - fi - - REGION="${{ vars.GCP_REGION || 'us-central1' }}" - echo "ENVIRONMENT=$ENV_NAME" >> $GITHUB_ENV - echo "REGION=$REGION" >> $GITHUB_ENV - - # Set project name and terraform bucket based on environment - if [ "$ENV_NAME" = "production" ] || [ "$ENV_NAME" = "prod" ]; then - echo "PROJECT_NAME=arxiv-production" >> $GITHUB_ENV - echo "TERRAFORM_BUCKET=prod-arxiv-terraform-state" >> $GITHUB_ENV - TERRAFORM_ARGS="-var-file=envs/prod.tfvars" - elif [ "$ENV_NAME" = "staging" ] || [ "$ENV_NAME" = "stage" ]; then - echo "PROJECT_NAME=arxiv-stage" >> $GITHUB_ENV - echo "TERRAFORM_BUCKET=stage-arxiv-terraform-state" >> $GITHUB_ENV - TERRAFORM_ARGS="-var-file=envs/staging.tfvars" - elif [ "$ENV_NAME" = "development" ] || [ "$ENV_NAME" = "develop" ] || [ "$ENV_NAME" = "dev" ]; then - echo "PROJECT_NAME=arxiv-development" >> $GITHUB_ENV - echo "TERRAFORM_BUCKET=dev-arxiv-terraform-state" >> $GITHUB_ENV - TERRAFORM_ARGS="-var-file=envs/dev.tfvars" - else - # For custom environments, use the environment name as project name - echo "PROJECT_NAME=$ENV_NAME" >> $GITHUB_ENV - echo "TERRAFORM_BUCKET=$ENV_NAME-arxiv-terraform-state" >> $GITHUB_ENV - TERRAFORM_ARGS="-var-file=envs/dev.tfvars \ - -var=\"gcp_project_id=$ENV_NAME\" \ - -var=\"gcp_region=$REGION\"" - fi - - # Add image_path to TERRAFORM_ARGS for all environments - TERRAFORM_ARGS="$TERRAFORM_ARGS \ - -var=\"image_path=gcr.io/arxiv-development/${{env.IMAGE_NAME}}:${{inputs.image_tag}}\"" - - echo "TERRAFORM_ARGS=$TERRAFORM_ARGS" >> $GITHUB_ENV - - - name: Authenticate with arxiv-development - uses: google-github-actions/auth@v3 - with: - workload_identity_provider: ${{ vars.ARXIV_DEVELOPMENT_PROVIDER_NAME }} - service_account: ${{ vars.ARXIV_DEVELOPMENT_SERVICE_ACCOUNT }} - - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v3 - - - name: Get project number - run: | - echo "Getting project number for project: ${{ env.PROJECT_NAME }}" - PROJECT_NUMBER=$(gcloud projects describe ${{ env.PROJECT_NAME }} --format="value(projectNumber)") - echo "PROJECT_NUMBER=$PROJECT_NUMBER" >> $GITHUB_ENV - echo "Project number: $PROJECT_NUMBER" - - - name: Authenticate with target project workload identity - uses: google-github-actions/auth@v3 - with: - workload_identity_provider: projects/${{ env.PROJECT_NUMBER }}/locations/global/workloadIdentityPools/github/providers/github-actions-provider - service_account: github-actions-sa@${{ env.PROJECT_NAME }}.iam.gserviceaccount.com - project_id: ${{ env.PROJECT_NAME }} - - - name: Set up terraform - uses: hashicorp/setup-terraform@v3 - - - name: Check version - run: terraform --version - - - name: Initialize remote backend - run: terraform init -backend-config="bucket=${{ env.TERRAFORM_BUCKET }}" - working-directory: terraform/stats_api - - - name: Apply - run: | - echo "Running terraform apply..." - terraform apply ${{ env.TERRAFORM_ARGS }} \ - -auto-approve - working-directory: terraform/stats_api \ No newline at end of file diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..cf6bc99f --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,71 @@ +name: Deploy + +on: + workflow_call: + inputs: + service_name: + type: string + description: Name of the service directory (e.g. stats_api) + required: true + commit_sha: + description: Commit SHA to deploy (leave blank for latest) + required: false + type: string + workflow_dispatch: + inputs: + service_name: + type: string + description: Name of the service directory (e.g. stats_api) + required: true + commit_sha: + description: Commit SHA to deploy (leave blank for latest) + required: false + type: string + environment: + type: environment + description: Select the environment + +env: + IMAGE_URI: gcr.io/arxiv-development/${{ env.IMAGE_NAME }}:${{ inputs.commit_sha || github.sha }} + +permissions: + contents: read + id-token: write + +jobs: + deploy: + runs-on: ubuntu-latest + environment: ${{ inputs.environment || "development" }} + # only deploy if merged to main + # if: github.ref == "refs/heads/main" + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v3 + with: + workload_identity_provider: ${{ vars.PROVIDER_NAME }} + service_account: ${{ vars.SERVICE_ACCOUNT }} + + - name: Set up Google Cloud SDK + uses: google-github-actions/setup-gcloud@v3 + + - name: Set up terraform + uses: hashicorp/setup-terraform@v3 + + - name: Check version + run: terraform --version + + - name: Initialize remote backend + run: terraform init -backend-config="bucket=${{ vars.ENV_VAR }}-arxiv-terraform-state" + working-directory: "terraform/${{ inputs.service_name }}" + + - name: Format check + run: terraform fmt -check + working-directory: "terraform/${{ inputs.service_name }}" + + - name: Apply + run: terraform apply -var-file=envs/${{ vars.ENV_VAR }}.tfvars -var="image_path=${{ env.IMAGE_PATH }}" -auto-approve + working-directory: "terraform/${{ inputs.service_name }}" diff --git a/.github/workflows/lint-and-test-api.yml b/.github/workflows/lint-test-api.yml similarity index 82% rename from .github/workflows/lint-and-test-api.yml rename to .github/workflows/lint-test-api.yml index 8fdcabb3..78428d6c 100644 --- a/.github/workflows/lint-and-test-api.yml +++ b/.github/workflows/lint-test-api.yml @@ -1,11 +1,7 @@ -name: Lint and test stats-api +name: Lint and test stats api on: - pull_request: - branches: - - main - paths: - - "stats-api/**" + workflow_call: workflow_dispatch: permissions: @@ -19,7 +15,7 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: "Set up Python" + - name: Set up Python uses: actions/setup-python@v5 with: python-version-file: "stats-api/pyproject.toml" diff --git a/.github/workflows/stats-api-triggers.yml b/.github/workflows/stats-api-triggers.yml new file mode 100644 index 00000000..c94cdca5 --- /dev/null +++ b/.github/workflows/stats-api-triggers.yml @@ -0,0 +1,59 @@ +name: Stats API triggers + +on: + push: + branches: + - main + paths: + - "stats-api/stats_api/**" + - "terraform/stats_api/**" + - ".github/workflows/deploy.yml" + - ".github/workflows/build-push-api.yml" + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + paths: + - "stats-api/stats_api/**" + - "terraform/stats_api/**" + - ".github/workflows/lint-test-api.yml" + - ".github/workflows/build-push-api.yml" + workflow_dispatch: + inputs: + commit_sha: + description: Commit SHA to deploy (leave blank for latest) + required: false + type: string + jobs_to_run: + description: Job(s) to run + required: true + type: choice + options: + - "build and push" + - "build, push, deploy" + - "deploy only" + +jobs: + call-lint-test: + name: Stats API ${{ github.ref_name }} ${{ github.sha }} + uses: ./.github/workflows/lint-test-api.yml + + call-build-push: + if: ${{ github.event_name != "workflow_dispatch" || inputs.jobs_to_run == "build and push" || inputs.jobs_to_run == "build, push, deploy" }} + name: Stats API ${{ github.ref_name }} ${{ github.sha }} + uses: ./.github/workflows/build-push-api.yml + with: + service_name: stats_api + dockerfile: Dockerfile.api + image_name: stats-api + port: "8080" + target: runtime + commit_sha: ${{ inputs.commit_sha || github.sha }} + + call-deploy: + if: ${{ github.event_name != "workflow_dispatch" || inputs.jobs_to_run == "build, push, deploy" || inputs.jobs_to_run == "deploy only" }} + name: Stats API ${{ github.ref_name }} ${{ github.sha }} + uses: ./.github/workflows/deploy.yml + with: + service_name: stats_api + commit_sha: ${{ inputs.commit_sha || github.sha }} diff --git a/stats-api/stats_api/factory.py b/stats-api/stats_api/factory.py index 166cc837..c3962659 100644 --- a/stats-api/stats_api/factory.py +++ b/stats-api/stats_api/factory.py @@ -38,4 +38,4 @@ def create_app(): host=app.config["HOST"], port=app.config["PORT"], debug=app.config["DEBUG"], - ) + ) # test diff --git a/terraform/monthly_submissions/main.tf b/terraform/monthly_submissions/main.tf index 066ac3c3..ce945054 100644 --- a/terraform/monthly_submissions/main.tf +++ b/terraform/monthly_submissions/main.tf @@ -57,7 +57,7 @@ resource "google_project_iam_member" "cloudsql_client" { resource "google_cloudfunctions2_function" "function" { name = "stats-monthly-submissions" # name should use kebab-case so generated Cloud Run service name will be the same - location = var.gcp_region # needs to be explicitly declared for Cloud Run + location = var.gcp_region # needs to be explicitly declared for Cloud Run description = "Cloud function to sum submissions and persist to a database" build_config { diff --git a/terraform/stats_api/main.tf b/terraform/stats_api/main.tf index fd63216c..bf28d22c 100644 --- a/terraform/stats_api/main.tf +++ b/terraform/stats_api/main.tf @@ -59,8 +59,6 @@ resource "google_cloud_run_v2_service" "stats_api" { name = "stats-api" location = var.gcp_region - deletion_protection = false - template { service_account = google_service_account.account.email containers {