diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml new file mode 100644 index 0000000..5969d96 --- /dev/null +++ b/.github/workflows/nightly-tests.yml @@ -0,0 +1,55 @@ +name: Nightly Full Test Suite + +on: + schedule: + # Run at 6:00 AM UTC (1:00 AM EST) daily + - cron: '0 6 * * *' + workflow_dispatch: + +env: + CI: true + +jobs: + full-unit-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set Up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Cache node_modules + id: cache-node-modules + uses: actions/cache@v4 + with: + path: node_modules + key: node-modules-${{ runner.os }}-node20-${{ hashFiles('package-lock.json') }} + + - name: Validate cached node_modules + id: validate-cache + if: steps.cache-node-modules.outputs.cache-hit == 'true' + run: | + if npm ls --depth=0 >/dev/null 2>&1; then + echo "valid=true" >> $GITHUB_OUTPUT + else + echo "valid=false" >> $GITHUB_OUTPUT + rm -rf node_modules + fi + + - name: Install Dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' || steps.validate-cache.outputs.valid == 'false' + run: npm ci + + - name: Run full unit test suite + run: npm run test:unit:coverage + + - name: Upload Coverage + uses: actions/upload-artifact@v4 + if: always() + with: + name: nightly-unit-test-coverage + path: coverage/ + retention-days: 7 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 290359d..081589a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,83 +4,33 @@ on: pull_request: branches: - main - push: - branches: - - main workflow_dispatch: - inputs: - force_rebuild: - description: 'Force rebuild all images (ignore cache)' - required: false - default: false - type: boolean - -env: - TEST_HOST: http://localhost - CI_DB_PORT: 5433 - CI_PORT: 8081 - PORT: 8081 - CI_BETTER_AUTH_URL: http://localhost:8083/auth - BETTER_AUTH_URL: http://localhost:8083/auth - DB_USERNAME: test-user - DB_PASSWORD: test-pw - DB_NAME: wxyc_db - AUTH_BYPASS: true jobs: # Detect what changed to conditionally run jobs detect-changes: runs-on: ubuntu-latest outputs: - backend: ${{ steps.changes.outputs.backend }} - auth: ${{ steps.changes.outputs.auth }} + apps: ${{ steps.changes.outputs.apps }} shared: ${{ steps.changes.outputs.shared }} tests: ${{ steps.changes.outputs.tests }} - db-init: ${{ steps.changes.outputs.db-init }} - docs-only: ${{ steps.changes.outputs.docs-only }} src: ${{ steps.changes.outputs.src }} run-integration: ${{ steps.should-run.outputs.run-integration }} - merge_base_sha: ${{ steps.merge-base.outputs.sha }} steps: - name: Checkout Code uses: actions/checkout@v4 - with: - fetch-depth: 0 # Need full history for merge-base - - - name: Get merge-base SHA - id: merge-base - run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - # For PRs, get the merge-base between the PR and main - MERGE_BASE=$(git merge-base origin/${{ github.base_ref }} ${{ github.sha }}) - echo "sha=$MERGE_BASE" >> $GITHUB_OUTPUT - echo "Merge-base SHA: $MERGE_BASE" - else - # For pushes to main, use the current SHA - echo "sha=${{ github.sha }}" >> $GITHUB_OUTPUT - echo "Current SHA: ${{ github.sha }}" - fi - name: Detect file changes uses: dorny/paths-filter@v3 id: changes with: filters: | - backend: - - 'apps/backend/**' - auth: - - 'apps/auth/**' + apps: + - 'apps/**' shared: - 'shared/**' tests: - 'tests/**' - db-init: - - 'dev_env/Dockerfile.init' - - 'dev_env/init-db.mjs' - - 'dev_env/package.init.json' - docs-only: - - '**/*.md' - - 'docs/**' src: - 'apps/**' - 'shared/**' @@ -92,10 +42,8 @@ jobs: - name: Determine if integration tests should run id: should-run run: | - FORCE_REBUILD="${{ inputs.force_rebuild }}" - if [[ "$FORCE_REBUILD" == "true" || \ - "${{ steps.changes.outputs.backend }}" == "true" || \ - "${{ steps.changes.outputs.auth }}" == "true" || \ + if [[ "${{ github.event_name }}" == "workflow_dispatch" || \ + "${{ steps.changes.outputs.apps }}" == "true" || \ "${{ steps.changes.outputs.shared }}" == "true" || \ "${{ steps.changes.outputs.tests }}" == "true" ]]; then echo "run-integration=true" >> $GITHUB_OUTPUT @@ -194,10 +142,9 @@ jobs: - name: Run affected unit tests run: | if [ "${{ github.event_name }}" = "pull_request" ]; then - # For PRs, run tests affected by changes since base branch npx jest --config jest.unit.config.ts --changedSince=origin/${{ github.base_ref }} --coverage --passWithNoTests else - # For pushes to main, run all unit tests + # workflow_dispatch: run all unit tests npm run test:unit:coverage fi @@ -209,13 +156,45 @@ jobs: path: coverage/ retention-days: 7 - # Integration tests - only when backend/auth/shared changes - # Note: Job always runs to report status, but steps are skipped when no relevant changes + # Integration tests - only when backend/auth/shared/tests change + # Note: Job always runs to report status, but steps are skipped when no relevant changes. + # The services block starts PostgreSQL even when skipped -- this is unavoidable with GHA + # service containers but harmless (starts in seconds, job exits immediately). Integration-Tests: - needs: detect-changes + needs: [detect-changes, lint-and-typecheck] + # Run when lint-and-typecheck succeeds or was skipped (no src changes). + # This ensures the required check reports status on docs-only PRs. + if: ${{ !failure() && !cancelled() }} runs-on: ubuntu-latest + services: + postgres: + image: postgres:18.0-alpine + env: + POSTGRES_USER: test-user + POSTGRES_PASSWORD: test-pw + POSTGRES_DB: wxyc_db + ports: + - 5433:5432 + options: >- + --health-cmd pg_isready + --health-interval 2s + --health-timeout 5s + --health-retries 10 env: RUN_TESTS: ${{ needs.detect-changes.outputs.run-integration }} + TEST_HOST: http://localhost + DB_HOST: localhost + DB_PORT: 5433 + DB_USERNAME: test-user + DB_PASSWORD: test-pw + DB_NAME: wxyc_db + PORT: 8081 + AUTH_PORT: 8083 + AUTH_BYPASS: true + BETTER_AUTH_URL: http://localhost:8083/auth + BETTER_AUTH_JWKS_URL: http://localhost:8083/auth/jwks + BETTER_AUTH_ISSUER: http://localhost:8083 + BETTER_AUTH_AUDIENCE: http://localhost:8083 steps: - name: Skip notification if: env.RUN_TESTS != 'true' @@ -225,118 +204,6 @@ jobs: if: env.RUN_TESTS == 'true' uses: actions/checkout@v4 - - name: Set up Docker Buildx - if: env.RUN_TESTS == 'true' - uses: docker/setup-buildx-action@v3 - - - name: Log in to Amazon ECR - if: env.RUN_TESTS == 'true' - id: login-ecr - continue-on-error: true - env: - AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin ${{ secrets.AWS_ECR_URI }} - - # db-init image: pull SHA-tagged image from ECR or build - - name: Pull db-init image from ECR - if: env.RUN_TESTS == 'true' && inputs.force_rebuild != true && needs.detect-changes.outputs.db-init == 'false' - id: pull-db-init - continue-on-error: true - env: - MERGE_BASE_SHA: ${{ needs.detect-changes.outputs.merge_base_sha }} - run: | - # Try SHA-tagged image first (most accurate), then fall back to :latest - SHA_TAG="sha-${MERGE_BASE_SHA}" - if docker pull ${{ secrets.AWS_ECR_URI }}/db-init:${SHA_TAG} 2>/dev/null; then - echo "✅ Pulled db-init image with SHA tag: ${SHA_TAG}" - docker tag ${{ secrets.AWS_ECR_URI }}/db-init:${SHA_TAG} wxyc-db-init-image - elif docker pull ${{ secrets.AWS_ECR_URI }}/db-init:latest 2>/dev/null; then - echo "⚠️ SHA-tagged image not found, using :latest (may be stale)" - docker tag ${{ secrets.AWS_ECR_URI }}/db-init:latest wxyc-db-init-image - else - echo "❌ No ECR image available, will build locally" - exit 1 - fi - - - name: Build db-init image - if: env.RUN_TESTS == 'true' && (inputs.force_rebuild == true || needs.detect-changes.outputs.db-init == 'true' || steps.pull-db-init.outcome == 'failure') - uses: docker/build-push-action@v6 - with: - context: . - file: dev_env/Dockerfile.init - load: true - push: false - tags: wxyc-db-init-image - cache-from: type=gha,scope=db-init - cache-to: type=gha,mode=max,scope=db-init - - # auth image: pull SHA-tagged image from ECR or build - - name: Pull auth image from ECR - if: env.RUN_TESTS == 'true' && inputs.force_rebuild != true && needs.detect-changes.outputs.auth == 'false' && needs.detect-changes.outputs.shared == 'false' - id: pull-auth - continue-on-error: true - env: - MERGE_BASE_SHA: ${{ needs.detect-changes.outputs.merge_base_sha }} - run: | - SHA_TAG="sha-${MERGE_BASE_SHA}" - if docker pull ${{ secrets.AWS_ECR_URI }}/auth:${SHA_TAG} 2>/dev/null; then - echo "✅ Pulled auth image with SHA tag: ${SHA_TAG}" - docker tag ${{ secrets.AWS_ECR_URI }}/auth:${SHA_TAG} wxyc_auth_service:ci - elif docker pull ${{ secrets.AWS_ECR_URI }}/auth:latest 2>/dev/null; then - echo "⚠️ SHA-tagged image not found, using :latest (may be stale)" - docker tag ${{ secrets.AWS_ECR_URI }}/auth:latest wxyc_auth_service:ci - else - echo "❌ No ECR image available, will build locally" - exit 1 - fi - - - name: Build auth image - if: env.RUN_TESTS == 'true' && (inputs.force_rebuild == true || needs.detect-changes.outputs.auth == 'true' || needs.detect-changes.outputs.shared == 'true' || steps.pull-auth.outcome == 'failure') - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile.auth - load: true - push: false - tags: wxyc_auth_service:ci - cache-from: type=gha,scope=auth - cache-to: type=gha,mode=max,scope=auth - - # backend image: pull SHA-tagged image from ECR or build - - name: Pull backend image from ECR - if: env.RUN_TESTS == 'true' && inputs.force_rebuild != true && needs.detect-changes.outputs.backend == 'false' && needs.detect-changes.outputs.shared == 'false' - id: pull-backend - continue-on-error: true - env: - MERGE_BASE_SHA: ${{ needs.detect-changes.outputs.merge_base_sha }} - run: | - SHA_TAG="sha-${MERGE_BASE_SHA}" - if docker pull ${{ secrets.AWS_ECR_URI }}/backend:${SHA_TAG} 2>/dev/null; then - echo "✅ Pulled backend image with SHA tag: ${SHA_TAG}" - docker tag ${{ secrets.AWS_ECR_URI }}/backend:${SHA_TAG} wxyc_backend_service:ci - elif docker pull ${{ secrets.AWS_ECR_URI }}/backend:latest 2>/dev/null; then - echo "⚠️ SHA-tagged image not found, using :latest (may be stale)" - docker tag ${{ secrets.AWS_ECR_URI }}/backend:latest wxyc_backend_service:ci - else - echo "❌ No ECR image available, will build locally" - exit 1 - fi - - - name: Build backend image - if: env.RUN_TESTS == 'true' && (inputs.force_rebuild == true || needs.detect-changes.outputs.backend == 'true' || needs.detect-changes.outputs.shared == 'true' || steps.pull-backend.outcome == 'failure') - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile.backend - load: true - push: false - tags: wxyc_backend_service:ci - cache-from: type=gha,scope=backend - cache-to: type=gha,mode=max,scope=backend - - name: Set Up Node.js if: env.RUN_TESTS == 'true' uses: actions/setup-node@v4 @@ -359,7 +226,7 @@ jobs: echo "valid=true" >> $GITHUB_OUTPUT else echo "valid=false" >> $GITHUB_OUTPUT - echo "⚠️ Cache corrupted, will reinstall" + echo "Cache corrupted, will reinstall" rm -rf node_modules fi @@ -371,11 +238,27 @@ jobs: if: env.RUN_TESTS == 'true' run: npm run lint:env - - name: Set Up Test Environment + - name: Build + if: env.RUN_TESTS == 'true' + run: npm run build + + - name: Initialize database if: env.RUN_TESTS == 'true' run: | touch .env - npm run ci:env + node dev_env/init-db.mjs + + - name: Start services + if: env.RUN_TESTS == 'true' + env: + NODE_ENV: test + USE_MOCK_SERVICES: 'true' + ANON_DEVICE_JWT_SECRET: ci-test-secret-key-for-anonymous-devices + run: | + node apps/auth/dist/app.js > /tmp/auth.log 2>&1 & + node apps/backend/dist/app.js > /tmp/backend.log 2>&1 & + timeout 30 bash -c 'until curl -sf http://localhost:8083/healthcheck > /dev/null 2>&1; do sleep 1; done' + timeout 30 bash -c 'until curl -sf http://localhost:8081/healthcheck > /dev/null 2>&1; do sleep 1; done' - name: Run Integration Tests if: env.RUN_TESTS == 'true' @@ -384,6 +267,15 @@ jobs: # causes tests to interfere with each other's join/leave operations. run: npm run ci:test + - name: Show service logs on failure + if: env.RUN_TESTS == 'true' && failure() + run: | + echo "=== Auth Service Log ===" + cat /tmp/auth.log || true + echo "" + echo "=== Backend Service Log ===" + cat /tmp/backend.log || true + - name: Upload Coverage Report uses: actions/upload-artifact@v4 if: env.RUN_TESTS == 'true' && always() @@ -391,41 +283,3 @@ jobs: name: integration-test-coverage path: coverage/ retention-days: 14 - - # On push to main, push built images to ECR with SHA tags for cache - # Note: db-init is skipped as it doesn't have an ECR repository (CI-only image) - - name: Push images to ECR cache - if: env.RUN_TESTS == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main' - env: - AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - SHA_TAG="sha-${{ github.sha }}" - ECR_URI="${{ secrets.AWS_ECR_URI }}" - - echo "Pushing images to ECR with SHA tag: ${SHA_TAG}" - - # Push auth if it was built - if docker image inspect wxyc_auth_service:ci > /dev/null 2>&1; then - echo "📦 Pushing auth..." - docker tag wxyc_auth_service:ci ${ECR_URI}/auth:${SHA_TAG} - docker tag wxyc_auth_service:ci ${ECR_URI}/auth:latest - docker push ${ECR_URI}/auth:${SHA_TAG} - docker push ${ECR_URI}/auth:latest - fi - - # Push backend if it was built - if docker image inspect wxyc_backend_service:ci > /dev/null 2>&1; then - echo "📦 Pushing backend..." - docker tag wxyc_backend_service:ci ${ECR_URI}/backend:${SHA_TAG} - docker tag wxyc_backend_service:ci ${ECR_URI}/backend:latest - docker push ${ECR_URI}/backend:${SHA_TAG} - docker push ${ECR_URI}/backend:latest - fi - - echo "✅ ECR cache updated with SHA: ${SHA_TAG}" - - - name: Clean Up Test Environment - if: env.RUN_TESTS == 'true' && always() - run: npm run ci:clean diff --git a/.husky/pre-push b/.husky/pre-push new file mode 100755 index 0000000..e1beefe --- /dev/null +++ b/.husky/pre-push @@ -0,0 +1 @@ +npm run typecheck && npm run lint diff --git a/README.md b/README.md index e4dc4d0..7b77042 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,18 @@ The dev experience makes extensive use of Node.js project scripts. Here's a rund - `npm run format` : Formats all files with Prettier. - `npm run format:check` : Verifies all files match Prettier formatting (used in CI). +#### Git Hooks + +This project uses [husky](https://typicode.github.io/husky/) to run a **pre-push** hook that validates your code before it reaches CI. When you run `git push`, the hook automatically runs: + +```bash +npm run typecheck && npm run lint +``` + +This catches type errors and lint violations locally (in ~15-30s) instead of waiting for CI to report them. The hooks are set up automatically when you run `npm install`. + +If you need to bypass the hook in exceptional cases, use `git push --no-verify`. + #### Environment Variables Here is an example environment variable file. Create a file with these contents named `.env` in the root of your locally cloned project to ensure your dev environment works properly. diff --git a/package-lock.json b/package-lock.json index 9ef6de8..e44f025 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "eslint": "^9.39.2", "eslint-config-prettier": "^10.1.8", "eslint-plugin-security": "^3.0.1", + "husky": "^9.1.7", "jest": "^30.0.5", "jest-html-reporters": "^3.1.7", "nodemon": "^3.1.7", @@ -7915,6 +7916,22 @@ "ms": "^2.0.0" } }, + "node_modules/husky": { + "version": "9.1.7", + "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz", + "integrity": "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==", + "dev": true, + "license": "MIT", + "bin": { + "husky": "bin.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/typicode" + } + }, "node_modules/iconv-lite": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.1.tgz", diff --git a/package.json b/package.json index 687e3c3..e67bb0d 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,8 @@ "drizzle:generate": "read -p \"Enter migration name (optional): \" MIGRATION_NAME && dotenvx run -f .env -- drizzle-kit generate --config 'drizzle.config.ts' --name \"$MIGRATION_NAME\"", "drizzle:migrate": "dotenvx run -f .env -- drizzle-kit migrate --config 'drizzle.config.ts'", "drizzle:drop": "dotenvx run -f .env -- drizzle-kit drop --out ./shared/database/src/migrations", - "setup:e2e-users": "dotenvx run -f .env -- tsx dev_env/setup-e2e-test-users.ts" + "setup:e2e-users": "dotenvx run -f .env -- tsx dev_env/setup-e2e-test-users.ts", + "prepare": "husky || true" }, "author": "AyBruno", "license": "MIT", @@ -60,6 +61,7 @@ "eslint": "^9.39.2", "eslint-config-prettier": "^10.1.8", "eslint-plugin-security": "^3.0.1", + "husky": "^9.1.7", "jest": "^30.0.5", "jest-html-reporters": "^3.1.7", "nodemon": "^3.1.7", diff --git a/shared/authentication/tsconfig.build.json b/shared/authentication/tsconfig.build.json index 9c02639..a7e2e39 100644 --- a/shared/authentication/tsconfig.build.json +++ b/shared/authentication/tsconfig.build.json @@ -1,6 +1,7 @@ { "extends": ["./tsconfig.json"], "compilerOptions": { - "composite": false + "composite": false, + "incremental": false } } diff --git a/shared/database/tsconfig.build.json b/shared/database/tsconfig.build.json index 9c02639..a7e2e39 100644 --- a/shared/database/tsconfig.build.json +++ b/shared/database/tsconfig.build.json @@ -1,6 +1,7 @@ { "extends": ["./tsconfig.json"], "compilerOptions": { - "composite": false + "composite": false, + "incremental": false } } diff --git a/tsconfig.base.json b/tsconfig.base.json index 32daf83..e687ff2 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -14,6 +14,7 @@ "alwaysStrict": true, "sourceMap": true, "declaration": true, + "incremental": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true, "resolveJsonModule": true,