Trigger full validation via a label

.github/workflows/fetch_and_log.yaml

@@ -9,7 +9,7 @@ jobs:
   fetch-workflows:
     runs-on: ubuntu-latest
     outputs:
-      workflow_ids: ${{ steps.fetch-ids.outputs.workflow_ids }}
+      workflow_ids: ${{ steps.fetch-unprocessed-run-ids.outputs.workflow_ids }}
 
     steps:
       - name: Get Last Processed Run ID
@@ -24,54 +24,48 @@ jobs:
       - name: Fetch Unprocessed Workflow Runs
         id: fetch-unprocessed-run-ids
         run: |
-          HOURS=1
-          # WORKFLOW_IDS=$(gh run list --repo $GITHUB_REPOSITORY --limit 100 --json databaseId,status,createdAt --jq '[.[] | select(.createdAt > (now - (1 * 3600))) | .databaseId] | @json')
-          # LAST_RUN_ID=$(echo $WORKFLOW_IDS | jq '.[-1]')
-
-          WORKFLOW_IDS=$(gh run list --repo $GITHUB_REPOSITORY --limit 100 --json databaseId,createdAt --jq "[.[] | select(.databaseId > $LAST_PROCESSED_RUN_ID) | .databaseId] | @json")
-          LAST_RUN_ID=$(echo $WORKFLOW_IDS | jq '.[-1]')  # Store the highest run ID
-
-          echo "LAST_RUN_ID=$LAST_RUN_ID" >> $GITHUB_ENV
+          # TODO: We need to address an important issue where workflow runs that are currently in progress may be included.
+          # These should be excluded from processing, and we should revisit them in the next run once they have finished.
+          WORKFLOW_IDS=$(gh run list --repo $GITHUB_REPOSITORY --limit 100 --json databaseId,createdAt,displayTitle --jq "[.[] | select(.databaseId > $LAST_PROCESSED_RUN_ID and .displayTitle != \"Fetch and Log Workflow Runs\")] | sort_by(.databaseId) | [.[].databaseId] | @json")
+          echo "Workflow runs to process: $WORKFLOW_IDS"
           echo "workflow_ids=$WORKFLOW_IDS" >> $GITHUB_OUTPUT
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  process-workflows:
+    needs: fetch-workflows
+    runs-on: ubuntu-latest
+    if: ${{ needs.fetch-workflows.outputs.workflow_ids != '[]' && needs.fetch-workflows.outputs.workflow_ids != '' }}
+    strategy:
+      matrix:
+        run_id: ${{ fromJson(needs.fetch-workflows.outputs.workflow_ids || '[]') }}
+      max-parallel: 1
+    steps:
+      - name: Workflow Run to Process
+        run:
+          echo ${{ matrix.run_id }}
+
+    #   - name: Send Workflow Logs to Splunk
+    #     uses: ykoer/github-workflow-splunk-logger@dev
+    #     with:
+    #       github_token: ${{ secrets.GITHUB_TOKEN }}
+    #       splunk_url: ${{ vars.HEC_URL }}
+    #       splunk_token: ${{ secrets.HEC_TOKEN }}
+    #       run_id: ${{ matrix.run_id }}
 
-      - name: Save Last Processed Run ID
+      - name: Save Processed Run ID
         if: success()
         run: |
-          gh secret set LAST_PROCESSED_ID --repo $GITHUB_REPOSITORY --body "$LAST_RUN_ID"
+          gh variable set LAST_PROCESSED_RUN_ID --repo $GITHUB_REPOSITORY --body "${{ matrix.run_id }}"
         env:
           GH_TOKEN: ${{ secrets.GH_PAT }}
-
-  process-logs:
+
+
+  # Add a fallback job that runs when there are no workflow Run's to process
+  process-workflows-empty:
     needs: fetch-workflows
     runs-on: ubuntu-latest
-    # strategy:
-    #   matrix:
-    #     run_id: ${{ fromJson(needs.fetch-workflows.outputs.workflow_ids || '[]') }}
+    if: ${{ needs.fetch-workflows.outputs.workflow_ids == '[]' || needs.fetch-workflows.outputs.workflow_ids == '' }}
     steps:
-        # - run:
-        #     echo ${{ matrix.run_id }}
-      - run: |
-          echo "processing logs...."
-          echo "-----------------"
-          echo ${{ needs.fetch-workflows.outputs.workflow_ids }}
-          echo "-----------------"
-          echo ${{ fromJson(needs.fetch-workflows.outputs.workflow_ids || '[]') }}
-          echo "-----------------"
-
-#   process-logs:
-#     needs: fetch-workflows
-#     runs-on: ubuntu-latest
-#     strategy:
-#       matrix:
-#         run_id: ${{ fromJson(needs.fetch-workflows.outputs.workflow_ids) }}
-
-#     steps:
-#       - name: Send Workflow Logs to Splunk
-#         uses: ykoer/github-workflow-splunk-logger@dev
-#         with:
-#           github_token: ${{ secrets.GITHUB_TOKEN }}
-#           splunk_url: ${{ vars.HEC_URL }}
-#           splunk_token: ${{ secrets.HEC_TOKEN }}
-#           run_id: ${{ matrix.run_id }}
+      - name: No workflows to process
+        run: echo "No new workflow runs to process"

.github/workflows/validate.yaml

@@ -0,0 +1,147 @@
+
+on:
+  pull_request:
+    types: [synchronize, opened, reopened, labeled]
+
+permissions:
+    pull-requests: write
+
+# concurrency:
+#   group: ${{ github.workflow }}-${{ github.ref }}
+#   cancel-in-progress: true
+
+jobs:
+  validate:
+    name: 'Validate Changed Packages - Github Hosted'
+    if: >
+      github.event.action == 'labeled' &&
+      contains(github.event.pull_request.labels.*.name, 'run-full-validation') &&
+      !contains(github.event.pull_request.labels.*.name, 'long-run')
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Run Validation Script
+        run: |
+          echo "Running validation because 'full validation' label was added"
+          # Add your validation logic here (e.g., linting, testing)
+
+          sleep 30
+          exit 0
+
+      - name: 'Remove label'
+        run: |
+          curl -X DELETE -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+              -H "Accept: application/vnd.github.v3+json" \
+              "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/run-full-validation"
+
+  validate-self-hosted:
+    name: 'Validate Changed Packages- Self-Hosted'
+    if: >
+      github.event.action == 'labeled' &&
+      contains(github.event.pull_request.labels.*.name, 'run-full-validation') &&
+      contains(github.event.pull_request.labels.*.name, 'long-run')
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Run Validation Script
+        run: |
+          echo "Running validation because 'full validation' label was added"
+          # Add your validation logic here (e.g., linting, testing)
+
+          sleep 30
+          exit 0
+
+      - name: 'Remove label'
+        run: |
+          curl -X DELETE -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+              -H "Accept: application/vnd.github.v3+json" \
+              "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/run-full-validation"
+
+      # - name: 'Remove label'
+      #   if: always()
+      #   run: |
+      #     PR_NUMBER=${{ github.event.pull_request.number }}
+      #     REPO=${{ github.repository }}
+      #     LABEL="full validation"
+
+      #     # URL-encodes the label by replacing special characters with their percent-encoded equivalents.
+      #     ENCODED_FULL_VALIDATION_LABEL=$(printf "%s" "${{ env.FULL_VALIDATION_LABEL }}" | sed -e 's/ /%20/g' -e 's/:/%3A/g' -e 's/\//%2F/g' -e 's/?/%3F/g' -e 's/&/%26/g' -e 's/=/%3D/g')
+
+      #     HTTP_RESPONSE=$(curl -s -o response.txt -w "%{http_code}" -X DELETE \
+      #       -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+      #       -H "Accept: application/vnd.github.v3+json" \
+      #       "https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels/${ENCODED_FULL_VALIDATION_LABEL}")
+
+      #     # Check if the HTTP response code is not 2xx and fail the step
+      #     if [[ $HTTP_RESPONSE -lt 200 || $HTTP_RESPONSE -ge 300 ]]; then
+      #       echo "Failed to remove label. HTTP Status: $HTTP_RESPONSE"
+      #       exit 1
+      #     fi
+
+  static-check:
+    name: 'See if Static Analysis should run'
+    if: github.event.action != 'labeled'
+    runs-on: ubuntu-latest
+
+    outputs:
+      all-changed-files: ${{ steps.changed-files.outputs.all_changed_and_modified_files }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Get all changed files for this PR
+        id: changed-files
+        run: |
+            # Simulating output for debugging purposes
+            echo "all_changed_and_modified_files=file1.txt,file2.txt,file3.txt"
+            echo "::set-output name=all_changed_and_modified_files::file1.txt,file2.txt,file3.txt"
+
+      - name: List changed files, skipping this job if there are no files to analyze
+        run: |
+          if [ "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" == "" ]; then
+            echo 'No files eligible for scanning were changed. Skipping Static Analysis.'
+            exit 0
+          else
+            echo ${{ steps.changed-files.outputs.all_changed_and_modified_files }}
+          fi
+
+  static:
+    name: 'Run Static Analysis'
+    runs-on: ubuntu-latest
+    needs: static-check
+    if: needs.static-check.outputs.all-changed-files != ''
+
+    steps:
+      - name: Check the outputs to determine whether to fail
+        run:  echo "Running static analyzer"
+
+#   remove-label:
+#     needs: validate
+#     if: always()  # Ensures this runs even if validation fails
+#     runs-on: ubuntu-latest
+
+#     steps:
+#       - name: Remove 'needs-validation' label
+#         run: |
+#           PR_NUMBER=${{ github.event.pull_request.number }}
+#           REPO=${{ github.repository }}
+#           LABEL="full validation"
+#           ENCODED_LABEL=$(printf "%s" "$LABEL" | sed -e 's/ /%20/g' -e 's/:/%3A/g' -e 's/\//%2F/g' -e 's/?/%3F/g' -e 's/&/%26/g' -e 's/=/%3D/g')
+
+#           echo $ENCODED_LABEL
+#           echo "https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels/$ENCODED_LABEL"
+#           curl -X DELETE -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+#               -H "Accept: application/vnd.github.v3+json" \
+#                "https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels/${ENCODED_LABEL}"