From 1ae65f4ea0347adb19a2777f8ac8fb28e7ff3338 Mon Sep 17 00:00:00 2001 From: Nicholas Clegg Date: Thu, 29 Jan 2026 12:22:58 -0500 Subject: [PATCH 1/4] Publish integ tests results to cloudwatch --- .github/scripts/upload-integ-test-metrics.py | 147 +++++++++++++++++++ .github/workflows/integration-test.yml | 6 + pyproject.toml | 5 +- 3 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/upload-integ-test-metrics.py diff --git a/.github/scripts/upload-integ-test-metrics.py b/.github/scripts/upload-integ-test-metrics.py new file mode 100644 index 000000000..d592f776d --- /dev/null +++ b/.github/scripts/upload-integ-test-metrics.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +import sys +import xml.etree.ElementTree as ET +from datetime import datetime, UTC +from dataclasses import dataclass +from typing import Any, Literal, TypedDict +import os +import boto3 + +STRANDS_METRIC_NAMESPACE = 'Strands/Tests' + + + +class Dimension(TypedDict): + Name: str + Value: str + + +class MetricDatum(TypedDict): + MetricName: str + Dimensions: list[Dimension] + Value: float + Unit: str + Timestamp: datetime + + +@dataclass +class TestResult: + name: str + classname: str + duration: float + outcome: Literal['failed', 'skipped', 'passed'] + + +def parse_junit_xml(xml_file_path: str) -> list[TestResult]: + try: + tree = ET.parse(xml_file_path) + except FileNotFoundError: + print(f"Warning: XML file not found: {xml_file_path}") + return [] + except ET.ParseError as e: + print(f"Warning: Failed to parse XML: {e}") + return [] + + results = [] + root = tree.getroot() + + for testcase in root.iter('testcase'): + name = testcase.get('name') + classname = testcase.get('classname') + duration = float(testcase.get('time', 0.0)) + + if not name or not classname: + continue + + if testcase.find('failure') is not None or testcase.find('error') is not None: + outcome = 'failed' + elif testcase.find('skipped') is not None: + outcome = 'skipped' + else: + outcome = 'passed' + + results.append(TestResult(name, classname, duration, outcome)) + + return results + + +def build_metric_data(test_results: list[TestResult], repository: str) -> list[MetricDatum]: + metrics: list[MetricDatum] = [] + timestamp = datetime.now(UTC) + + for test in test_results: + test_name = f"{test.classname}.{test.name}" + dimensions: list[Dimension] = [ + Dimension(Name='TestName', Value=test_name), + Dimension(Name='Repository', Value=repository) + ] + + metrics.append(MetricDatum( + MetricName='TestPassed', + Dimensions=dimensions, + Value=1.0 if test.outcome == 'passed' else 0.0, + Unit='Count', + Timestamp=timestamp + )) + + metrics.append(MetricDatum( + MetricName='TestFailed', + Dimensions=dimensions, + Value=1.0 if test.outcome == 'failed' else 0.0, + Unit='Count', + Timestamp=timestamp + )) + + metrics.append(MetricDatum( + MetricName='TestSkipped', + Dimensions=dimensions, + Value=1.0 if test.outcome == 'skipped' else 0.0, + Unit='Count', + Timestamp=timestamp + )) + + metrics.append(MetricDatum( + MetricName='TestDuration', + Dimensions=dimensions, + Value=test.duration, + Unit='Seconds', + Timestamp=timestamp + )) + + return metrics + + +def publish_metrics(metric_data: list[dict[str, Any]], region: str): + cloudwatch = boto3.client('cloudwatch', region_name=region) + + batch_size = 1000 + for i in range(0, len(metric_data), batch_size): + batch = metric_data[i:i + batch_size] + try: + cloudwatch.put_metric_data(Namespace=STRANDS_METRIC_NAMESPACE, MetricData=batch) + print(f"Published {len(batch)} metrics to CloudWatch") + except Exception as e: + print(f"Warning: Failed to publish metrics batch: {e}") + + +def main(): + if len(sys.argv) != 3: + print("Usage: python upload-integ-test-metrics.py ") + sys.exit(0) + + xml_file = sys.argv[1] + repository = sys.argv[2] + region = os.environ.get('AWS_REGION', 'us-east-1') + + test_results = parse_junit_xml(xml_file) + if not test_results: + print("No test results found") + sys.exit(1) + + print(f"Found {len(test_results)} test results") + metric_data = build_metric_data(test_results, repository) + publish_metrics(metric_data, region) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 65c785f30..be0988c78 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -37,6 +37,7 @@ jobs: role-to-assume: ${{ secrets.STRANDS_INTEG_TEST_ROLE }} aws-region: us-east-1 mask-aws-account-id: true + - name: Checkout head commit uses: actions/checkout@v6 with: @@ -57,3 +58,8 @@ jobs: id: tests run: | hatch test tests_integ + + - name: Publish test metrics to CloudWatch + if: ${{ github.event.organization == 'Unshure' }} + run: | + python .github/scripts/upload-integ-test-metrics.py ./build/test-results.xml ${{ github.event.repository }} diff --git a/pyproject.toml b/pyproject.toml index 7f816880d..ba635cc48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -149,6 +149,7 @@ dependencies = [ "pytest-asyncio>=1.0.0,<1.4.0", "pytest-timeout>=2.0.0,<3.0.0", "pytest-xdist>=3.0.0,<4.0.0", + "pytest-timeout>=2.0.0,<3.0.0", "moto>=5.1.0,<6.0.0", ] @@ -240,7 +241,7 @@ convention = "google" [tool.pytest.ini_options] testpaths = ["tests"] asyncio_default_fixture_loop_scope = "function" -addopts = "--ignore=tests/strands/experimental/bidi --ignore=tests_integ/bidi" +addopts = "--ignore=tests/strands/experimental/bidi --ignore=tests_integ/bidi --junit-xml=build/test-results.xml" timeout = 45 @@ -298,7 +299,7 @@ prepare = [ "hatch run bidi-test:test-cov", ] -[tools.hatch.envs.bidi-lint] +[tool.hatch.envs.bidi-lint] template = "bidi" [tool.hatch.envs.bidi-lint.scripts] From 0645afaaa5f8801d20efaf930fbac5bbc1a1eb52 Mon Sep 17 00:00:00 2001 From: Nicholas Clegg Date: Thu, 29 Jan 2026 12:28:02 -0500 Subject: [PATCH 2/4] disable api key tests for testing --- .github/workflows/integration-test.yml | 1 - tests_integ/conftest.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index be0988c78..96e0eb7c4 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -60,6 +60,5 @@ jobs: hatch test tests_integ - name: Publish test metrics to CloudWatch - if: ${{ github.event.organization == 'Unshure' }} run: | python .github/scripts/upload-integ-test-metrics.py ./build/test-results.xml ${{ github.event.repository }} diff --git a/tests_integ/conftest.py b/tests_integ/conftest.py index 9de00089b..d4fd94186 100644 --- a/tests_integ/conftest.py +++ b/tests_integ/conftest.py @@ -189,7 +189,7 @@ def _load_api_keys_from_secrets_manager(): Validate that required environment variables are set when running in GitHub Actions. This prevents tests from being unintentionally skipped due to missing credentials. """ - if os.environ.get("GITHUB_ACTIONS") != "true": + if os.environ.get("GITHUB_ACTIONS") != "false": logger.warning("Tests running outside GitHub Actions, skipping required provider validation") return From 03b20583a3b5228dadb3ec430980472ed037c08f Mon Sep 17 00:00:00 2001 From: Nicholas Clegg Date: Thu, 29 Jan 2026 12:36:20 -0500 Subject: [PATCH 3/4] Always run --- .github/scripts/upload-integ-test-metrics.py | 4 ++-- .github/workflows/integration-test.yml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/scripts/upload-integ-test-metrics.py b/.github/scripts/upload-integ-test-metrics.py index d592f776d..28595d647 100644 --- a/.github/scripts/upload-integ-test-metrics.py +++ b/.github/scripts/upload-integ-test-metrics.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import sys import xml.etree.ElementTree as ET -from datetime import datetime, UTC +from datetime import datetime from dataclasses import dataclass from typing import Any, Literal, TypedDict import os @@ -67,7 +67,7 @@ def parse_junit_xml(xml_file_path: str) -> list[TestResult]: def build_metric_data(test_results: list[TestResult], repository: str) -> list[MetricDatum]: metrics: list[MetricDatum] = [] - timestamp = datetime.now(UTC) + timestamp = datetime.utcnow() for test in test_results: test_name = f"{test.classname}.{test.name}" diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 96e0eb7c4..ffe00bf6b 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -60,5 +60,7 @@ jobs: hatch test tests_integ - name: Publish test metrics to CloudWatch + if: always() run: | + pip install --no-cache-dir boto3 python .github/scripts/upload-integ-test-metrics.py ./build/test-results.xml ${{ github.event.repository }} From 6013b4e321eeb1a2bd1ccbf7e1216ca5691facf7 Mon Sep 17 00:00:00 2001 From: Nick Clegg Date: Thu, 29 Jan 2026 12:59:21 -0500 Subject: [PATCH 4/4] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 8e4d9d0e8..e21f8bd0b 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,6 @@ Ensure you have Python 3.10+ installed, then: python -m venv .venv source .venv/bin/activate # On Windows use: .venv\Scripts\activate -# Install Strands and tools pip install strands-agents strands-agents-tools ```