diff --git a/.github/scripts/upload-integ-test-metrics.py b/.github/scripts/upload-integ-test-metrics.py new file mode 100644 index 000000000..28595d647 --- /dev/null +++ b/.github/scripts/upload-integ-test-metrics.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +import sys +import xml.etree.ElementTree as ET +from datetime import datetime +from dataclasses import dataclass +from typing import Any, Literal, TypedDict +import os +import boto3 + +STRANDS_METRIC_NAMESPACE = 'Strands/Tests' + + + +class Dimension(TypedDict): + Name: str + Value: str + + +class MetricDatum(TypedDict): + MetricName: str + Dimensions: list[Dimension] + Value: float + Unit: str + Timestamp: datetime + + +@dataclass +class TestResult: + name: str + classname: str + duration: float + outcome: Literal['failed', 'skipped', 'passed'] + + +def parse_junit_xml(xml_file_path: str) -> list[TestResult]: + try: + tree = ET.parse(xml_file_path) + except FileNotFoundError: + print(f"Warning: XML file not found: {xml_file_path}") + return [] + except ET.ParseError as e: + print(f"Warning: Failed to parse XML: {e}") + return [] + + results = [] + root = tree.getroot() + + for testcase in root.iter('testcase'): + name = testcase.get('name') + classname = testcase.get('classname') + duration = float(testcase.get('time', 0.0)) + + if not name or not classname: + continue + + if testcase.find('failure') is not None or testcase.find('error') is not None: + outcome = 'failed' + elif testcase.find('skipped') is not None: + outcome = 'skipped' + else: + outcome = 'passed' + + results.append(TestResult(name, classname, duration, outcome)) + + return results + + +def build_metric_data(test_results: list[TestResult], repository: str) -> list[MetricDatum]: + metrics: list[MetricDatum] = [] + timestamp = datetime.utcnow() + + for test in test_results: + test_name = f"{test.classname}.{test.name}" + dimensions: list[Dimension] = [ + Dimension(Name='TestName', Value=test_name), + Dimension(Name='Repository', Value=repository) + ] + + metrics.append(MetricDatum( + MetricName='TestPassed', + Dimensions=dimensions, + Value=1.0 if test.outcome == 'passed' else 0.0, + Unit='Count', + Timestamp=timestamp + )) + + metrics.append(MetricDatum( + MetricName='TestFailed', + Dimensions=dimensions, + Value=1.0 if test.outcome == 'failed' else 0.0, + Unit='Count', + Timestamp=timestamp + )) + + metrics.append(MetricDatum( + MetricName='TestSkipped', + Dimensions=dimensions, + Value=1.0 if test.outcome == 'skipped' else 0.0, + Unit='Count', + Timestamp=timestamp + )) + + metrics.append(MetricDatum( + MetricName='TestDuration', + Dimensions=dimensions, + Value=test.duration, + Unit='Seconds', + Timestamp=timestamp + )) + + return metrics + + +def publish_metrics(metric_data: list[dict[str, Any]], region: str): + cloudwatch = boto3.client('cloudwatch', region_name=region) + + batch_size = 1000 + for i in range(0, len(metric_data), batch_size): + batch = metric_data[i:i + batch_size] + try: + cloudwatch.put_metric_data(Namespace=STRANDS_METRIC_NAMESPACE, MetricData=batch) + print(f"Published {len(batch)} metrics to CloudWatch") + except Exception as e: + print(f"Warning: Failed to publish metrics batch: {e}") + + +def main(): + if len(sys.argv) != 3: + print("Usage: python upload-integ-test-metrics.py ") + sys.exit(0) + + xml_file = sys.argv[1] + repository = sys.argv[2] + region = os.environ.get('AWS_REGION', 'us-east-1') + + test_results = parse_junit_xml(xml_file) + if not test_results: + print("No test results found") + sys.exit(1) + + print(f"Found {len(test_results)} test results") + metric_data = build_metric_data(test_results, repository) + publish_metrics(metric_data, region) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 65c785f30..ffe00bf6b 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -37,6 +37,7 @@ jobs: role-to-assume: ${{ secrets.STRANDS_INTEG_TEST_ROLE }} aws-region: us-east-1 mask-aws-account-id: true + - name: Checkout head commit uses: actions/checkout@v6 with: @@ -57,3 +58,9 @@ jobs: id: tests run: | hatch test tests_integ + + - name: Publish test metrics to CloudWatch + if: always() + run: | + pip install --no-cache-dir boto3 + python .github/scripts/upload-integ-test-metrics.py ./build/test-results.xml ${{ github.event.repository }} diff --git a/README.md b/README.md index 8e4d9d0e8..e21f8bd0b 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,6 @@ Ensure you have Python 3.10+ installed, then: python -m venv .venv source .venv/bin/activate # On Windows use: .venv\Scripts\activate -# Install Strands and tools pip install strands-agents strands-agents-tools ``` diff --git a/pyproject.toml b/pyproject.toml index 7f816880d..ba635cc48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -149,6 +149,7 @@ dependencies = [ "pytest-asyncio>=1.0.0,<1.4.0", "pytest-timeout>=2.0.0,<3.0.0", "pytest-xdist>=3.0.0,<4.0.0", + "pytest-timeout>=2.0.0,<3.0.0", "moto>=5.1.0,<6.0.0", ] @@ -240,7 +241,7 @@ convention = "google" [tool.pytest.ini_options] testpaths = ["tests"] asyncio_default_fixture_loop_scope = "function" -addopts = "--ignore=tests/strands/experimental/bidi --ignore=tests_integ/bidi" +addopts = "--ignore=tests/strands/experimental/bidi --ignore=tests_integ/bidi --junit-xml=build/test-results.xml" timeout = 45 @@ -298,7 +299,7 @@ prepare = [ "hatch run bidi-test:test-cov", ] -[tools.hatch.envs.bidi-lint] +[tool.hatch.envs.bidi-lint] template = "bidi" [tool.hatch.envs.bidi-lint.scripts] diff --git a/tests_integ/conftest.py b/tests_integ/conftest.py index 9de00089b..d4fd94186 100644 --- a/tests_integ/conftest.py +++ b/tests_integ/conftest.py @@ -189,7 +189,7 @@ def _load_api_keys_from_secrets_manager(): Validate that required environment variables are set when running in GitHub Actions. This prevents tests from being unintentionally skipped due to missing credentials. """ - if os.environ.get("GITHUB_ACTIONS") != "true": + if os.environ.get("GITHUB_ACTIONS") != "false": logger.warning("Tests running outside GitHub Actions, skipping required provider validation") return