From 8639e2d9fc2e01e8936e46795099b43eebe33a08 Mon Sep 17 00:00:00 2001 From: Nitin Ahuja Date: Fri, 9 Jan 2026 14:28:21 -0800 Subject: [PATCH 1/9] Add DocumentDB functional testing framework - Implement complete test framework with pytest - 36 tests covering find, aggregate, and insert operations - Multi-engine support with custom connection strings - Automatic test isolation and cleanup - Tag-based test organization and filtering - Parallel execution support with pytest-xdist - Add smart result analyzer - Automatic marker detection using heuristics - Filters test names, file names, and engine names - Categorizes failures: PASS/FAIL/UNSUPPORTED/INFRA_ERROR - CLI tool: docdb-analyze with text and JSON output - Configure development tools - Black for code formatting - isort for import sorting - flake8 for linting - mypy for type checking - pytest-cov for coverage reporting - Add comprehensive documentation - README with usage examples and best practices - CONTRIBUTING guide for writing tests - result_analyzer/README explaining analyzer behavior - All code formatted and linted - Add Docker support - Dockerfile for containerized testing - .dockerignore for clean builds Test Results: All 36 tests passed (100%) against DocumentDB --- .dockerignore | 53 ++++ .flake8 | 12 + .gitignore | 3 + CONTRIBUTING.md | 291 +++++++++++++++++++ Dockerfile | 42 +++ README.md | 374 ++++++++++++++++++++++++- conftest.py | 159 +++++++++++ pyproject.toml | 36 +++ pytest.ini | 51 ++++ requirements-dev.txt | 11 + requirements.txt | 14 + result_analyzer/README.md | 88 ++++++ result_analyzer/__init__.py | 11 + result_analyzer/analyzer.py | 251 +++++++++++++++++ result_analyzer/cli.py | 124 ++++++++ result_analyzer/report_generator.py | 134 +++++++++ setup.py | 39 +++ tests/__init__.py | 5 + tests/aggregate/__init__.py | 1 + tests/aggregate/test_group_stage.py | 128 +++++++++ tests/aggregate/test_match_stage.py | 87 ++++++ tests/common/__init__.py | 1 + tests/common/assertions.py | 103 +++++++ tests/find/__init__.py | 1 + tests/find/test_basic_queries.py | 136 +++++++++ tests/find/test_projections.py | 117 ++++++++ tests/find/test_query_operators.py | 158 +++++++++++ tests/insert/__init__.py | 1 + tests/insert/test_insert_operations.py | 145 ++++++++++ 29 files changed, 2574 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 .flake8 create mode 100644 CONTRIBUTING.md create mode 100644 Dockerfile create mode 100644 conftest.py create mode 100644 pyproject.toml create mode 100644 pytest.ini create mode 100644 requirements-dev.txt create mode 100644 requirements.txt create mode 100644 result_analyzer/README.md create mode 100644 result_analyzer/__init__.py create mode 100644 result_analyzer/analyzer.py create mode 100644 result_analyzer/cli.py create mode 100644 result_analyzer/report_generator.py create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/aggregate/__init__.py create mode 100644 tests/aggregate/test_group_stage.py create mode 100644 tests/aggregate/test_match_stage.py create mode 100644 tests/common/__init__.py create mode 100644 tests/common/assertions.py create mode 100644 tests/find/__init__.py create mode 100644 tests/find/test_basic_queries.py create mode 100644 tests/find/test_projections.py create mode 100644 tests/find/test_query_operators.py create mode 100644 tests/insert/__init__.py create mode 100644 tests/insert/test_insert_operations.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ca0dc97 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,53 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg-info/ +dist/ +build/ +.pytest_cache/ +.coverage +htmlcov/ + +# Virtual environments +venv/ +env/ +ENV/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Git +.git/ +.gitignore +.gitattributes + +# Documentation +docs/ +*.md +!README.md + +# Test results +.test-results/ +test-results/ +*.log + +# Development files +requirements-dev.txt +.flake8 +.mypy.ini + +# CI/CD +.github/ +.gitlab-ci.yml +.circleci/ + +# OS +.DS_Store +Thumbs.db diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..95b9a60 --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +max-line-length = 100 +extend-ignore = E203, W503 +exclude = + .git, + __pycache__, + .venv, + venv, + build, + dist, + .eggs, + *.egg-info diff --git a/.gitignore b/.gitignore index b7faf40..f2a869e 100644 --- a/.gitignore +++ b/.gitignore @@ -205,3 +205,6 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ + +# Test results directory +.test-results/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d8e5637 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,291 @@ +# Contributing to DocumentDB Functional Tests + +Thank you for your interest in contributing to the DocumentDB Functional Tests! This document provides guidelines for contributing to the project. + +## Getting Started + +### Prerequisites + +- Python 3.9 or higher +- Git +- Access to a DocumentDB or MongoDB instance for testing + +### Development Setup + +1. Fork the repository on GitHub +2. Clone your fork locally: + ```bash + git clone https://github.com/YOUR_USERNAME/functional-tests.git + cd functional-tests + ``` + +3. Install development dependencies: + ```bash + pip install -r requirements-dev.txt + ``` + +4. Create a branch for your changes: + ```bash + git checkout -b feature/your-feature-name + ``` + +## Writing Tests + +### Test File Organization + +- Place tests in the appropriate directory based on the operation being tested +- Use descriptive file names: `test_.py` +- Group related tests in the same file + +### Test Structure + +Every test should follow this structure: + +```python +import pytest +from tests.common.assertions import assert_document_match + +@pytest.mark. # Required: e.g., find, insert, aggregate +@pytest.mark. # Optional: e.g., rbac, decimal128 +@pytest.mark.documents([ # Optional: test data + {"name": "Alice", "age": 30} +]) +def test_descriptive_name(collection): + """ + Clear description of what this test validates. + + This should explain: + - What feature/behavior is being tested + - Expected outcome + - Any special conditions or edge cases + """ + # Setup (if needed beyond @pytest.mark.documents) + + # Execute the operation being tested + result = collection.find({"name": "Alice"}) + + # Assert expected behavior + assert len(list(result)) == 1 +``` + +### Naming Conventions + +- **Test functions**: `test_` + - Good: `test_find_with_gt_operator` + - Bad: `test_1`, `test_query` + +- **Test files**: `test_.py` + - Good: `test_query_operators.py` + - Bad: `tests.py`, `test.py` + +### Required Tags + +Every test MUST have at least one horizontal tag (operation): +- `@pytest.mark.find` +- `@pytest.mark.insert` +- `@pytest.mark.update` +- `@pytest.mark.delete` +- `@pytest.mark.aggregate` +- `@pytest.mark.index` +- `@pytest.mark.admin` +- `@pytest.mark.collection_mgmt` + +### Optional Tags + +Add vertical tags for cross-cutting features: +- `@pytest.mark.rbac` - Role-based access control +- `@pytest.mark.decimal128` - Decimal128 data type +- `@pytest.mark.collation` - Collation/sorting +- `@pytest.mark.transactions` - Transactions +- `@pytest.mark.geospatial` - Geospatial queries +- `@pytest.mark.text_search` - Text search +- `@pytest.mark.validation` - Schema validation +- `@pytest.mark.ttl` - Time-to-live indexes + +Add special tags when appropriate: +- `@pytest.mark.smoke` - Quick smoke tests +- `@pytest.mark.slow` - Tests taking > 5 seconds + +### Using Fixtures + +The framework provides three main fixtures: + +1. **engine_client**: Raw MongoDB client + ```python + def test_with_client(engine_client): + db = engine_client.test_db + collection = db.test_collection + # ... test code + ``` + +2. **database_client**: Database with automatic cleanup + ```python + def test_with_database(database_client): + collection = database_client.my_collection + # ... test code + # Database automatically dropped after test + ``` + +3. **collection**: Collection with automatic data setup and cleanup + ```python + @pytest.mark.documents([{"name": "Alice"}]) + def test_with_collection(collection): + # Documents already inserted + result = collection.find_one({"name": "Alice"}) + # ... assertions + # Collection automatically dropped after test + ``` + +### Custom Assertions + +Use the provided assertion helpers for common scenarios: + +```python +from tests.common.assertions import ( + assert_document_match, + assert_documents_match, + assert_field_exists, + assert_field_not_exists, + assert_count +) + +# Good: Use custom assertions +assert_document_match(actual, expected, ignore_id=True) +assert_count(collection, {"status": "active"}, 5) + +# Avoid: Manual comparison that's verbose +actual_doc = {k: v for k, v in actual.items() if k != "_id"} +expected_doc = {k: v for k, v in expected.items() if k != "_id"} +assert actual_doc == expected_doc +``` + +## Code Quality + +### Before Submitting + +Run these commands to ensure code quality: + +```bash +# Format code +black . + +# Sort imports +isort . + +# Run linter +flake8 + +# Type checking (optional but recommended) +mypy . + +# Run tests +pytest +``` + +### Code Style + +- Follow PEP 8 style guidelines +- Use meaningful variable names +- Add docstrings to test functions +- Keep test functions focused on a single behavior +- Avoid complex logic in tests + +## Testing Your Changes + +### Run Tests Locally + +```bash +# Run all tests +pytest + +# Run specific test file +pytest tests/find/test_basic_queries.py + +# Run specific test +pytest tests/find/test_basic_queries.py::test_find_all_documents + +# Run with your changes only +pytest -m "your_new_tag" +``` + +### Test Against Multiple Engines + +```bash +# Test against both DocumentDB and MongoDB +pytest --engine documentdb=mongodb://localhost:27017 \ + --engine mongodb=mongodb://mongo:27017 +``` + +## Submitting Changes + +### Pull Request Process + +1. Ensure your code follows the style guidelines +2. Add tests for new functionality +3. Update documentation if needed +4. Commit with clear, descriptive messages: + ```bash + git commit -m "Add tests for $group stage with $avg operator" + ``` + +5. Push to your fork: + ```bash + git push origin feature/your-feature-name + ``` + +6. Create a Pull Request on GitHub + +### Pull Request Guidelines + +Your PR should: +- Have a clear title describing the change +- Include a description explaining: + - What the change does + - Why it's needed + - How to test it +- Reference any related issues +- Pass all CI checks + +### Commit Message Guidelines + +- Use present tense: "Add test" not "Added test" +- Be descriptive but concise +- Reference issues: "Fix #123: Add validation tests" + +## Adding New Test Categories + +If you're adding tests for a new feature area: + +1. Create a new directory under `tests/`: + ```bash + mkdir tests/update + ``` + +2. Add `__init__.py`: + ```python + """Update operation tests.""" + ``` + +3. Create test files following naming conventions + +4. Add appropriate markers to `pytest.ini`: + ```ini + markers = + update: Update operation tests + ``` + +5. Update documentation in README.md + +## Questions? + +- Open an issue for questions about contributing +- Check existing tests for examples +- Review the RFC document for framework design + +## Code of Conduct + +This project follows the Contributor Covenant Code of Conduct. By participating, you are expected to uphold this code. + +## License + +By contributing, you agree that your contributions will be licensed under the MIT License. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9fc7292 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +# Multi-stage build for DocumentDB Functional Tests + +# Stage 1: Build stage +FROM python:3.11-slim as builder + +WORKDIR /build + +# Copy requirements +COPY requirements.txt . + +# Install dependencies +RUN pip install --no-cache-dir --user -r requirements.txt + +# Stage 2: Runtime stage +FROM python:3.11-slim + +# Set working directory +WORKDIR /app + +# Copy installed packages from builder +COPY --from=builder /root/.local /root/.local + +# Copy application code +COPY tests/ tests/ +COPY result_analyzer/ result_analyzer/ +COPY conftest.py . +COPY pytest.ini . +COPY setup.py . + +# Ensure scripts are in PATH +ENV PATH=/root/.local/bin:$PATH + +# Create directory for test results +RUN mkdir -p .test-results + +# Set Python to run unbuffered (so logs appear immediately) +ENV PYTHONUNBUFFERED=1 + +# Default command: run all tests +# Users can override with command line arguments +ENTRYPOINT ["pytest"] +CMD ["--help"] diff --git a/README.md b/README.md index dcdb725..6db5fa9 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,372 @@ -# functional-tests -End-to-end functional testing framework for DocumentDB using pytest +# DocumentDB Functional Tests + +End-to-end functional testing framework for DocumentDB using pytest. This framework validates DocumentDB functionality against specifications through comprehensive test suites. + +## Overview + +This testing framework provides: + +- **Specification-based Testing**: Tests define explicit expected behavior for DocumentDB features +- **Multi-Engine Support**: Run the same tests against DocumentDB, MongoDB, and other compatible engines +- **Parallel Execution**: Fast test execution using pytest-xdist +- **Tag-Based Organization**: Flexible test filtering using pytest markers +- **Result Analysis**: Automatic categorization and reporting of test results + +## Quick Start + +### Prerequisites + +- Python 3.9 or higher +- Access to a DocumentDB or MongoDB instance +- pip package manager + +### Installation + +```bash +# Clone the repository +git clone https://github.com/documentdb/functional-tests.git +cd functional-tests + +# Install dependencies +pip install -r requirements.txt +``` + +### Running Tests + +#### Basic Usage + +```bash +# Run all tests against default localhost +pytest + +# Run against specific engine +pytest --engine documentdb=mongodb://localhost:27017 + +# Run against multiple engines +pytest --engine documentdb=mongodb://localhost:27017 \ + --engine mongodb=mongodb://mongo:27017 +``` + +#### Filter by Tags + +```bash +# Run only find operation tests +pytest -m find + +# Run smoke tests +pytest -m smoke + +# Run find tests with RBAC +pytest -m "find and rbac" + +# Exclude slow tests +pytest -m "not slow" +``` + +#### Parallel Execution + +The framework supports parallel test execution using pytest-xdist, which can significantly reduce test execution time. + +```bash +# Run with 4 parallel processes +pytest -n 4 + +# Auto-detect number of CPUs (recommended) +pytest -n auto + +# Combine with other options +pytest -n auto -m smoke --engine documentdb=mongodb://localhost:27017 +``` + +**Performance Benefits:** +- Significantly faster test execution with multiple workers +- Scales with number of available CPU cores +- Particularly effective for large test suites + +**Best Practices:** +- Use `-n auto` to automatically detect optimal worker count +- Parallel execution works best with 4+ workers +- Each worker runs tests in isolation (separate database/collection) +- Safe for tests with automatic cleanup (all framework tests are safe) + +**When to Use:** +- Large test suites +- Local development for quick validation + +**Example with full options:** +```bash +pytest -n 4 \ + --engine documentdb=mongodb://localhost:27017 \ + -m "find or aggregate" \ + -v \ + --json-report --json-report-file=results.json +``` + +#### Output Formats + +```bash +# Generate JSON report +pytest --json-report --json-report-file=results.json + +# Generate JUnit XML +pytest --junitxml=results.xml + +# Verbose output +pytest -v + +# Show local variables on failure +pytest -l +``` + +## Docker Usage + +### Build the Image + +```bash +docker build -t documentdb/functional-tests . +``` + +### Run Tests in Container + +```bash +# Run against DocumentDB +docker run --network host \ + documentdb/functional-tests \ + --engine documentdb=mongodb://localhost:27017 + +# Run specific tags +docker run documentdb/functional-tests \ + --engine documentdb=mongodb://cluster.docdb.amazonaws.com:27017 \ + -m smoke + +# Run with parallel execution +docker run documentdb/functional-tests \ + --engine documentdb=mongodb://localhost:27017 \ + -n 4 +``` + +## Test Organization + +Tests are organized by API operations with cross-cutting feature tags: + +``` +tests/ +├── find/ # Find operation tests +│ ├── test_basic_queries.py +│ ├── test_query_operators.py +│ └── test_projections.py +├── aggregate/ # Aggregation tests +│ ├── test_match_stage.py +│ └── test_group_stage.py +├── insert/ # Insert operation tests +│ └── test_insert_operations.py +└── common/ # Shared utilities + └── assertions.py +``` + +## Test Tags + +### Horizontal Tags (API Operations) +- `find`: Find operation tests +- `insert`: Insert operation tests +- `update`: Update operation tests +- `delete`: Delete operation tests +- `aggregate`: Aggregation pipeline tests +- `index`: Index management tests +- `admin`: Administrative command tests +- `collection_mgmt`: Collection management tests + +### Vertical Tags (Cross-cutting Features) +- `rbac`: Role-based access control tests +- `decimal128`: Decimal128 data type tests +- `collation`: Collation and sorting tests +- `transactions`: Transaction tests +- `geospatial`: Geospatial query tests +- `text_search`: Text search tests +- `validation`: Schema validation tests +- `ttl`: Time-to-live index tests + +### Special Tags +- `smoke`: Quick smoke tests for feature detection +- `slow`: Tests that take longer to execute + +## Writing Tests + +### Basic Test Structure + +```python +import pytest +from tests.common.assertions import assert_document_match + +@pytest.mark.find # Required: operation tag +@pytest.mark.smoke # Optional: additional tags +@pytest.mark.documents([ # Optional: test data + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25} +]) +def test_find_with_filter(collection): + """Test description.""" + # Execute operation + result = list(collection.find({"age": {"$gt": 25}})) + + # Verify results + assert len(result) == 1 + assert_document_match(result[0], {"name": "Alice", "age": 30}) +``` + +### Test Fixtures + +- `engine_client`: MongoDB client for the engine +- `database_client`: Database with automatic cleanup +- `collection`: Collection with automatic test data setup and cleanup + +### Custom Assertions + +```python +from tests.common.assertions import ( + assert_document_match, + assert_documents_match, + assert_field_exists, + assert_field_not_exists, + assert_count +) + +# Compare documents ignoring _id +assert_document_match(actual, expected, ignore_id=True) + +# Compare lists of documents +assert_documents_match(actual_list, expected_list, ignore_order=True) + +# Check field existence +assert_field_exists(document, "user.name") +assert_field_not_exists(document, "password") + +# Count documents matching filter +assert_count(collection, {"status": "active"}, 5) +``` + +## Result Analysis + +The framework includes a command-line tool to analyze test results and generate detailed reports categorized by feature tags. + +### Installation + +```bash +# Install the package to get the CLI tool +pip install -e . +``` + +### CLI Tool Usage + +```bash +# Analyze default report location +docdb-analyze + +# Analyze specific report +docdb-analyze --input custom-results.json + +# Generate text report +docdb-analyze --output report.txt --format text + +# Generate JSON analysis +docdb-analyze --output analysis.json --format json + +# Quiet mode (only write to file, no console output) +docdb-analyze --output report.txt --quiet + +# Get help +docdb-analyze --help +``` + +### Programmatic Usage + +You can also use the result analyzer as a Python library: + +```python +from result_analyzer import analyze_results, generate_report, print_summary + +# Analyze JSON report +analysis = analyze_results(".test-results/report.json") + +# Print summary to console +print_summary(analysis) + +# Generate text report +generate_report(analysis, "report.txt", format="text") + +# Generate JSON report +generate_report(analysis, "report.json", format="json") +``` + +### Failure Categories + +Tests are automatically categorized into: +- **PASS**: Test succeeded, behavior matches specification +- **FAIL**: Test failed, feature exists but behaves incorrectly +- **UNSUPPORTED**: Feature not implemented +- **INFRA_ERROR**: Infrastructure issue (connection, timeout, etc.) + +### Example Workflow + +```bash +# Run tests with JSON report +pytest --engine documentdb=mongodb://localhost:27017 \ + --json-report --json-report-file=.test-results/report.json + +# Analyze results +docdb-analyze + +# Generate detailed text report +docdb-analyze --output detailed-report.txt --format text + +# Generate JSON for further processing +docdb-analyze --output analysis.json --format json +``` + +## Development + +### Install Development Dependencies + +```bash +pip install -r requirements-dev.txt +``` + +### Code Quality + +```bash +# Format code +black . + +# Sort imports +isort . + +# Lint +flake8 + +# Type checking +mypy . +``` + +### Run Tests with Coverage + +```bash +pytest --cov=. --cov-report=html +``` + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Write tests following the test structure guidelines +4. Ensure tests pass locally +5. Submit a pull request + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. + +## Support + +For issues and questions: +- GitHub Issues: https://github.com/documentdb/functional-tests/issues +- Documentation: https://github.com/documentdb/functional-tests/docs diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..422e021 --- /dev/null +++ b/conftest.py @@ -0,0 +1,159 @@ +""" +Global pytest fixtures for functional testing framework. + +This module provides fixtures for: +- Engine parametrization +- Database connection management +- Test isolation +""" + + +import pytest +from pymongo import MongoClient + + +def pytest_addoption(parser): + """Add custom command-line options for pytest.""" + parser.addoption( + "--engine", + action="append", + default=[], + help="Engine to test against. Format: name=connection_string. " + "Example: --engine documentdb=mongodb://localhost:27017 " + "--engine mongodb=mongodb://mongo:27017", + ) + + +def pytest_configure(config): + """Configure pytest with custom settings.""" + # Parse engine configurations + engines = {} + for engine_spec in config.getoption("--engine"): + if "=" not in engine_spec: + raise ValueError( + f"Invalid engine specification: {engine_spec}. " + "Expected format: name=connection_string" + ) + name, connection_string = engine_spec.split("=", 1) + engines[name] = connection_string + + # Store in config for access by fixtures + config.engines = engines + + # If no engines specified, default to localhost + if not engines: + config.engines = {"default": "mongodb://localhost:27017"} + + +def pytest_generate_tests(metafunc): + """ + Parametrize tests to run against multiple engines. + + Tests that use the 'engine_client' fixture will automatically + run against all configured engines. + """ + if "engine_client" in metafunc.fixturenames: + engines = metafunc.config.engines + metafunc.parametrize( + "engine_name,engine_connection_string", + [(name, conn) for name, conn in engines.items()], + ids=list(engines.keys()), + scope="function", + ) + + +@pytest.fixture(scope="function") +def engine_client(engine_name, engine_connection_string): + """ + Create a MongoDB client for the specified engine. + + This fixture is parametrized to run tests against all configured engines. + + Args: + engine_name: Name of the engine (from --engine option) + engine_connection_string: Connection string for the engine + + Yields: + MongoClient: Connected MongoDB client + """ + client = MongoClient(engine_connection_string) + + # Verify connection + try: + client.admin.command("ping") + except Exception as e: + pytest.skip(f"Cannot connect to {engine_name}: {e}") + + yield client + + # Cleanup: close connection + client.close() + + +@pytest.fixture(scope="function") +def database_client(engine_client, request): + """ + Provide a database client with automatic cleanup. + + Creates a test database named after the test function for isolation. + Automatically drops the database after the test completes. + + Args: + engine_client: MongoDB client from engine_client fixture + request: pytest request object + + Yields: + Database: MongoDB database object + """ + # Create unique database name based on test name + test_name = request.node.name.replace("[", "_").replace("]", "_") + db_name = f"test_{test_name}"[:63] # MongoDB database name limit + + db = engine_client[db_name] + + yield db + + # Cleanup: drop test database + try: + engine_client.drop_database(db_name) + except Exception: + pass # Best effort cleanup + + +@pytest.fixture(scope="function") +def collection(database_client, request): + """ + Provide a collection with automatic test data setup and cleanup. + + If the test is marked with @pytest.mark.documents([...]), this fixture + will automatically insert those documents before the test runs. + + Args: + database_client: Database from database_client fixture + request: pytest request object + + Yields: + Collection: MongoDB collection object + """ + # Use test name as collection name for isolation + collection_name = request.node.name.replace("[", "_").replace("]", "_")[:100] + coll = database_client[collection_name] + + # Check if test has @pytest.mark.documents decorator + marker = request.node.get_closest_marker("documents") + if marker and marker.args: + documents = marker.args[0] + if documents: + coll.insert_many(documents) + + yield coll + + # Cleanup: drop collection + try: + coll.drop() + except Exception: + pass # Best effort cleanup + + +# Custom marker for test data setup +pytest.mark.documents = pytest.mark.documents diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..905e866 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,36 @@ +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.venv + | venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +line_length = 100 +skip_gitignore = true + +[tool.mypy] +python_version = "3.9" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +ignore_missing_imports = true +exclude = [ + 'venv', + '.venv', + 'build', + 'dist', +] + +[tool.pytest.ini_options] +# Already in pytest.ini, keeping this for reference diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..3cc2c44 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,51 @@ +[pytest] +# Test discovery +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* + +# Output options +addopts = + -v + --strict-markers + --tb=short + --color=yes + +# Markers for test categorization +markers = + # Horizontal tags (API Operations) + find: Find operation tests + insert: Insert operation tests + update: Update operation tests + delete: Delete operation tests + aggregate: Aggregation pipeline tests + index: Index management tests + admin: Administrative command tests + collection_mgmt: Collection management tests + + # Vertical tags (Cross-cutting Features) + rbac: Role-based access control tests + decimal128: Decimal128 data type tests + collation: Collation and sorting tests + transactions: Transaction tests + geospatial: Geospatial query tests + text_search: Text search tests + validation: Schema validation tests + ttl: Time-to-live index tests + + # Special markers + smoke: Quick smoke tests for feature detection + slow: Tests that take longer to execute + +# Timeout for tests (seconds) +timeout = 300 + +# Parallel execution settings +# Use with: pytest -n auto +# Or: pytest -n 4 (for 4 processes) + +# JSON report configuration +json_report = .test-results/report.json +json_report_indent = 2 +json_report_omit = log diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..e8f0654 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,11 @@ +# Development dependencies +-r requirements.txt + +# Code quality +black>=23.7.0 # Code formatting +flake8>=6.1.0 # Linting +mypy>=1.5.0 # Type checking +isort>=5.12.0 # Import sorting + +# Testing +pytest-cov>=4.1.0 # Coverage reporting diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0577522 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +# Core testing dependencies +pytest>=7.4.0 +pytest-xdist>=3.3.0 # Parallel test execution +pytest-json-report>=1.5.0 # JSON output format +pytest-timeout>=2.1.0 # Test timeout support + +# Database connectivity +pymongo>=4.5.0 + +# Configuration management +pyyaml>=6.0 + +# Reporting +jinja2>=3.1.2 # Template rendering for reports diff --git a/result_analyzer/README.md b/result_analyzer/README.md new file mode 100644 index 0000000..151882b --- /dev/null +++ b/result_analyzer/README.md @@ -0,0 +1,88 @@ +# Result Analyzer + +The Result Analyzer automatically processes pytest JSON reports and categorizes results by meaningful test markers. + +## Features + +- **Automatic Marker Detection**: Uses heuristics to identify meaningful test markers +- **Smart Filtering**: Automatically excludes test names, file names, and internal markers +- **Flexible**: Supports new markers without code changes +- **CLI Tool**: `docdb-analyze` command for quick analysis + +## Marker Detection + +The analyzer automatically identifies meaningful markers by excluding: + +### Excluded by Pattern +- **Test function names**: Any marker containing `[` or starting with `test_` +- **File names**: Any marker containing `.py` or `/` +- **Directory names**: Common names like `tests`, `functional-tests`, `src`, `lib` + +### Excluded by Name +- **Pytest internal**: `parametrize`, `usefixtures`, `filterwarnings`, `pytestmark` +- **Fixture markers**: `documents` (used for test data setup) +- **Engine names**: `documentdb`, `mongodb`, `cosmosdb`, `default` (from parametrization) + +### Result +Only meaningful categorization markers remain: +- Horizontal tags: `find`, `insert`, `update`, `delete`, `aggregate`, `index`, `admin`, `collection_mgmt` +- Vertical tags: `rbac`, `decimal128`, `collation`, `transactions`, `geospatial`, `text_search`, `validation`, `ttl` +- Special tags: `smoke`, `slow` + +## Adding New Markers + +Simply add the marker to `pytest.ini` and use it in your tests: + +```python +@pytest.mark.mynewfeature +def test_something(): + pass +``` + +The analyzer will automatically detect and report on it - no code changes needed! + +## Failure Categorization + +Tests are categorized into four types: + +1. **PASS**: Test succeeded +2. **FAIL**: Test failed, feature exists but behaves incorrectly +3. **UNSUPPORTED**: Feature not implemented (skipped tests) +4. **INFRA_ERROR**: Infrastructure issue (connection, timeout, etc.) + +## Usage + +### CLI +```bash +# Quick analysis +docdb-analyze + +# Custom input/output +docdb-analyze --input results.json --output report.txt +``` + +### Programmatic +```python +from result_analyzer import analyze_results, generate_report + +analysis = analyze_results("report.json") +generate_report(analysis, "report.txt", format="text") +``` + +## Maintenance + +The heuristic-based approach means: +- ✅ **No maintenance** for new test markers +- ✅ **Automatic adaptation** to new patterns +- ⚠️ **May need updates** if new fixture patterns emerge (e.g., new engine names) + +To add a new fixture marker or engine name to exclude, update the filter in `analyzer.py`: + +```python +# Skip fixture markers +if marker in {"documents", "mynewfixture"}: + continue + +# Skip engine names +if marker in {"documentdb", "mongodb", "mynewengine"}: + continue diff --git a/result_analyzer/__init__.py b/result_analyzer/__init__.py new file mode 100644 index 0000000..63faf0a --- /dev/null +++ b/result_analyzer/__init__.py @@ -0,0 +1,11 @@ +""" +Result Analyzer for DocumentDB Functional Tests. + +This module provides tools for analyzing pytest test results and generating +reports categorized by feature tags. +""" + +from .analyzer import analyze_results, categorize_failure +from .report_generator import generate_report, print_summary + +__all__ = ["analyze_results", "categorize_failure", "generate_report", "print_summary"] diff --git a/result_analyzer/analyzer.py b/result_analyzer/analyzer.py new file mode 100644 index 0000000..af2b2a1 --- /dev/null +++ b/result_analyzer/analyzer.py @@ -0,0 +1,251 @@ +""" +Result analyzer for parsing and categorizing test results. + +This module provides functions to analyze pytest JSON output and categorize +test results by tags and failure types. +""" + +import json +from collections import defaultdict +from typing import Any, Dict, List + + +class FailureType: + """Enumeration of failure types.""" + + PASS = "PASS" + FAIL = "FAIL" + UNSUPPORTED = "UNSUPPORTED" + INFRA_ERROR = "INFRA_ERROR" + + +def categorize_failure(test_result: Dict[str, Any]) -> str: + """ + Categorize a test failure based on error information. + + Args: + test_result: Test result dictionary from pytest JSON report + + Returns: + One of: PASS, FAIL, UNSUPPORTED, INFRA_ERROR + """ + outcome = test_result.get("outcome", "") + + if outcome == "passed": + return FailureType.PASS + + if outcome == "skipped": + # Skipped tests typically indicate unsupported features + return FailureType.UNSUPPORTED + + if outcome == "failed": + # Analyze the failure to determine if it's infrastructure or functionality + call_info = test_result.get("call", {}) + longrepr = call_info.get("longrepr", "") + + # Check for infrastructure-related errors + infra_keywords = [ + "connection", + "timeout", + "network", + "cannot connect", + "refused", + "unreachable", + "host", + ] + + if any(keyword in longrepr.lower() for keyword in infra_keywords): + return FailureType.INFRA_ERROR + + # Otherwise, it's a functional failure + return FailureType.FAIL + + # Unknown outcome, treat as infrastructure error + return FailureType.INFRA_ERROR + + +def extract_markers(test_result: Dict[str, Any]) -> List[str]: + """ + Extract pytest markers (tags) from a test result. + + Automatically filters out test names, file names, and pytest internal markers + using heuristics, keeping only meaningful test categorization markers. + + Args: + test_result: Test result dictionary from pytest JSON report + + Returns: + List of marker names + """ + markers = [] + + # Extract from keywords + keywords = test_result.get("keywords", []) + if isinstance(keywords, list): + markers.extend(keywords) + + # Extract from markers field if present + test_markers = test_result.get("markers", []) + if isinstance(test_markers, list): + for marker in test_markers: + if isinstance(marker, dict): + markers.append(marker.get("name", "")) + else: + markers.append(str(marker)) + + # Filter out non-meaningful markers using heuristics + filtered_markers = [] + for marker in markers: + # Skip empty strings + if not marker: + continue + + # Skip if it looks like a test function name (contains brackets or starts with test_) + if "[" in marker or marker.startswith("test_"): + continue + + # Skip if it looks like a file name (contains .py or /) + if ".py" in marker or "/" in marker: + continue + + # Skip if it's a directory name (common directory names) + if marker in {"tests", "functional-tests", "src", "lib"}: + continue + + # Skip pytest internal markers + if marker in {"parametrize", "usefixtures", "filterwarnings", "pytestmark"}: + continue + + # Skip fixture markers (markers used for setup, not categorization) + # These are markers that take arguments in the decorator + if marker in {"documents"}: + continue + + # Skip engine names (from parametrization like [documentdb], [mongodb]) + # Common engine names that appear in test results + if marker in {"documentdb", "mongodb", "cosmosdb", "default"}: + continue + + # If it passed all filters, it's likely a meaningful marker + filtered_markers.append(marker) + + return filtered_markers + + +def analyze_results(json_report_path: str) -> Dict[str, Any]: + """ + Analyze pytest JSON report and generate categorized results. + + Args: + json_report_path: Path to pytest JSON report file + + Returns: + Dictionary containing analysis results with structure: + { + "summary": { + "total": int, + "passed": int, + "failed": int, + "unsupported": int, + "infra_error": int + }, + "by_tag": { + "tag_name": { + "passed": int, + "failed": int, + "unsupported": int, + "infra_error": int, + "pass_rate": float + } + }, + "tests": [ + { + "name": str, + "outcome": str, + "duration": float, + "tags": List[str], + "error": str (optional) + } + ] + } + """ + # Load JSON report + with open(json_report_path, "r") as f: + report = json.load(f) + + # Initialize counters + summary: Dict[str, Any] = { + "total": 0, + "passed": 0, + "failed": 0, + "unsupported": 0, + "infra_error": 0, + } + + by_tag: Dict[str, Dict[str, int]] = defaultdict( + lambda: {"passed": 0, "failed": 0, "unsupported": 0, "infra_error": 0} + ) + + tests_details = [] + + # Process each test + tests = report.get("tests", []) + for test in tests: + summary["total"] += 1 + + # Categorize the result + failure_type = categorize_failure(test) + + # Extract tags + tags = extract_markers(test) + + # Update summary counters + if failure_type == FailureType.PASS: + summary["passed"] += 1 + elif failure_type == FailureType.FAIL: + summary["failed"] += 1 + elif failure_type == FailureType.UNSUPPORTED: + summary["unsupported"] += 1 + elif failure_type == FailureType.INFRA_ERROR: + summary["infra_error"] += 1 + + # Update tag-specific counters + for tag in tags: + if failure_type == FailureType.PASS: + by_tag[tag]["passed"] += 1 + elif failure_type == FailureType.FAIL: + by_tag[tag]["failed"] += 1 + elif failure_type == FailureType.UNSUPPORTED: + by_tag[tag]["unsupported"] += 1 + elif failure_type == FailureType.INFRA_ERROR: + by_tag[tag]["infra_error"] += 1 + + # Store test details + test_detail = { + "name": test.get("nodeid", ""), + "outcome": failure_type, + "duration": test.get("duration", 0), + "tags": tags, + } + + # Add error information if failed + if failure_type in [FailureType.FAIL, FailureType.INFRA_ERROR]: + call_info = test.get("call", {}) + test_detail["error"] = call_info.get("longrepr", "") + + tests_details.append(test_detail) + + # Calculate pass rates for each tag + by_tag_with_rates = {} + for tag, counts in by_tag.items(): + total = counts["passed"] + counts["failed"] + counts["unsupported"] + counts["infra_error"] + pass_rate = (counts["passed"] / total * 100) if total > 0 else 0 + + by_tag_with_rates[tag] = {**counts, "total": total, "pass_rate": round(pass_rate, 2)} + + # Calculate overall pass rate + summary["pass_rate"] = round( + (summary["passed"] / summary["total"] * 100) if summary["total"] > 0 else 0, 2 + ) + + return {"summary": summary, "by_tag": by_tag_with_rates, "tests": tests_details} diff --git a/result_analyzer/cli.py b/result_analyzer/cli.py new file mode 100644 index 0000000..99ccad7 --- /dev/null +++ b/result_analyzer/cli.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +""" +DocumentDB Functional Test Results Analyzer - CLI + +Command-line interface for analyzing pytest JSON reports and generating +categorized results by feature tags. +""" + +import argparse +import sys +from pathlib import Path + +from .analyzer import analyze_results +from .report_generator import generate_report, print_summary + + +def main(): + parser = argparse.ArgumentParser( + description="Analyze DocumentDB functional test results and generate reports", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Analyze default report location + %(prog)s + + # Analyze specific report + %(prog)s --input my-results.json + + # Generate text report + %(prog)s --output report.txt --format text + + # Generate JSON analysis + %(prog)s --output analysis.json --format json + + # Quiet mode (no console output) + %(prog)s --output report.txt --quiet + """, + ) + + parser.add_argument( + "-i", + "--input", + default=".test-results/report.json", + help="Path to pytest JSON report file (default: .test-results/report.json)", + ) + + parser.add_argument( + "-o", "--output", help="Path to output report file (if not specified, prints to console)" + ) + + parser.add_argument( + "-f", + "--format", + choices=["text", "json"], + default="text", + help="Output format: text or json (default: text)", + ) + + parser.add_argument( + "-q", + "--quiet", + action="store_true", + help="Suppress console output (only write to output file)", + ) + + parser.add_argument( + "--no-summary", action="store_true", help="Skip printing summary to console" + ) + + args = parser.parse_args() + + # Check if input file exists + input_path = Path(args.input) + if not input_path.exists(): + print(f"Error: Input file not found: {args.input}", file=sys.stderr) + print("\nMake sure to run pytest with --json-report first:", file=sys.stderr) + print(f" pytest --json-report --json-report-file={args.input}", file=sys.stderr) + return 1 + + try: + # Analyze the results + if not args.quiet: + print(f"Analyzing test results from: {args.input}") + + analysis = analyze_results(args.input) + + # Print summary to console (unless quiet or no-summary) + if not args.quiet and not args.no_summary: + print_summary(analysis) + + # Generate output file if specified + if args.output: + generate_report(analysis, args.output, format=args.format) + if not args.quiet: + print(f"\nReport saved to: {args.output}") + + # If no output file and quiet mode, print to stdout + elif not args.quiet: + print("\nResults by Tag:") + print("-" * 60) + for tag, stats in sorted( + analysis["by_tag"].items(), key=lambda x: x[1]["pass_rate"], reverse=True + ): + passed = stats["passed"] + total = stats["total"] + rate = stats["pass_rate"] + print(f"{tag:30s} | {passed:3d}/{total:3d} passed ({rate:5.1f}%)") + + # Return exit code based on test results + if analysis["summary"]["failed"] > 0: + return 1 + return 0 + + except Exception as e: + print(f"Error analyzing results: {e}", file=sys.stderr) + if not args.quiet: + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/result_analyzer/report_generator.py b/result_analyzer/report_generator.py new file mode 100644 index 0000000..c0d2b26 --- /dev/null +++ b/result_analyzer/report_generator.py @@ -0,0 +1,134 @@ +""" +Report generator for creating human-readable reports from analysis results. + +This module provides functions to generate various report formats from +analyzed test results. +""" + +import json +from datetime import datetime +from typing import Any, Dict + + +def generate_report(analysis: Dict[str, Any], output_path: str, format: str = "json"): + """ + Generate a report from analysis results. + + Args: + analysis: Analysis results from analyze_results() + output_path: Path to write the report + format: Report format ("json" or "text") + """ + if format == "json": + generate_json_report(analysis, output_path) + elif format == "text": + generate_text_report(analysis, output_path) + else: + raise ValueError(f"Unsupported report format: {format}") + + +def generate_json_report(analysis: Dict[str, Any], output_path: str): + """ + Generate a JSON report. + + Args: + analysis: Analysis results from analyze_results() + output_path: Path to write the JSON report + """ + report = { + "generated_at": datetime.now().isoformat(), + "summary": analysis["summary"], + "by_tag": analysis["by_tag"], + "tests": analysis["tests"], + } + + with open(output_path, "w") as f: + json.dump(report, f, indent=2) + + +def generate_text_report(analysis: Dict[str, Any], output_path: str): + """ + Generate a human-readable text report. + + Args: + analysis: Analysis results from analyze_results() + output_path: Path to write the text report + """ + lines = [] + + # Header + lines.append("=" * 80) + lines.append("DocumentDB Functional Test Results") + lines.append("=" * 80) + lines.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + lines.append("") + + # Summary + summary = analysis["summary"] + lines.append("SUMMARY") + lines.append("-" * 80) + lines.append(f"Total Tests: {summary['total']}") + lines.append(f"Passed: {summary['passed']} ({summary['pass_rate']}%)") + lines.append(f"Failed: {summary['failed']}") + lines.append(f"Unsupported: {summary['unsupported']}") + lines.append(f"Infrastructure Errors: {summary['infra_error']}") + lines.append("") + + # Results by tag + lines.append("RESULTS BY TAG") + lines.append("-" * 80) + + if analysis["by_tag"]: + # Sort tags by pass rate (ascending) to highlight problematic areas + sorted_tags = sorted(analysis["by_tag"].items(), key=lambda x: x[1]["pass_rate"]) + + for tag, stats in sorted_tags: + lines.append(f"\n{tag}:") + lines.append(f" Total: {stats['total']}") + lines.append(f" Passed: {stats['passed']} ({stats['pass_rate']}%)") + lines.append(f" Failed: {stats['failed']}") + lines.append(f" Unsupported: {stats['unsupported']}") + lines.append(f" Infra Error: {stats['infra_error']}") + else: + lines.append("No tags found in test results.") + + lines.append("") + + # Failed tests details + failed_tests = [t for t in analysis["tests"] if t["outcome"] == "FAIL"] + if failed_tests: + lines.append("FAILED TESTS") + lines.append("-" * 80) + for test in failed_tests: + lines.append(f"\n{test['name']}") + lines.append(f" Tags: {', '.join(test['tags'])}") + lines.append(f" Duration: {test['duration']:.2f}s") + if "error" in test: + error_preview = test["error"][:200] + lines.append(f" Error: {error_preview}...") + + lines.append("") + lines.append("=" * 80) + + # Write report + with open(output_path, "w") as f: + f.write("\n".join(lines)) + + +def print_summary(analysis: Dict[str, Any]): + """ + Print a brief summary to console. + + Args: + analysis: Analysis results from analyze_results() + """ + summary = analysis["summary"] + print("\n" + "=" * 60) + print("Test Results Summary") + print("=" * 60) + print(f"Total: {summary['total']}") + print(f"Passed: {summary['passed']} ({summary['pass_rate']}%)") + print(f"Failed: {summary['failed']}") + print(f"Unsupported: {summary['unsupported']}") + print(f"Infra Error: {summary['infra_error']}") + print("=" * 60 + "\n") diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..22194cf --- /dev/null +++ b/setup.py @@ -0,0 +1,39 @@ +from setuptools import find_packages, setup + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +setup( + name="documentdb-functional-tests", + version="0.1.0", + author="DocumentDB Contributors", + description="End-to-end functional testing framework for DocumentDB", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/documentdb/functional-tests", + packages=find_packages(), + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + ], + python_requires=">=3.9", + install_requires=[ + "pytest>=7.4.0", + "pytest-xdist>=3.3.0", + "pytest-json-report>=1.5.0", + "pytest-timeout>=2.1.0", + "pymongo>=4.5.0", + "pyyaml>=6.0", + "jinja2>=3.1.2", + ], + entry_points={ + "console_scripts": [ + "docdb-analyze=result_analyzer.cli:main", + ], + }, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..3492497 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,5 @@ +""" +DocumentDB Functional Tests + +End-to-end functional testing suite for DocumentDB. +""" diff --git a/tests/aggregate/__init__.py b/tests/aggregate/__init__.py new file mode 100644 index 0000000..1103a4a --- /dev/null +++ b/tests/aggregate/__init__.py @@ -0,0 +1 @@ +"""Aggregation pipeline tests.""" diff --git a/tests/aggregate/test_group_stage.py b/tests/aggregate/test_group_stage.py new file mode 100644 index 0000000..1a51748 --- /dev/null +++ b/tests/aggregate/test_group_stage.py @@ -0,0 +1,128 @@ +""" +Aggregation $group stage tests. + +Tests for the $group stage in aggregation pipelines. +""" + +import pytest + + +@pytest.mark.aggregate +@pytest.mark.smoke +@pytest.mark.documents( + [ + {"name": "Alice", "department": "Engineering", "salary": 100000}, + {"name": "Bob", "department": "Engineering", "salary": 90000}, + {"name": "Charlie", "department": "Sales", "salary": 80000}, + {"name": "David", "department": "Sales", "salary": 75000}, + ] +) +def test_group_with_count(collection): + """Test $group stage with count aggregation.""" + # Execute aggregation to count documents by department + pipeline = [{"$group": {"_id": "$department", "count": {"$sum": 1}}}] + result = list(collection.aggregate(pipeline)) + + # Verify results + assert len(result) == 2, "Expected 2 departments" + + # Convert to dict for easier verification + dept_counts = {doc["_id"]: doc["count"] for doc in result} + assert dept_counts["Engineering"] == 2, "Expected 2 employees in Engineering" + assert dept_counts["Sales"] == 2, "Expected 2 employees in Sales" + + +@pytest.mark.aggregate +@pytest.mark.documents( + [ + {"name": "Alice", "department": "Engineering", "salary": 100000}, + {"name": "Bob", "department": "Engineering", "salary": 90000}, + {"name": "Charlie", "department": "Sales", "salary": 80000}, + ] +) +def test_group_with_sum(collection): + """Test $group stage with sum aggregation.""" + # Execute aggregation to sum salaries by department + pipeline = [{"$group": {"_id": "$department", "totalSalary": {"$sum": "$salary"}}}] + result = list(collection.aggregate(pipeline)) + + # Verify results + assert len(result) == 2, "Expected 2 departments" + + # Convert to dict for easier verification + dept_salaries = {doc["_id"]: doc["totalSalary"] for doc in result} + assert dept_salaries["Engineering"] == 190000, "Expected total Engineering salary of 190000" + assert dept_salaries["Sales"] == 80000, "Expected total Sales salary of 80000" + + +@pytest.mark.aggregate +@pytest.mark.documents( + [ + {"name": "Alice", "department": "Engineering", "salary": 100000}, + {"name": "Bob", "department": "Engineering", "salary": 90000}, + {"name": "Charlie", "department": "Sales", "salary": 80000}, + ] +) +def test_group_with_avg(collection): + """Test $group stage with average aggregation.""" + # Execute aggregation to calculate average salary by department + pipeline = [{"$group": {"_id": "$department", "avgSalary": {"$avg": "$salary"}}}] + result = list(collection.aggregate(pipeline)) + + # Verify results + assert len(result) == 2, "Expected 2 departments" + + # Convert to dict for easier verification + dept_avg = {doc["_id"]: doc["avgSalary"] for doc in result} + assert dept_avg["Engineering"] == 95000, "Expected average Engineering salary of 95000" + assert dept_avg["Sales"] == 80000, "Expected average Sales salary of 80000" + + +@pytest.mark.aggregate +@pytest.mark.documents( + [ + {"name": "Alice", "department": "Engineering", "salary": 100000}, + {"name": "Bob", "department": "Engineering", "salary": 90000}, + {"name": "Charlie", "department": "Sales", "salary": 80000}, + ] +) +def test_group_with_min_max(collection): + """Test $group stage with min and max aggregations.""" + # Execute aggregation to find min and max salary by department + pipeline = [ + { + "$group": { + "_id": "$department", + "minSalary": {"$min": "$salary"}, + "maxSalary": {"$max": "$salary"}, + } + } + ] + result = list(collection.aggregate(pipeline)) + + # Verify results + assert len(result) == 2, "Expected 2 departments" + + # Verify Engineering department + eng_dept = next(doc for doc in result if doc["_id"] == "Engineering") + assert eng_dept["minSalary"] == 90000, "Expected min Engineering salary of 90000" + assert eng_dept["maxSalary"] == 100000, "Expected max Engineering salary of 100000" + + +@pytest.mark.aggregate +@pytest.mark.documents( + [ + {"item": "A", "quantity": 5}, + {"item": "B", "quantity": 10}, + {"item": "A", "quantity": 3}, + ] +) +def test_group_all_documents(collection): + """Test $group stage grouping all documents (using null as _id).""" + # Execute aggregation to sum quantities across all documents + pipeline = [{"$group": {"_id": None, "totalQuantity": {"$sum": "$quantity"}}}] + result = list(collection.aggregate(pipeline)) + + # Verify results + assert len(result) == 1, "Expected single result grouping all documents" + assert result[0]["totalQuantity"] == 18, "Expected total quantity of 18" diff --git a/tests/aggregate/test_match_stage.py b/tests/aggregate/test_match_stage.py new file mode 100644 index 0000000..886b78e --- /dev/null +++ b/tests/aggregate/test_match_stage.py @@ -0,0 +1,87 @@ +""" +Aggregation $match stage tests. + +Tests for the $match stage in aggregation pipelines. +""" + +import pytest + + +@pytest.mark.aggregate +@pytest.mark.smoke +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30, "status": "active"}, + {"name": "Bob", "age": 25, "status": "active"}, + {"name": "Charlie", "age": 35, "status": "inactive"}, + ] +) +def test_match_simple_filter(collection): + """Test $match stage with simple equality filter.""" + # Execute aggregation with $match stage + pipeline = [{"$match": {"status": "active"}}] + result = list(collection.aggregate(pipeline)) + + # Verify results + assert len(result) == 2, "Expected 2 active users" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Bob"}, "Expected Alice and Bob" + + +@pytest.mark.aggregate +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ] +) +def test_match_with_comparison_operator(collection): + """Test $match stage with comparison operators.""" + # Execute aggregation with $match using $gt + pipeline = [{"$match": {"age": {"$gt": 25}}}] + result = list(collection.aggregate(pipeline)) + + # Verify results + assert len(result) == 2, "Expected 2 users with age > 25" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Charlie"}, "Expected Alice and Charlie" + + +@pytest.mark.aggregate +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30, "city": "NYC"}, + {"name": "Bob", "age": 25, "city": "SF"}, + {"name": "Charlie", "age": 35, "city": "NYC"}, + ] +) +def test_match_multiple_conditions(collection): + """Test $match stage with multiple filter conditions.""" + # Execute aggregation with multiple conditions in $match + pipeline = [{"$match": {"city": "NYC", "age": {"$gte": 30}}}] + result = list(collection.aggregate(pipeline)) + + # Verify results + assert len(result) == 2, "Expected 2 users from NYC with age >= 30" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Charlie"}, "Expected Alice and Charlie" + + +@pytest.mark.aggregate +def test_match_empty_result(collection): + """Test $match stage that matches no documents.""" + # Insert test data + collection.insert_many( + [ + {"name": "Alice", "status": "active"}, + {"name": "Bob", "status": "active"}, + ] + ) + + # Execute aggregation with $match that matches nothing + pipeline = [{"$match": {"status": "inactive"}}] + result = list(collection.aggregate(pipeline)) + + # Verify empty result + assert result == [], "Expected empty result when no documents match" diff --git a/tests/common/__init__.py b/tests/common/__init__.py new file mode 100644 index 0000000..b34d9ac --- /dev/null +++ b/tests/common/__init__.py @@ -0,0 +1 @@ +"""Common utilities and helpers for functional tests.""" diff --git a/tests/common/assertions.py b/tests/common/assertions.py new file mode 100644 index 0000000..c6d9e89 --- /dev/null +++ b/tests/common/assertions.py @@ -0,0 +1,103 @@ +""" +Custom assertion helpers for functional tests. + +Provides convenient assertion methods for common test scenarios. +""" + +from typing import Dict, List + + +def assert_document_match(actual: Dict, expected: Dict, ignore_id: bool = True): + """ + Assert that a document matches the expected structure and values. + + Args: + actual: The actual document from the database + expected: The expected document structure + ignore_id: If True, ignore _id field in comparison (default: True) + """ + if ignore_id: + actual = {k: v for k, v in actual.items() if k != "_id"} + expected = {k: v for k, v in expected.items() if k != "_id"} + + assert actual == expected, f"Document mismatch.\nExpected: {expected}\nActual: {actual}" + + +def assert_documents_match( + actual: List[Dict], expected: List[Dict], ignore_id: bool = True, ignore_order: bool = False +): + """ + Assert that a list of documents matches the expected list. + + Args: + actual: List of actual documents from the database + expected: List of expected documents + ignore_id: If True, ignore _id field in comparison (default: True) + ignore_order: If True, sort both lists before comparison (default: False) + """ + if ignore_id: + actual = [{k: v for k, v in doc.items() if k != "_id"} for doc in actual] + expected = [{k: v for k, v in doc.items() if k != "_id"} for doc in expected] + + assert len(actual) == len( + expected + ), f"Document count mismatch. Expected {len(expected)}, got {len(actual)}" + + if ignore_order: + # Sort for comparison + actual = sorted(actual, key=lambda x: str(x)) + expected = sorted(expected, key=lambda x: str(x)) + + for i, (act, exp) in enumerate(zip(actual, expected)): + assert act == exp, f"Document at index {i} does not match.\nExpected: {exp}\nActual: {act}" + + +def assert_field_exists(document: Dict, field_path: str): + """ + Assert that a field exists in a document (supports nested paths). + + Args: + document: The document to check + field_path: Dot-notation field path (e.g., "user.name") + """ + parts = field_path.split(".") + current = document + + for part in parts: + assert part in current, f"Field '{field_path}' does not exist in document" + current = current[part] + + +def assert_field_not_exists(document: Dict, field_path: str): + """ + Assert that a field does not exist in a document (supports nested paths). + + Args: + document: The document to check + field_path: Dot-notation field path (e.g., "user.name") + """ + parts = field_path.split(".") + current = document + + for i, part in enumerate(parts): + if part not in current: + return # Field doesn't exist, assertion passes + if i == len(parts) - 1: + raise AssertionError(f"Field '{field_path}' exists in document but should not") + current = current[part] + + +def assert_count(collection, filter_query: Dict, expected_count: int): + """ + Assert that a collection contains the expected number of documents matching a filter. + + Args: + collection: MongoDB collection object + filter_query: Query filter + expected_count: Expected number of matching documents + """ + actual_count = collection.count_documents(filter_query) + assert actual_count == expected_count, ( + f"Document count mismatch for filter {filter_query}. " + f"Expected {expected_count}, got {actual_count}" + ) diff --git a/tests/find/__init__.py b/tests/find/__init__.py new file mode 100644 index 0000000..0bdc7e7 --- /dev/null +++ b/tests/find/__init__.py @@ -0,0 +1 @@ +"""Find operation tests.""" diff --git a/tests/find/test_basic_queries.py b/tests/find/test_basic_queries.py new file mode 100644 index 0000000..9a27d1d --- /dev/null +++ b/tests/find/test_basic_queries.py @@ -0,0 +1,136 @@ +""" +Basic find operation tests. + +Tests for fundamental find() and findOne() operations. +""" + +import pytest + +from tests.common.assertions import assert_document_match + + +@pytest.mark.find +@pytest.mark.smoke +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30, "status": "active"}, + {"name": "Bob", "age": 25, "status": "active"}, + {"name": "Charlie", "age": 35, "status": "inactive"}, + ] +) +def test_find_all_documents(collection): + """Test finding all documents in a collection.""" + # Execute find operation + result = list(collection.find()) + + # Verify all documents are returned + assert len(result) == 3, "Expected to find 3 documents" + + # Verify document content + names = {doc["name"] for doc in result} + assert names == {"Alice", "Bob", "Charlie"}, "Expected to find all three users" + + +@pytest.mark.find +@pytest.mark.smoke +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30, "status": "active"}, + {"name": "Bob", "age": 25, "status": "active"}, + {"name": "Charlie", "age": 35, "status": "inactive"}, + ] +) +def test_find_with_filter(collection): + """Test find operation with a simple equality filter.""" + # Execute find with filter + result = list(collection.find({"status": "active"})) + + # Verify only active users are returned + assert len(result) == 2, "Expected to find 2 active users" + + # Verify all returned documents have status "active" + for doc in result: + assert doc["status"] == "active", "All returned documents should have status 'active'" + + # Verify correct users are returned + names = {doc["name"] for doc in result} + assert names == {"Alice", "Bob"}, "Expected to find Alice and Bob" + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + ] +) +def test_find_one(collection): + """Test findOne operation returns a single document.""" + # Execute findOne + result = collection.find_one({"name": "Alice"}) + + # Verify document is returned + assert result is not None, "Expected to find a document" + + # Verify document content + assert_document_match(result, {"name": "Alice", "age": 30}) + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30}, + ] +) +def test_find_one_not_found(collection): + """Test findOne returns None when no document matches.""" + # Execute findOne with non-matching filter + result = collection.find_one({"name": "NonExistent"}) + + # Verify None is returned + assert result is None, "Expected None when no document matches" + + +@pytest.mark.find +def test_find_empty_collection(collection): + """Test find on an empty collection returns empty result.""" + # Execute find on empty collection + result = list(collection.find()) + + # Verify empty result + assert result == [], "Expected empty result for empty collection" + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30, "city": "NYC"}, + {"name": "Bob", "age": 25, "city": "SF"}, + {"name": "Charlie", "age": 35, "city": "NYC"}, + ] +) +def test_find_with_multiple_conditions(collection): + """Test find with multiple filter conditions (implicit AND).""" + # Execute find with multiple conditions + result = list(collection.find({"city": "NYC", "age": 30})) + + # Verify only matching document is returned + assert len(result) == 1, "Expected to find 1 document" + assert_document_match(result[0], {"name": "Alice", "age": 30, "city": "NYC"}) + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "profile": {"age": 30, "city": "NYC"}}, + {"name": "Bob", "profile": {"age": 25, "city": "SF"}}, + ] +) +def test_find_nested_field(collection): + """Test find with nested field query using dot notation.""" + # Execute find with nested field + result = list(collection.find({"profile.city": "NYC"})) + + # Verify correct document is returned + assert len(result) == 1, "Expected to find 1 document" + assert result[0]["name"] == "Alice", "Expected to find Alice" diff --git a/tests/find/test_projections.py b/tests/find/test_projections.py new file mode 100644 index 0000000..51c6e49 --- /dev/null +++ b/tests/find/test_projections.py @@ -0,0 +1,117 @@ +""" +Projection tests for find operations. + +Tests for field inclusion, exclusion, and projection operators. +""" + +import pytest + +from tests.common.assertions import assert_field_exists, assert_field_not_exists + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30, "email": "alice@example.com", "city": "NYC"}, + {"name": "Bob", "age": 25, "email": "bob@example.com", "city": "SF"}, + ] +) +def test_find_with_field_inclusion(collection): + """Test find with explicit field inclusion.""" + # Find with projection to include only name and age + result = list(collection.find({}, {"name": 1, "age": 1})) + + # Verify results + assert len(result) == 2, "Expected 2 documents" + + # Verify included fields exist + for doc in result: + assert_field_exists(doc, "_id") # _id is included by default + assert_field_exists(doc, "name") + assert_field_exists(doc, "age") + + # Verify excluded fields don't exist + assert_field_not_exists(doc, "email") + assert_field_not_exists(doc, "city") + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30, "email": "alice@example.com", "city": "NYC"}, + {"name": "Bob", "age": 25, "email": "bob@example.com", "city": "SF"}, + ] +) +def test_find_with_field_exclusion(collection): + """Test find with explicit field exclusion.""" + # Find with projection to exclude email and city + result = list(collection.find({}, {"email": 0, "city": 0})) + + # Verify results + assert len(result) == 2, "Expected 2 documents" + + # Verify included fields exist + for doc in result: + assert_field_exists(doc, "_id") + assert_field_exists(doc, "name") + assert_field_exists(doc, "age") + + # Verify excluded fields don't exist + assert_field_not_exists(doc, "email") + assert_field_not_exists(doc, "city") + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30, "email": "alice@example.com"}, + ] +) +def test_find_exclude_id(collection): + """Test find with _id exclusion.""" + # Find with projection to exclude _id + result = list(collection.find({}, {"_id": 0, "name": 1, "age": 1})) + + # Verify results + assert len(result) == 1, "Expected 1 document" + + # Verify _id is excluded + assert_field_not_exists(result[0], "_id") + assert_field_exists(result[0], "name") + assert_field_exists(result[0], "age") + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "profile": {"age": 30, "city": "NYC", "email": "alice@example.com"}}, + ] +) +def test_find_nested_field_projection(collection): + """Test find with nested field projection.""" + # Find with projection for nested field + result = list(collection.find({}, {"name": 1, "profile.age": 1})) + + # Verify results + assert len(result) == 1, "Expected 1 document" + assert_field_exists(result[0], "name") + assert_field_exists(result[0], "profile.age") + + # Verify other nested fields are not included + assert "city" not in result[0].get("profile", {}), "city should not be in profile" + assert "email" not in result[0].get("profile", {}), "email should not be in profile" + + +@pytest.mark.find +def test_find_empty_projection(collection): + """Test find with empty projection returns all fields.""" + # Insert test data + collection.insert_one({"name": "Alice", "age": 30}) + + # Find with empty projection + result = collection.find_one({}, {}) + + # Verify all fields are present + assert_field_exists(result, "_id") + assert_field_exists(result, "name") + assert_field_exists(result, "age") diff --git a/tests/find/test_query_operators.py b/tests/find/test_query_operators.py new file mode 100644 index 0000000..780f3d4 --- /dev/null +++ b/tests/find/test_query_operators.py @@ -0,0 +1,158 @@ +""" +Query operator tests for find operations. + +Tests for comparison operators: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin +""" + +import pytest + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ] +) +def test_find_gt_operator(collection): + """Test find with $gt (greater than) operator.""" + # Find documents where age > 25 + result = list(collection.find({"age": {"$gt": 25}})) + + # Verify results + assert len(result) == 2, "Expected 2 documents with age > 25" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Charlie"}, "Expected Alice and Charlie" + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ] +) +def test_find_gte_operator(collection): + """Test find with $gte (greater than or equal) operator.""" + # Find documents where age >= 30 + result = list(collection.find({"age": {"$gte": 30}})) + + # Verify results + assert len(result) == 2, "Expected 2 documents with age >= 30" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Charlie"}, "Expected Alice and Charlie" + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ] +) +def test_find_lt_operator(collection): + """Test find with $lt (less than) operator.""" + # Find documents where age < 30 + result = list(collection.find({"age": {"$lt": 30}})) + + # Verify results + assert len(result) == 1, "Expected 1 document with age < 30" + assert result[0]["name"] == "Bob", "Expected Bob" + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ] +) +def test_find_lte_operator(collection): + """Test find with $lte (less than or equal) operator.""" + # Find documents where age <= 30 + result = list(collection.find({"age": {"$lte": 30}})) + + # Verify results + assert len(result) == 2, "Expected 2 documents with age <= 30" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Bob"}, "Expected Alice and Bob" + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ] +) +def test_find_ne_operator(collection): + """Test find with $ne (not equal) operator.""" + # Find documents where age != 30 + result = list(collection.find({"age": {"$ne": 30}})) + + # Verify results + assert len(result) == 2, "Expected 2 documents with age != 30" + names = {doc["name"] for doc in result} + assert names == {"Bob", "Charlie"}, "Expected Bob and Charlie" + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "status": "active"}, + {"name": "Bob", "status": "inactive"}, + {"name": "Charlie", "status": "pending"}, + {"name": "David", "status": "active"}, + ] +) +def test_find_in_operator(collection): + """Test find with $in operator.""" + # Find documents where status is in ["active", "pending"] + result = list(collection.find({"status": {"$in": ["active", "pending"]}})) + + # Verify results + assert len(result) == 3, "Expected 3 documents with status in [active, pending]" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Charlie", "David"}, "Expected Alice, Charlie, and David" + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "status": "active"}, + {"name": "Bob", "status": "inactive"}, + {"name": "Charlie", "status": "pending"}, + ] +) +def test_find_nin_operator(collection): + """Test find with $nin (not in) operator.""" + # Find documents where status is not in ["active", "pending"] + result = list(collection.find({"status": {"$nin": ["active", "pending"]}})) + + # Verify results + assert len(result) == 1, "Expected 1 document with status not in [active, pending]" + assert result[0]["name"] == "Bob", "Expected Bob" + + +@pytest.mark.find +@pytest.mark.documents( + [ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ] +) +def test_find_range_query(collection): + """Test find with range query (combining $gte and $lte).""" + # Find documents where 25 <= age <= 30 + result = list(collection.find({"age": {"$gte": 25, "$lte": 30}})) + + # Verify results + assert len(result) == 2, "Expected 2 documents with age between 25 and 30" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Bob"}, "Expected Alice and Bob" diff --git a/tests/insert/__init__.py b/tests/insert/__init__.py new file mode 100644 index 0000000..47ae134 --- /dev/null +++ b/tests/insert/__init__.py @@ -0,0 +1 @@ +"""Insert operation tests.""" diff --git a/tests/insert/test_insert_operations.py b/tests/insert/test_insert_operations.py new file mode 100644 index 0000000..659d26b --- /dev/null +++ b/tests/insert/test_insert_operations.py @@ -0,0 +1,145 @@ +""" +Insert operation tests. + +Tests for insertOne, insertMany, and related operations. +""" + +import pytest +from bson import ObjectId + +from tests.common.assertions import assert_count + + +@pytest.mark.insert +@pytest.mark.smoke +def test_insert_one_document(collection): + """Test inserting a single document.""" + # Insert one document + document = {"name": "Alice", "age": 30} + result = collection.insert_one(document) + + # Verify insert was acknowledged + assert result.acknowledged, "Insert should be acknowledged" + assert result.inserted_id is not None, "Should return inserted _id" + assert isinstance(result.inserted_id, ObjectId), "Inserted _id should be ObjectId" + + # Verify document exists in collection + assert_count(collection, {}, 1) + + # Verify document content + found = collection.find_one({"name": "Alice"}) + assert found is not None, "Document should exist" + assert found["name"] == "Alice" + assert found["age"] == 30 + + +@pytest.mark.insert +@pytest.mark.smoke +def test_insert_many_documents(collection): + """Test inserting multiple documents.""" + # Insert multiple documents + documents = [ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ] + result = collection.insert_many(documents) + + # Verify insert was acknowledged + assert result.acknowledged, "Insert should be acknowledged" + assert len(result.inserted_ids) == 3, "Should return 3 inserted IDs" + + # Verify all IDs are ObjectIds + for inserted_id in result.inserted_ids: + assert isinstance(inserted_id, ObjectId), "Each inserted _id should be ObjectId" + + # Verify all documents exist + assert_count(collection, {}, 3) + + +@pytest.mark.insert +def test_insert_with_custom_id(collection): + """Test inserting document with custom _id.""" + # Insert document with custom _id + custom_id = "custom_123" + document = {"_id": custom_id, "name": "Alice"} + result = collection.insert_one(document) + + # Verify custom _id is used + assert result.inserted_id == custom_id, "Should use custom _id" + + # Verify document can be retrieved by custom _id + found = collection.find_one({"_id": custom_id}) + assert found is not None, "Document should exist" + assert found["name"] == "Alice" + + +@pytest.mark.insert +def test_insert_duplicate_id_fails(collection): + """Test that inserting duplicate _id raises error.""" + # Insert first document + document = {"_id": "duplicate_id", "name": "Alice"} + collection.insert_one(document) + + # Try to insert document with same _id + duplicate = {"_id": "duplicate_id", "name": "Bob"} + + # Should raise exception + with pytest.raises(Exception): # DuplicateKeyError or similar + collection.insert_one(duplicate) + + # Verify only first document exists + assert_count(collection, {}, 1) + found = collection.find_one({"_id": "duplicate_id"}) + assert found["name"] == "Alice", "Should have first document" + + +@pytest.mark.insert +def test_insert_nested_document(collection): + """Test inserting document with nested structure.""" + # Insert document with nested fields + document = { + "name": "Alice", + "profile": {"age": 30, "address": {"city": "NYC", "country": "USA"}}, + } + result = collection.insert_one(document) + + # Verify insert + assert result.inserted_id is not None + + # Verify nested structure is preserved + found = collection.find_one({"name": "Alice"}) + assert found["profile"]["age"] == 30 + assert found["profile"]["address"]["city"] == "NYC" + + +@pytest.mark.insert +def test_insert_array_field(collection): + """Test inserting document with array fields.""" + # Insert document with array + document = {"name": "Alice", "tags": ["python", "mongodb", "testing"], "scores": [95, 87, 92]} + result = collection.insert_one(document) + + # Verify insert + assert result.inserted_id is not None + + # Verify array fields are preserved + found = collection.find_one({"name": "Alice"}) + assert found["tags"] == ["python", "mongodb", "testing"] + assert found["scores"] == [95, 87, 92] + + +@pytest.mark.insert +def test_insert_empty_document(collection): + """Test inserting an empty document.""" + # Insert empty document + result = collection.insert_one({}) + + # Verify insert was successful + assert result.inserted_id is not None + + # Verify document exists (only has _id) + found = collection.find_one({"_id": result.inserted_id}) + assert found is not None + assert len(found) == 1 # Only _id field + assert "_id" in found From d1e49c8e55bb2cc5c2cd5fe8a8f33578914eadab Mon Sep 17 00:00:00 2001 From: Nitin Ahuja Date: Mon, 12 Jan 2026 10:07:34 -0800 Subject: [PATCH 2/9] Fix tests to consistently use @pytest.mark.documents decorator - Update test_find_empty_projection to use documents marker instead of manual insert - Update test_match_empty_result to use documents marker instead of manual insert - Ensures consistent test data setup and automatic cleanup - All 36 tests still passing --- tests/aggregate/test_match_stage.py | 14 ++++++-------- tests/find/test_projections.py | 4 +--- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/aggregate/test_match_stage.py b/tests/aggregate/test_match_stage.py index 886b78e..05640af 100644 --- a/tests/aggregate/test_match_stage.py +++ b/tests/aggregate/test_match_stage.py @@ -69,16 +69,14 @@ def test_match_multiple_conditions(collection): @pytest.mark.aggregate +@pytest.mark.documents( + [ + {"name": "Alice", "status": "active"}, + {"name": "Bob", "status": "active"}, + ] +) def test_match_empty_result(collection): """Test $match stage that matches no documents.""" - # Insert test data - collection.insert_many( - [ - {"name": "Alice", "status": "active"}, - {"name": "Bob", "status": "active"}, - ] - ) - # Execute aggregation with $match that matches nothing pipeline = [{"$match": {"status": "inactive"}}] result = list(collection.aggregate(pipeline)) diff --git a/tests/find/test_projections.py b/tests/find/test_projections.py index 51c6e49..c841f63 100644 --- a/tests/find/test_projections.py +++ b/tests/find/test_projections.py @@ -103,11 +103,9 @@ def test_find_nested_field_projection(collection): @pytest.mark.find +@pytest.mark.documents([{"name": "Alice", "age": 30}]) def test_find_empty_projection(collection): """Test find with empty projection returns all fields.""" - # Insert test data - collection.insert_one({"name": "Alice", "age": 30}) - # Find with empty projection result = collection.find_one({}, {}) From 51d29bfbbfc7e72fa8c4bb39dcf4f98d4a7c91b6 Mon Sep 17 00:00:00 2001 From: Nitin Ahuja Date: Mon, 12 Jan 2026 10:18:00 -0800 Subject: [PATCH 3/9] Fix pytest.ini warnings by removing invalid config options - Remove json_report, json_report_indent, json_report_omit from config - These are command-line options, not pytest.ini settings - Add comment explaining proper usage - All 36 tests still passing with no warnings --- pytest.ini | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pytest.ini b/pytest.ini index 3cc2c44..f9c263e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -45,7 +45,6 @@ timeout = 300 # Use with: pytest -n auto # Or: pytest -n 4 (for 4 processes) -# JSON report configuration -json_report = .test-results/report.json -json_report_indent = 2 -json_report_omit = log +# JSON report options +# These are command-line options, not config file options +# Use with: pytest --json-report --json-report-file=.test-results/report.json From c0cda4bd2dd6bcaca457f14b7c377321e6f96326 Mon Sep 17 00:00:00 2001 From: Nitin Ahuja Date: Mon, 12 Jan 2026 12:07:11 -0800 Subject: [PATCH 4/9] Add multi-platform Docker build workflow - Add GitHub Actions workflow for automated Docker builds - Build for linux/amd64 and linux/arm64 platforms - Push to GitHub Container Registry (ghcr.io) - Auto-tags images: latest, sha-*, version tags - Update README with pre-built image pull instructions - Fix Dockerfile casing warning (FROM...AS) Workflow Features: - Runs on push to main and on pull requests - Multi-platform support for Intel/AMD and ARM/Graviton - Automatic versioning from git tags - GitHub Actions cache for faster builds - Uses dynamic repository variable (works on forks and upstream) --- .github/workflows/docker-build.yml | 68 ++++++++++++++++++++++++++++++ Dockerfile | 2 +- README.md | 37 +++++++++++++++- 3 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/docker-build.yml diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 0000000..c223330 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,68 @@ +name: Build and Push Docker Image + +on: + push: + branches: + - main + tags: + - 'v*.*.*' + pull_request: + branches: + - main + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,prefix=sha- + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Image digest + run: echo ${{ steps.meta.outputs.tags }} diff --git a/Dockerfile b/Dockerfile index 9fc7292..d1cc5ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Multi-stage build for DocumentDB Functional Tests # Stage 1: Build stage -FROM python:3.11-slim as builder +FROM python:3.11-slim AS builder WORKDIR /build diff --git a/README.md b/README.md index 6db5fa9..0476a5a 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,42 @@ pytest -l ## Docker Usage -### Build the Image +### Option 1: Use Pre-built Image (Recommended) + +Pull the latest image from GitHub Container Registry: + +```bash +# Pull latest version +docker pull ghcr.io/documentdb/functional-tests:latest + +# Or pull a specific version +docker pull ghcr.io/documentdb/functional-tests:v1.0.0 +``` + +Run tests with the pre-built image: + +```bash +# Run all tests +docker run --network host \ + ghcr.io/documentdb/functional-tests:latest \ + --engine documentdb="mongodb://user:pass@host:port/?tls=true&tlsAllowInvalidCertificates=true" + +# Run specific tags +docker run --network host \ + ghcr.io/documentdb/functional-tests:latest \ + -m smoke \ + --engine documentdb="mongodb://localhost:10260/?tls=true" + +# Run with parallel execution +docker run --network host \ + ghcr.io/documentdb/functional-tests:latest \ + -n 4 \ + --engine documentdb="mongodb://localhost:27017" +``` + +### Option 2: Build Locally + +If you need to build from source: ```bash docker build -t documentdb/functional-tests . From 57fb249ddfdc14f54d983766d118c3d7374aa35b Mon Sep 17 00:00:00 2001 From: Nitin Ahuja Date: Mon, 12 Jan 2026 12:17:42 -0800 Subject: [PATCH 5/9] Fix workflow error: remove problematic image digest step - Remove the 'Image digest' step that was causing exit code 127 - The metadata and tags are already captured by the build step - Build step itself will show all relevant information in logs --- .github/workflows/docker-build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index c223330..135473e 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -63,6 +63,3 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - - - name: Image digest - run: echo ${{ steps.meta.outputs.tags }} From e4f5c2eb8d89f2c1dcfefc2670bef9a33508844d Mon Sep 17 00:00:00 2001 From: Nitin Ahuja Date: Wed, 4 Feb 2026 10:57:16 -0800 Subject: [PATCH 6/9] refactor: Adopt standard pytest patterns and improve analyzer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove custom @pytest.mark.documents pattern (29 tests refactored) - Use direct data insertion with Arrange-Act-Assert structure - Simplify collection fixture (remove marker handling logic) - Rename FailureType → TestOutcome (more accurate) - Fix infrastructure error detection (exception-based, not keyword-based) - Add dynamic marker loading from pytest.ini (eliminate duplication) - Optimize analyzer with module-level constants and simplified logic - Fix database/collection name collisions for parallel execution - Fix SKIPPED categorization (raise ConnectionError for infra issues) All tests passing (36/37, 1 expected unsupported feature). --- conftest.py | 143 +++++++------ result_analyzer/__init__.py | 4 +- result_analyzer/analyzer.py | 302 ++++++++++++++++++++-------- result_analyzer/report_generator.py | 3 + tests/aggregate/test_group_stage.py | 80 ++++---- tests/aggregate/test_match_stage.py | 65 +++--- tests/find/test_basic_queries.py | 100 +++++---- tests/find/test_projections.py | 68 ++++--- tests/find/test_query_operators.py | 129 ++++++------ 9 files changed, 518 insertions(+), 376 deletions(-) diff --git a/conftest.py b/conftest.py index 422e021..a9b17e4 100644 --- a/conftest.py +++ b/conftest.py @@ -8,6 +8,7 @@ """ +import hashlib import pytest from pymongo import MongoClient @@ -15,74 +16,68 @@ def pytest_addoption(parser): """Add custom command-line options for pytest.""" parser.addoption( - "--engine", - action="append", - default=[], - help="Engine to test against. Format: name=connection_string. " - "Example: --engine documentdb=mongodb://localhost:27017 " - "--engine mongodb=mongodb://mongo:27017", + "--connection-string", + action="store", + default=None, + help="Database connection string. " + "Example: --connection-string mongodb://localhost:27017", + ) + parser.addoption( + "--engine-name", + action="store", + default="default", + help="Optional engine identifier for metadata. " + "Example: --engine-name documentdb", ) def pytest_configure(config): """Configure pytest with custom settings.""" - # Parse engine configurations - engines = {} - for engine_spec in config.getoption("--engine"): - if "=" not in engine_spec: - raise ValueError( - f"Invalid engine specification: {engine_spec}. " - "Expected format: name=connection_string" - ) - name, connection_string = engine_spec.split("=", 1) - engines[name] = connection_string + # Get connection string and engine name + connection_string = config.getoption("--connection-string") + engine_name = config.getoption("--engine-name") # Store in config for access by fixtures - config.engines = engines - - # If no engines specified, default to localhost - if not engines: - config.engines = {"default": "mongodb://localhost:27017"} - + config.connection_string = connection_string + config.engine_name = engine_name -def pytest_generate_tests(metafunc): - """ - Parametrize tests to run against multiple engines. - - Tests that use the 'engine_client' fixture will automatically - run against all configured engines. - """ - if "engine_client" in metafunc.fixturenames: - engines = metafunc.config.engines - metafunc.parametrize( - "engine_name,engine_connection_string", - [(name, conn) for name, conn in engines.items()], - ids=list(engines.keys()), - scope="function", - ) + # If no connection string specified, default to localhost + if not connection_string: + config.connection_string = "mongodb://localhost:27017" + if engine_name == "default": + # If using default connection and no engine name specified, set to "default" + config.engine_name = "default" @pytest.fixture(scope="function") -def engine_client(engine_name, engine_connection_string): +def engine_client(request): """ - Create a MongoDB client for the specified engine. - - This fixture is parametrized to run tests against all configured engines. + Create a MongoDB client for the configured engine. Args: - engine_name: Name of the engine (from --engine option) - engine_connection_string: Connection string for the engine + request: pytest request object Yields: MongoClient: Connected MongoDB client + + Raises: + ConnectionError: If unable to connect to the database """ - client = MongoClient(engine_connection_string) + connection_string = request.config.connection_string + engine_name = request.config.engine_name + + client = MongoClient(connection_string) # Verify connection try: client.admin.command("ping") except Exception as e: - pytest.skip(f"Cannot connect to {engine_name}: {e}") + # Close the client before raising + client.close() + # Raise ConnectionError so analyzer categorizes as INFRA_ERROR + raise ConnectionError( + f"Cannot connect to {engine_name} at {connection_string}: {e}" + ) from e yield client @@ -95,7 +90,8 @@ def database_client(engine_client, request): """ Provide a database client with automatic cleanup. - Creates a test database named after the test function for isolation. + Creates a test database with a collision-free name for parallel execution. + The name includes worker ID, hash, and abbreviated test name. Automatically drops the database after the test completes. Args: @@ -105,9 +101,22 @@ def database_client(engine_client, request): Yields: Database: MongoDB database object """ - # Create unique database name based on test name + # Get worker ID for parallel execution (e.g., 'gw0', 'gw1', or 'main' for single process) + worker_id = getattr(request.config, 'workerinput', {}).get('workerid', 'main') + + # Get full test identifier (includes file path and test name) + full_test_id = request.node.nodeid + + # Create a short hash for uniqueness (first 8 chars of SHA256) + name_hash = hashlib.sha256(full_test_id.encode()).hexdigest()[:8] + + # Get abbreviated test name for readability (sanitize and truncate) test_name = request.node.name.replace("[", "_").replace("]", "_") - db_name = f"test_{test_name}"[:63] # MongoDB database name limit + abbreviated = test_name[:20] + + # Combine: test_{worker}_{hash}_{abbreviated} + # Example: test_gw0_a1b2c3d4_find_all_documents + db_name = f"test_{worker_id}_{name_hash}_{abbreviated}"[:63] # MongoDB limit db = engine_client[db_name] @@ -123,29 +132,37 @@ def database_client(engine_client, request): @pytest.fixture(scope="function") def collection(database_client, request): """ - Provide a collection with automatic test data setup and cleanup. + Provide an empty collection with automatic cleanup. - If the test is marked with @pytest.mark.documents([...]), this fixture - will automatically insert those documents before the test runs. + Creates a collection with a collision-free name for parallel execution. + Tests should directly insert any required test data. Args: database_client: Database from database_client fixture request: pytest request object Yields: - Collection: MongoDB collection object + Collection: Empty MongoDB collection object """ - # Use test name as collection name for isolation - collection_name = request.node.name.replace("[", "_").replace("]", "_")[:100] + # Get worker ID for parallel execution (e.g., 'gw0', 'gw1', or 'main') + worker_id = getattr(request.config, 'workerinput', {}).get('workerid', 'main') + + # Get full test identifier + full_test_id = request.node.nodeid + + # Create a short hash for uniqueness (first 8 chars of SHA256) + name_hash = hashlib.sha256(full_test_id.encode()).hexdigest()[:8] + + # Get abbreviated test name for readability (sanitize and truncate) + test_name = request.node.name.replace("[", "_").replace("]", "_") + abbreviated = test_name[:25] + + # Combine: coll_{worker}_{hash}_{abbreviated} + # Example: coll_gw0_a1b2c3d4_find_all_documents + collection_name = f"coll_{worker_id}_{name_hash}_{abbreviated}"[:100] # Collection name limit + coll = database_client[collection_name] - # Check if test has @pytest.mark.documents decorator - marker = request.node.get_closest_marker("documents") - if marker and marker.args: - documents = marker.args[0] - if documents: - coll.insert_many(documents) - yield coll # Cleanup: drop collection @@ -153,7 +170,3 @@ def collection(database_client, request): coll.drop() except Exception: pass # Best effort cleanup - - -# Custom marker for test data setup -pytest.mark.documents = pytest.mark.documents diff --git a/result_analyzer/__init__.py b/result_analyzer/__init__.py index 63faf0a..dd179b8 100644 --- a/result_analyzer/__init__.py +++ b/result_analyzer/__init__.py @@ -5,7 +5,7 @@ reports categorized by feature tags. """ -from .analyzer import analyze_results, categorize_failure +from .analyzer import analyze_results, categorize_outcome from .report_generator import generate_report, print_summary -__all__ = ["analyze_results", "categorize_failure", "generate_report", "print_summary"] +__all__ = ["analyze_results", "categorize_outcome", "generate_report", "print_summary"] diff --git a/result_analyzer/analyzer.py b/result_analyzer/analyzer.py index af2b2a1..156cf01 100644 --- a/result_analyzer/analyzer.py +++ b/result_analyzer/analyzer.py @@ -6,77 +6,246 @@ """ import json +import re from collections import defaultdict +from pathlib import Path from typing import Any, Dict, List -class FailureType: - """Enumeration of failure types.""" +# Module-level constants and caches +INFRA_EXCEPTIONS = { + # Python built-in connection errors + "ConnectionError", + "ConnectionRefusedError", + "ConnectionResetError", + "ConnectionAbortedError", + # Python timeout errors + "TimeoutError", + "socket.timeout", + "socket.error", + # PyMongo connection errors + "pymongo.errors.ConnectionFailure", + "pymongo.errors.ServerSelectionTimeoutError", + "pymongo.errors.NetworkTimeout", + "pymongo.errors.AutoReconnect", + "pymongo.errors.ExecutionTimeout", + # Generic network/OS errors + "OSError", +} + +_REGISTERED_MARKERS_CACHE = None + + +# Mapping from TestOutcome to counter key names +OUTCOME_TO_KEY = { + "PASS": "passed", + "FAIL": "failed", + "UNSUPPORTED": "unsupported", + "SKIPPED": "skipped", + "INFRA_ERROR": "infra_error", +} + + +class TestOutcome: + """Enumeration of test outcomes.""" PASS = "PASS" FAIL = "FAIL" - UNSUPPORTED = "UNSUPPORTED" + UNSUPPORTED = "UNSUPPORTED" # Error code 115 - feature not implemented + SKIPPED = "SKIPPED" # Environmental or conditional skip INFRA_ERROR = "INFRA_ERROR" -def categorize_failure(test_result: Dict[str, Any]) -> str: +def categorize_outcome(test_result: Dict[str, Any]) -> str: """ - Categorize a test failure based on error information. + Categorize a test outcome based on test result information. + + Categories: + - PASS: Test passed + - UNSUPPORTED: Failed with error code 115 (unsupported feature) + - INFRA_ERROR: Infrastructure issues (connection, timeout, etc.) + - FAIL: Functional failure (bug/incorrect behavior) + - SKIPPED: Environmental or conditional skip Args: test_result: Test result dictionary from pytest JSON report Returns: - One of: PASS, FAIL, UNSUPPORTED, INFRA_ERROR + One of: PASS, FAIL, UNSUPPORTED, SKIPPED, INFRA_ERROR """ outcome = test_result.get("outcome", "") if outcome == "passed": - return FailureType.PASS - - if outcome == "skipped": - # Skipped tests typically indicate unsupported features - return FailureType.UNSUPPORTED + return TestOutcome.PASS if outcome == "failed": - # Analyze the failure to determine if it's infrastructure or functionality + # Analyze the failure to determine the type call_info = test_result.get("call", {}) longrepr = call_info.get("longrepr", "") - # Check for infrastructure-related errors - infra_keywords = [ - "connection", - "timeout", - "network", - "cannot connect", - "refused", - "unreachable", - "host", - ] + # Check for error code 115 (unsupported feature) - highest priority + if is_unsupported_error(longrepr): + return TestOutcome.UNSUPPORTED - if any(keyword in longrepr.lower() for keyword in infra_keywords): - return FailureType.INFRA_ERROR + # Check for infrastructure-related errors (pass full test_result) + if is_infrastructure_error(test_result): + return TestOutcome.INFRA_ERROR # Otherwise, it's a functional failure - return FailureType.FAIL + return TestOutcome.FAIL + + if outcome == "skipped": + # Environmental or conditional skip + return TestOutcome.SKIPPED # Unknown outcome, treat as infrastructure error - return FailureType.INFRA_ERROR + return TestOutcome.INFRA_ERROR + + +def is_unsupported_error(error_text: str) -> bool: + """ + Check if error contains code 115 (unsupported feature). + + DocumentDB returns error code 115 for unsupported features in the + error dictionary as 'code': 115 or "code": 115. + + Args: + error_text: Error message text from test failure + + Returns: + True if error contains code 115, False otherwise + """ + if not error_text: + return False + + # Match 'code': 115 or "code": 115 in error dict + return bool(re.search(r"['\"]code['\"]:\s*115", error_text)) + + +def extract_exception_type(crash_message: str) -> str: + """ + Extract exception type from pytest crash message. + + Args: + crash_message: Message like "module.Exception: error details" + + Returns: + Full exception type (e.g., "pymongo.errors.OperationFailure") + or empty string if not found + """ + # Match pattern: "module.exception.Type: message" + # Capture everything before the first colon + match = re.match(r'^([a-zA-Z0-9_.]+):\s', crash_message) + if match: + return match.group(1) + + return "" + + +def is_infrastructure_error(test_result: Dict[str, Any]) -> bool: + """ + Check if error is infrastructure-related based on exception type. + + This checks the actual exception type rather than keywords in error messages, + preventing false positives from error messages that happen to contain + infrastructure-related words (e.g., "host" in an assertion message). + + Args: + test_result: Full test result dict from pytest JSON + + Returns: + True if error is infrastructure-related, False otherwise + """ + # Get the crash info from call + call_info = test_result.get("call", {}) + crash_info = call_info.get("crash", {}) + crash_message = crash_info.get("message", "") + + if not crash_message: + return False + + # Extract exception type from "module.ExceptionClass: message" format + exception_type = extract_exception_type(crash_message) + + if not exception_type: + return False + + # Check against module-level constant + return exception_type in INFRA_EXCEPTIONS + + +def load_registered_markers(pytest_ini_path: str = "pytest.ini") -> set: + """ + Load registered markers from pytest.ini. + + Parses the markers section to extract marker names, ensuring we only + use markers that are explicitly registered in pytest configuration. + + Args: + pytest_ini_path: Path to pytest.ini file (defaults to "pytest.ini") + + Returns: + Set of registered marker names + """ + # Check if pytest.ini exists + if not Path(pytest_ini_path).exists(): + return set() + + registered_markers = set() + + try: + with open(pytest_ini_path, 'r') as f: + in_markers_section = False + + for line in f: + # Check if we're entering the markers section + if line.strip() == "markers =": + in_markers_section = True + continue + + if in_markers_section: + # Marker lines are indented, config keys are not + if line and not line[0].isspace(): + # Non-indented line means we left the markers section + break + + # Parse indented marker lines like " find: Find operation tests" + match = re.match(r'^\s+([a-zA-Z0-9_]+):', line) + if match: + registered_markers.add(match.group(1)) + + except Exception: + # If parsing fails, return empty set + pass + + return registered_markers def extract_markers(test_result: Dict[str, Any]) -> List[str]: """ Extract pytest markers (tags) from a test result. - Automatically filters out test names, file names, and pytest internal markers - using heuristics, keeping only meaningful test categorization markers. + Uses registered markers from pytest.ini as an allow list. + This ensures only intentional test categorization markers are included, + avoiding brittle heuristics that could break with future pytest versions. + + Markers are automatically loaded from pytest.ini, so they only need to be + defined in one place. Args: test_result: Test result dictionary from pytest JSON report Returns: - List of marker names + List of marker names that match registered markers from pytest.ini """ + global _REGISTERED_MARKERS_CACHE + + # Load registered markers from pytest.ini (cached for performance) + if _REGISTERED_MARKERS_CACHE is None: + _REGISTERED_MARKERS_CACHE = load_registered_markers() + + registered_markers = _REGISTERED_MARKERS_CACHE + markers = [] # Extract from keywords @@ -93,43 +262,8 @@ def extract_markers(test_result: Dict[str, Any]) -> List[str]: else: markers.append(str(marker)) - # Filter out non-meaningful markers using heuristics - filtered_markers = [] - for marker in markers: - # Skip empty strings - if not marker: - continue - - # Skip if it looks like a test function name (contains brackets or starts with test_) - if "[" in marker or marker.startswith("test_"): - continue - - # Skip if it looks like a file name (contains .py or /) - if ".py" in marker or "/" in marker: - continue - - # Skip if it's a directory name (common directory names) - if marker in {"tests", "functional-tests", "src", "lib"}: - continue - - # Skip pytest internal markers - if marker in {"parametrize", "usefixtures", "filterwarnings", "pytestmark"}: - continue - - # Skip fixture markers (markers used for setup, not categorization) - # These are markers that take arguments in the decorator - if marker in {"documents"}: - continue - - # Skip engine names (from parametrization like [documentdb], [mongodb]) - # Common engine names that appear in test results - if marker in {"documentdb", "mongodb", "cosmosdb", "default"}: - continue - - # If it passed all filters, it's likely a meaningful marker - filtered_markers.append(marker) - - return filtered_markers + # Filter to only registered markers + return [m for m in markers if m in registered_markers] def analyze_results(json_report_path: str) -> Dict[str, Any]: @@ -179,11 +313,12 @@ def analyze_results(json_report_path: str) -> Dict[str, Any]: "passed": 0, "failed": 0, "unsupported": 0, + "skipped": 0, "infra_error": 0, } by_tag: Dict[str, Dict[str, int]] = defaultdict( - lambda: {"passed": 0, "failed": 0, "unsupported": 0, "infra_error": 0} + lambda: {"passed": 0, "failed": 0, "unsupported": 0, "skipped": 0, "infra_error": 0} ) tests_details = [] @@ -193,43 +328,32 @@ def analyze_results(json_report_path: str) -> Dict[str, Any]: for test in tests: summary["total"] += 1 - # Categorize the result - failure_type = categorize_failure(test) + # Categorize the outcome + test_outcome = categorize_outcome(test) # Extract tags tags = extract_markers(test) - # Update summary counters - if failure_type == FailureType.PASS: - summary["passed"] += 1 - elif failure_type == FailureType.FAIL: - summary["failed"] += 1 - elif failure_type == FailureType.UNSUPPORTED: - summary["unsupported"] += 1 - elif failure_type == FailureType.INFRA_ERROR: - summary["infra_error"] += 1 + # Update summary counters using mapping + counter_key = OUTCOME_TO_KEY.get(test_outcome) + if counter_key: + summary[counter_key] += 1 # Update tag-specific counters - for tag in tags: - if failure_type == FailureType.PASS: - by_tag[tag]["passed"] += 1 - elif failure_type == FailureType.FAIL: - by_tag[tag]["failed"] += 1 - elif failure_type == FailureType.UNSUPPORTED: - by_tag[tag]["unsupported"] += 1 - elif failure_type == FailureType.INFRA_ERROR: - by_tag[tag]["infra_error"] += 1 + if counter_key: + for tag in tags: + by_tag[tag][counter_key] += 1 # Store test details test_detail = { "name": test.get("nodeid", ""), - "outcome": failure_type, + "outcome": test_outcome, "duration": test.get("duration", 0), "tags": tags, } # Add error information if failed - if failure_type in [FailureType.FAIL, FailureType.INFRA_ERROR]: + if test_outcome in [TestOutcome.FAIL, TestOutcome.INFRA_ERROR]: call_info = test.get("call", {}) test_detail["error"] = call_info.get("longrepr", "") diff --git a/result_analyzer/report_generator.py b/result_analyzer/report_generator.py index c0d2b26..ed6b44c 100644 --- a/result_analyzer/report_generator.py +++ b/result_analyzer/report_generator.py @@ -71,6 +71,7 @@ def generate_text_report(analysis: Dict[str, Any], output_path: str): lines.append(f"Passed: {summary['passed']} ({summary['pass_rate']}%)") lines.append(f"Failed: {summary['failed']}") lines.append(f"Unsupported: {summary['unsupported']}") + lines.append(f"Skipped: {summary['skipped']}") lines.append(f"Infrastructure Errors: {summary['infra_error']}") lines.append("") @@ -88,6 +89,7 @@ def generate_text_report(analysis: Dict[str, Any], output_path: str): lines.append(f" Passed: {stats['passed']} ({stats['pass_rate']}%)") lines.append(f" Failed: {stats['failed']}") lines.append(f" Unsupported: {stats['unsupported']}") + lines.append(f" Skipped: {stats['skipped']}") lines.append(f" Infra Error: {stats['infra_error']}") else: lines.append("No tags found in test results.") @@ -130,5 +132,6 @@ def print_summary(analysis: Dict[str, Any]): print(f"Passed: {summary['passed']} ({summary['pass_rate']}%)") print(f"Failed: {summary['failed']}") print(f"Unsupported: {summary['unsupported']}") + print(f"Skipped: {summary['skipped']}") print(f"Infra Error: {summary['infra_error']}") print("=" * 60 + "\n") diff --git a/tests/aggregate/test_group_stage.py b/tests/aggregate/test_group_stage.py index 1a51748..3af0710 100644 --- a/tests/aggregate/test_group_stage.py +++ b/tests/aggregate/test_group_stage.py @@ -9,21 +9,21 @@ @pytest.mark.aggregate @pytest.mark.smoke -@pytest.mark.documents( - [ +def test_group_with_count(collection): + """Test $group stage with count aggregation.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "department": "Engineering", "salary": 100000}, {"name": "Bob", "department": "Engineering", "salary": 90000}, {"name": "Charlie", "department": "Sales", "salary": 80000}, {"name": "David", "department": "Sales", "salary": 75000}, - ] -) -def test_group_with_count(collection): - """Test $group stage with count aggregation.""" - # Execute aggregation to count documents by department + ]) + + # Act - Execute aggregation to count documents by department pipeline = [{"$group": {"_id": "$department", "count": {"$sum": 1}}}] result = list(collection.aggregate(pipeline)) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 departments" # Convert to dict for easier verification @@ -33,20 +33,20 @@ def test_group_with_count(collection): @pytest.mark.aggregate -@pytest.mark.documents( - [ +def test_group_with_sum(collection): + """Test $group stage with sum aggregation.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "department": "Engineering", "salary": 100000}, {"name": "Bob", "department": "Engineering", "salary": 90000}, {"name": "Charlie", "department": "Sales", "salary": 80000}, - ] -) -def test_group_with_sum(collection): - """Test $group stage with sum aggregation.""" - # Execute aggregation to sum salaries by department + ]) + + # Act - Execute aggregation to sum salaries by department pipeline = [{"$group": {"_id": "$department", "totalSalary": {"$sum": "$salary"}}}] result = list(collection.aggregate(pipeline)) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 departments" # Convert to dict for easier verification @@ -56,20 +56,20 @@ def test_group_with_sum(collection): @pytest.mark.aggregate -@pytest.mark.documents( - [ +def test_group_with_avg(collection): + """Test $group stage with average aggregation.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "department": "Engineering", "salary": 100000}, {"name": "Bob", "department": "Engineering", "salary": 90000}, {"name": "Charlie", "department": "Sales", "salary": 80000}, - ] -) -def test_group_with_avg(collection): - """Test $group stage with average aggregation.""" - # Execute aggregation to calculate average salary by department + ]) + + # Act - Execute aggregation to calculate average salary by department pipeline = [{"$group": {"_id": "$department", "avgSalary": {"$avg": "$salary"}}}] result = list(collection.aggregate(pipeline)) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 departments" # Convert to dict for easier verification @@ -79,16 +79,16 @@ def test_group_with_avg(collection): @pytest.mark.aggregate -@pytest.mark.documents( - [ +def test_group_with_min_max(collection): + """Test $group stage with min and max aggregations.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "department": "Engineering", "salary": 100000}, {"name": "Bob", "department": "Engineering", "salary": 90000}, {"name": "Charlie", "department": "Sales", "salary": 80000}, - ] -) -def test_group_with_min_max(collection): - """Test $group stage with min and max aggregations.""" - # Execute aggregation to find min and max salary by department + ]) + + # Act - Execute aggregation to find min and max salary by department pipeline = [ { "$group": { @@ -100,7 +100,7 @@ def test_group_with_min_max(collection): ] result = list(collection.aggregate(pipeline)) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 departments" # Verify Engineering department @@ -110,19 +110,19 @@ def test_group_with_min_max(collection): @pytest.mark.aggregate -@pytest.mark.documents( - [ +def test_group_all_documents(collection): + """Test $group stage grouping all documents (using null as _id).""" + # Arrange - Insert test data + collection.insert_many([ {"item": "A", "quantity": 5}, {"item": "B", "quantity": 10}, {"item": "A", "quantity": 3}, - ] -) -def test_group_all_documents(collection): - """Test $group stage grouping all documents (using null as _id).""" - # Execute aggregation to sum quantities across all documents + ]) + + # Act - Execute aggregation to sum quantities across all documents pipeline = [{"$group": {"_id": None, "totalQuantity": {"$sum": "$quantity"}}}] result = list(collection.aggregate(pipeline)) - # Verify results + # Assert - Verify results assert len(result) == 1, "Expected single result grouping all documents" assert result[0]["totalQuantity"] == 18, "Expected total quantity of 18" diff --git a/tests/aggregate/test_match_stage.py b/tests/aggregate/test_match_stage.py index 05640af..0643fea 100644 --- a/tests/aggregate/test_match_stage.py +++ b/tests/aggregate/test_match_stage.py @@ -9,77 +9,78 @@ @pytest.mark.aggregate @pytest.mark.smoke -@pytest.mark.documents( - [ +def test_match_simple_filter(collection): + """Test $match stage with simple equality filter.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "age": 30, "status": "active"}, {"name": "Bob", "age": 25, "status": "active"}, {"name": "Charlie", "age": 35, "status": "inactive"}, - ] -) -def test_match_simple_filter(collection): - """Test $match stage with simple equality filter.""" - # Execute aggregation with $match stage + ]) + + # Act - Execute aggregation with $match stage pipeline = [{"$match": {"status": "active"}}] result = list(collection.aggregate(pipeline)) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 active users" names = {doc["name"] for doc in result} assert names == {"Alice", "Bob"}, "Expected Alice and Bob" @pytest.mark.aggregate -@pytest.mark.documents( - [ +def test_match_with_comparison_operator(collection): + """Test $match stage with comparison operators.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}, {"name": "Charlie", "age": 35}, - ] -) -def test_match_with_comparison_operator(collection): - """Test $match stage with comparison operators.""" - # Execute aggregation with $match using $gt + ]) + + # Act - Execute aggregation with $match using $gt pipeline = [{"$match": {"age": {"$gt": 25}}}] result = list(collection.aggregate(pipeline)) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 users with age > 25" names = {doc["name"] for doc in result} assert names == {"Alice", "Charlie"}, "Expected Alice and Charlie" @pytest.mark.aggregate -@pytest.mark.documents( - [ +def test_match_multiple_conditions(collection): + """Test $match stage with multiple filter conditions.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "age": 30, "city": "NYC"}, {"name": "Bob", "age": 25, "city": "SF"}, {"name": "Charlie", "age": 35, "city": "NYC"}, - ] -) -def test_match_multiple_conditions(collection): - """Test $match stage with multiple filter conditions.""" - # Execute aggregation with multiple conditions in $match + ]) + + # Act - Execute aggregation with multiple conditions in $match pipeline = [{"$match": {"city": "NYC", "age": {"$gte": 30}}}] result = list(collection.aggregate(pipeline)) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 users from NYC with age >= 30" names = {doc["name"] for doc in result} assert names == {"Alice", "Charlie"}, "Expected Alice and Charlie" @pytest.mark.aggregate -@pytest.mark.documents( - [ - {"name": "Alice", "status": "active"}, - {"name": "Bob", "status": "active"}, - ] -) +@pytest.mark.find def test_match_empty_result(collection): """Test $match stage that matches no documents.""" - # Execute aggregation with $match that matches nothing + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "status": "active"}, + {"name": "Bob", "status": "active"}, + ]) + + # Act - Execute aggregation with $match that matches nothing pipeline = [{"$match": {"status": "inactive"}}] result = list(collection.aggregate(pipeline)) - # Verify empty result + # Assert - Verify empty result assert result == [], "Expected empty result when no documents match" diff --git a/tests/find/test_basic_queries.py b/tests/find/test_basic_queries.py index 9a27d1d..3de9c70 100644 --- a/tests/find/test_basic_queries.py +++ b/tests/find/test_basic_queries.py @@ -11,19 +11,19 @@ @pytest.mark.find @pytest.mark.smoke -@pytest.mark.documents( - [ +def test_find_all_documents(collection): + """Test finding all documents in a collection.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "age": 30, "status": "active"}, {"name": "Bob", "age": 25, "status": "active"}, {"name": "Charlie", "age": 35, "status": "inactive"}, - ] -) -def test_find_all_documents(collection): - """Test finding all documents in a collection.""" - # Execute find operation + ]) + + # Act - Execute find operation result = list(collection.find()) - # Verify all documents are returned + # Assert - Verify all documents are returned assert len(result) == 3, "Expected to find 3 documents" # Verify document content @@ -33,19 +33,19 @@ def test_find_all_documents(collection): @pytest.mark.find @pytest.mark.smoke -@pytest.mark.documents( - [ +def test_find_with_filter(collection): + """Test find operation with a simple equality filter.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "age": 30, "status": "active"}, {"name": "Bob", "age": 25, "status": "active"}, {"name": "Charlie", "age": 35, "status": "inactive"}, - ] -) -def test_find_with_filter(collection): - """Test find operation with a simple equality filter.""" - # Execute find with filter + ]) + + # Act - Execute find with filter result = list(collection.find({"status": "active"})) - # Verify only active users are returned + # Assert - Verify only active users are returned assert len(result) == 2, "Expected to find 2 active users" # Verify all returned documents have status "active" @@ -58,79 +58,77 @@ def test_find_with_filter(collection): @pytest.mark.find -@pytest.mark.documents( - [ - {"name": "Alice", "age": 30}, - {"name": "Bob", "age": 25}, - ] -) def test_find_one(collection): """Test findOne operation returns a single document.""" - # Execute findOne + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + ]) + + # Act - Execute findOne result = collection.find_one({"name": "Alice"}) - # Verify document is returned + # Assert - Verify document is returned and content matches assert result is not None, "Expected to find a document" - - # Verify document content assert_document_match(result, {"name": "Alice", "age": 30}) @pytest.mark.find -@pytest.mark.documents( - [ - {"name": "Alice", "age": 30}, - ] -) def test_find_one_not_found(collection): """Test findOne returns None when no document matches.""" - # Execute findOne with non-matching filter + # Arrange - Insert test data + collection.insert_one({"name": "Alice", "age": 30}) + + # Act - Execute findOne with non-matching filter result = collection.find_one({"name": "NonExistent"}) - # Verify None is returned + # Assert - Verify None is returned assert result is None, "Expected None when no document matches" @pytest.mark.find def test_find_empty_collection(collection): """Test find on an empty collection returns empty result.""" - # Execute find on empty collection + # Arrange - Collection is already empty (no insertion needed) + + # Act - Execute find on empty collection result = list(collection.find()) - # Verify empty result + # Assert - Verify empty result assert result == [], "Expected empty result for empty collection" @pytest.mark.find -@pytest.mark.documents( - [ +def test_find_with_multiple_conditions(collection): + """Test find with multiple filter conditions (implicit AND).""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "age": 30, "city": "NYC"}, {"name": "Bob", "age": 25, "city": "SF"}, {"name": "Charlie", "age": 35, "city": "NYC"}, - ] -) -def test_find_with_multiple_conditions(collection): - """Test find with multiple filter conditions (implicit AND).""" - # Execute find with multiple conditions + ]) + + # Act - Execute find with multiple conditions result = list(collection.find({"city": "NYC", "age": 30})) - # Verify only matching document is returned + # Assert - Verify only matching document is returned assert len(result) == 1, "Expected to find 1 document" assert_document_match(result[0], {"name": "Alice", "age": 30, "city": "NYC"}) @pytest.mark.find -@pytest.mark.documents( - [ - {"name": "Alice", "profile": {"age": 30, "city": "NYC"}}, - {"name": "Bob", "profile": {"age": 25, "city": "SF"}}, - ] -) def test_find_nested_field(collection): """Test find with nested field query using dot notation.""" - # Execute find with nested field + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "profile": {"age": 30, "city": "NYC"}}, + {"name": "Bob", "profile": {"age": 25, "city": "SF"}}, + ]) + + # Act - Execute find with nested field result = list(collection.find({"profile.city": "NYC"})) - # Verify correct document is returned + # Assert - Verify correct document is returned assert len(result) == 1, "Expected to find 1 document" assert result[0]["name"] == "Alice", "Expected to find Alice" diff --git a/tests/find/test_projections.py b/tests/find/test_projections.py index c841f63..ab941bd 100644 --- a/tests/find/test_projections.py +++ b/tests/find/test_projections.py @@ -10,18 +10,18 @@ @pytest.mark.find -@pytest.mark.documents( - [ - {"name": "Alice", "age": 30, "email": "alice@example.com", "city": "NYC"}, - {"name": "Bob", "age": 25, "email": "bob@example.com", "city": "SF"}, - ] -) def test_find_with_field_inclusion(collection): """Test find with explicit field inclusion.""" - # Find with projection to include only name and age + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30, "email": "alice@example.com", "city": "NYC"}, + {"name": "Bob", "age": 25, "email": "bob@example.com", "city": "SF"}, + ]) + + # Act - Find with projection to include only name and age result = list(collection.find({}, {"name": 1, "age": 1})) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 documents" # Verify included fields exist @@ -36,18 +36,18 @@ def test_find_with_field_inclusion(collection): @pytest.mark.find -@pytest.mark.documents( - [ - {"name": "Alice", "age": 30, "email": "alice@example.com", "city": "NYC"}, - {"name": "Bob", "age": 25, "email": "bob@example.com", "city": "SF"}, - ] -) def test_find_with_field_exclusion(collection): """Test find with explicit field exclusion.""" - # Find with projection to exclude email and city + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30, "email": "alice@example.com", "city": "NYC"}, + {"name": "Bob", "age": 25, "email": "bob@example.com", "city": "SF"}, + ]) + + # Act - Find with projection to exclude email and city result = list(collection.find({}, {"email": 0, "city": 0})) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 documents" # Verify included fields exist @@ -62,17 +62,15 @@ def test_find_with_field_exclusion(collection): @pytest.mark.find -@pytest.mark.documents( - [ - {"name": "Alice", "age": 30, "email": "alice@example.com"}, - ] -) def test_find_exclude_id(collection): """Test find with _id exclusion.""" - # Find with projection to exclude _id + # Arrange - Insert test data + collection.insert_one({"name": "Alice", "age": 30, "email": "alice@example.com"}) + + # Act - Find with projection to exclude _id result = list(collection.find({}, {"_id": 0, "name": 1, "age": 1})) - # Verify results + # Assert - Verify results assert len(result) == 1, "Expected 1 document" # Verify _id is excluded @@ -82,17 +80,18 @@ def test_find_exclude_id(collection): @pytest.mark.find -@pytest.mark.documents( - [ - {"name": "Alice", "profile": {"age": 30, "city": "NYC", "email": "alice@example.com"}}, - ] -) def test_find_nested_field_projection(collection): """Test find with nested field projection.""" - # Find with projection for nested field + # Arrange - Insert test data + collection.insert_one({ + "name": "Alice", + "profile": {"age": 30, "city": "NYC", "email": "alice@example.com"} + }) + + # Act - Find with projection for nested field result = list(collection.find({}, {"name": 1, "profile.age": 1})) - # Verify results + # Assert - Verify results assert len(result) == 1, "Expected 1 document" assert_field_exists(result[0], "name") assert_field_exists(result[0], "profile.age") @@ -103,13 +102,16 @@ def test_find_nested_field_projection(collection): @pytest.mark.find -@pytest.mark.documents([{"name": "Alice", "age": 30}]) +@pytest.mark.smoke def test_find_empty_projection(collection): """Test find with empty projection returns all fields.""" - # Find with empty projection + # Arrange - Insert test data + collection.insert_one({"name": "Alice", "age": 30}) + + # Act - Find with empty projection result = collection.find_one({}, {}) - # Verify all fields are present + # Assert - Verify all fields are present assert_field_exists(result, "_id") assert_field_exists(result, "name") assert_field_exists(result, "age") diff --git a/tests/find/test_query_operators.py b/tests/find/test_query_operators.py index 780f3d4..9cd6bd7 100644 --- a/tests/find/test_query_operators.py +++ b/tests/find/test_query_operators.py @@ -8,151 +8,152 @@ @pytest.mark.find -@pytest.mark.documents( - [ +def test_find_gt_operator(collection): + """Test find with $gt (greater than) operator.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}, {"name": "Charlie", "age": 35}, - ] -) -def test_find_gt_operator(collection): - """Test find with $gt (greater than) operator.""" - # Find documents where age > 25 + ]) + + # Act - Find documents where age > 25 result = list(collection.find({"age": {"$gt": 25}})) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 documents with age > 25" names = {doc["name"] for doc in result} assert names == {"Alice", "Charlie"}, "Expected Alice and Charlie" @pytest.mark.find -@pytest.mark.documents( - [ +def test_find_gte_operator(collection): + """Test find with $gte (greater than or equal) operator.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}, {"name": "Charlie", "age": 35}, - ] -) -def test_find_gte_operator(collection): - """Test find with $gte (greater than or equal) operator.""" - # Find documents where age >= 30 + ]) + + # Act - Find documents where age >= 30 result = list(collection.find({"age": {"$gte": 30}})) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 documents with age >= 30" names = {doc["name"] for doc in result} assert names == {"Alice", "Charlie"}, "Expected Alice and Charlie" @pytest.mark.find -@pytest.mark.documents( - [ +def test_find_lt_operator(collection): + """Test find with $lt (less than) operator.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}, {"name": "Charlie", "age": 35}, - ] -) -def test_find_lt_operator(collection): - """Test find with $lt (less than) operator.""" - # Find documents where age < 30 + ]) + + # Act - Find documents where age < 30 result = list(collection.find({"age": {"$lt": 30}})) - # Verify results + # Assert - Verify results assert len(result) == 1, "Expected 1 document with age < 30" assert result[0]["name"] == "Bob", "Expected Bob" @pytest.mark.find -@pytest.mark.documents( - [ +def test_find_lte_operator(collection): + """Test find with $lte (less than or equal) operator.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}, {"name": "Charlie", "age": 35}, - ] -) -def test_find_lte_operator(collection): - """Test find with $lte (less than or equal) operator.""" - # Find documents where age <= 30 + ]) + + # Act - Find documents where age <= 30 result = list(collection.find({"age": {"$lte": 30}})) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 documents with age <= 30" names = {doc["name"] for doc in result} assert names == {"Alice", "Bob"}, "Expected Alice and Bob" @pytest.mark.find -@pytest.mark.documents( - [ +def test_find_ne_operator(collection): + """Test find with $ne (not equal) operator.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}, {"name": "Charlie", "age": 35}, - ] -) -def test_find_ne_operator(collection): - """Test find with $ne (not equal) operator.""" - # Find documents where age != 30 + ]) + + # Act - Find documents where age != 30 result = list(collection.find({"age": {"$ne": 30}})) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 documents with age != 30" names = {doc["name"] for doc in result} assert names == {"Bob", "Charlie"}, "Expected Bob and Charlie" @pytest.mark.find -@pytest.mark.documents( - [ +def test_find_in_operator(collection): + """Test find with $in operator.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "status": "active"}, {"name": "Bob", "status": "inactive"}, {"name": "Charlie", "status": "pending"}, {"name": "David", "status": "active"}, - ] -) -def test_find_in_operator(collection): - """Test find with $in operator.""" - # Find documents where status is in ["active", "pending"] + ]) + + # Act - Find documents where status is in ["active", "pending"] result = list(collection.find({"status": {"$in": ["active", "pending"]}})) - # Verify results + # Assert - Verify results assert len(result) == 3, "Expected 3 documents with status in [active, pending]" names = {doc["name"] for doc in result} assert names == {"Alice", "Charlie", "David"}, "Expected Alice, Charlie, and David" @pytest.mark.find -@pytest.mark.documents( - [ +def test_find_nin_operator(collection): + """Test find with $nin (not in) operator.""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "status": "active"}, {"name": "Bob", "status": "inactive"}, {"name": "Charlie", "status": "pending"}, - ] -) -def test_find_nin_operator(collection): - """Test find with $nin (not in) operator.""" - # Find documents where status is not in ["active", "pending"] + ]) + + # Act - Find documents where status is not in ["active", "pending"] result = list(collection.find({"status": {"$nin": ["active", "pending"]}})) - # Verify results + # Assert - Verify results assert len(result) == 1, "Expected 1 document with status not in [active, pending]" assert result[0]["name"] == "Bob", "Expected Bob" @pytest.mark.find -@pytest.mark.documents( - [ +@pytest.mark.smoke +def test_find_range_query(collection): + """Test find with range query (combining $gte and $lte).""" + # Arrange - Insert test data + collection.insert_many([ {"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}, {"name": "Charlie", "age": 35}, - ] -) -def test_find_range_query(collection): - """Test find with range query (combining $gte and $lte).""" - # Find documents where 25 <= age <= 30 + ]) + + # Act - Find documents where 25 <= age <= 30 result = list(collection.find({"age": {"$gte": 25, "$lte": 30}})) - # Verify results + # Assert - Verify results assert len(result) == 2, "Expected 2 documents with age between 25 and 30" names = {doc["name"] for doc in result} assert names == {"Alice", "Bob"}, "Expected Alice and Bob" From eb5dbbf61bc10a865d3b96dde2158cb3dfb6c09d Mon Sep 17 00:00:00 2001 From: Nitin Ahuja Date: Wed, 4 Feb 2026 11:07:01 -0800 Subject: [PATCH 7/9] docs: Update CLI examples and standardize on .test-results/ directory --- CONTRIBUTING.md | 10 +++++++--- README.md | 34 ++++++++++++++++++++-------------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d8e5637..b0ae8da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -210,10 +210,14 @@ pytest -m "your_new_tag" ### Test Against Multiple Engines +To test against multiple engines, run pytest separately for each engine: + ```bash -# Test against both DocumentDB and MongoDB -pytest --engine documentdb=mongodb://localhost:27017 \ - --engine mongodb=mongodb://mongo:27017 +# Test against DocumentDB +pytest --connection-string mongodb://localhost:27017 --engine-name documentdb + +# Test against MongoDB +pytest --connection-string mongodb://mongo:27017 --engine-name mongodb ``` ## Submitting Changes diff --git a/README.md b/README.md index 0476a5a..6d03189 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,10 @@ pip install -r requirements.txt pytest # Run against specific engine -pytest --engine documentdb=mongodb://localhost:27017 +pytest --connection-string mongodb://localhost:27017 --engine-name documentdb -# Run against multiple engines -pytest --engine documentdb=mongodb://localhost:27017 \ - --engine mongodb=mongodb://mongo:27017 +# Run with just connection string (engine-name defaults to "default") +pytest --connection-string mongodb://localhost:27017 ``` #### Filter by Tags @@ -75,7 +74,7 @@ pytest -n 4 pytest -n auto # Combine with other options -pytest -n auto -m smoke --engine documentdb=mongodb://localhost:27017 +pytest -n auto -m smoke --connection-string mongodb://localhost:27017 --engine-name documentdb ``` **Performance Benefits:** @@ -96,10 +95,11 @@ pytest -n auto -m smoke --engine documentdb=mongodb://localhost:27017 **Example with full options:** ```bash pytest -n 4 \ - --engine documentdb=mongodb://localhost:27017 \ + --connection-string mongodb://localhost:27017 \ + --engine-name documentdb \ -m "find or aggregate" \ -v \ - --json-report --json-report-file=results.json + --json-report --json-report-file=.test-results/report.json ``` #### Output Formats @@ -138,19 +138,21 @@ Run tests with the pre-built image: # Run all tests docker run --network host \ ghcr.io/documentdb/functional-tests:latest \ - --engine documentdb="mongodb://user:pass@host:port/?tls=true&tlsAllowInvalidCertificates=true" + --connection-string "mongodb://user:pass@host:port/?tls=true&tlsAllowInvalidCertificates=true" \ + --engine-name documentdb # Run specific tags docker run --network host \ ghcr.io/documentdb/functional-tests:latest \ -m smoke \ - --engine documentdb="mongodb://localhost:10260/?tls=true" + --connection-string "mongodb://localhost:10260/?tls=true" # Run with parallel execution docker run --network host \ ghcr.io/documentdb/functional-tests:latest \ -n 4 \ - --engine documentdb="mongodb://localhost:27017" + --connection-string mongodb://localhost:27017 \ + --engine-name documentdb ``` ### Option 2: Build Locally @@ -167,16 +169,19 @@ docker build -t documentdb/functional-tests . # Run against DocumentDB docker run --network host \ documentdb/functional-tests \ - --engine documentdb=mongodb://localhost:27017 + --connection-string mongodb://localhost:27017 \ + --engine-name documentdb # Run specific tags docker run documentdb/functional-tests \ - --engine documentdb=mongodb://cluster.docdb.amazonaws.com:27017 \ + --connection-string mongodb://cluster.docdb.amazonaws.com:27017 \ + --engine-name documentdb \ -m smoke # Run with parallel execution docker run documentdb/functional-tests \ - --engine documentdb=mongodb://localhost:27017 \ + --connection-string mongodb://localhost:27017 \ + --engine-name documentdb \ -n 4 ``` @@ -345,7 +350,8 @@ Tests are automatically categorized into: ```bash # Run tests with JSON report -pytest --engine documentdb=mongodb://localhost:27017 \ +pytest --connection-string mongodb://localhost:27017 \ + --engine-name documentdb \ --json-report --json-report-file=.test-results/report.json # Analyze results From a6d6e82361d48d82fb148d9d59f4110c125002d0 Mon Sep 17 00:00:00 2001 From: Nitin Ahuja Date: Wed, 4 Feb 2026 11:09:08 -0800 Subject: [PATCH 8/9] test: Add capped collection test to validate code 115 detection --- tests/collection/__init__.py | 1 + tests/collection/test_capped_collections.py | 43 +++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 tests/collection/__init__.py create mode 100644 tests/collection/test_capped_collections.py diff --git a/tests/collection/__init__.py b/tests/collection/__init__.py new file mode 100644 index 0000000..b73cf50 --- /dev/null +++ b/tests/collection/__init__.py @@ -0,0 +1 @@ +"""Collection management tests.""" diff --git a/tests/collection/test_capped_collections.py b/tests/collection/test_capped_collections.py new file mode 100644 index 0000000..0f1acd4 --- /dev/null +++ b/tests/collection/test_capped_collections.py @@ -0,0 +1,43 @@ +""" +Tests for capped collection operations. + +NOTE: This test was added by design to demonstrate how the Result Analyzer +automatically detects and categorizes unsupported features (error code 115). +When run against DocumentDB, this test will fail with error code 115, which +the analyzer will correctly categorize as UNSUPPORTED rather than FAIL. +""" + +import pytest +from pymongo.errors import OperationFailure + + +@pytest.mark.collection_mgmt +def test_create_capped_collection(database_client): + """ + Test creating a capped collection. + + Capped collections are fixed-size collections that maintain insertion order. + This feature is not supported in DocumentDB and should return error code 115. + + Expected behavior: + - Creates a capped collection successfully + """ + collection_name = "capped_test_collection" + + try: + # Attempt to create capped collection + database_client.create_collection( + collection_name, + capped=True, + size=100000 + ) + + # Verify it's actually capped + collection_info = database_client[collection_name].options() + assert collection_info.get("capped") is True, "Collection should be capped" + + # Cleanup + database_client.drop_collection(collection_name) + + except OperationFailure as e: + raise From 204994ce754ea70920c482840ea63dbd07eddeca Mon Sep 17 00:00:00 2001 From: Nitin Ahuja Date: Wed, 4 Feb 2026 11:09:14 -0800 Subject: [PATCH 9/9] chore: Add project governance files --- CODEOWNERS | 4 ++++ MAINTAINERS.md | 7 +++++++ 2 files changed, 11 insertions(+) create mode 100644 CODEOWNERS create mode 100644 MAINTAINERS.md diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..368506a --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,4 @@ +# Code owners for DocumentDB Functional Tests +# These owners will be automatically requested for review when PRs are opened + +* @documentdb/functional-tests-maintainers diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 0000000..7795279 --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,7 @@ +# Maintainers + +## Current Maintainers + +| Member | Organization | +| --- | --- | +| Nitin Ahuja | Amazon |