diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ca0dc97 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,53 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg-info/ +dist/ +build/ +.pytest_cache/ +.coverage +htmlcov/ + +# Virtual environments +venv/ +env/ +ENV/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Git +.git/ +.gitignore +.gitattributes + +# Documentation +docs/ +*.md +!README.md + +# Test results +.test-results/ +test-results/ +*.log + +# Development files +requirements-dev.txt +.flake8 +.mypy.ini + +# CI/CD +.github/ +.gitlab-ci.yml +.circleci/ + +# OS +.DS_Store +Thumbs.db diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..95b9a60 --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +max-line-length = 100 +extend-ignore = E203, W503 +exclude = + .git, + __pycache__, + .venv, + venv, + build, + dist, + .eggs, + *.egg-info diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 0000000..135473e --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,65 @@ +name: Build and Push Docker Image + +on: + push: + branches: + - main + tags: + - 'v*.*.*' + pull_request: + branches: + - main + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,prefix=sha- + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index b7faf40..f2a869e 100644 --- a/.gitignore +++ b/.gitignore @@ -205,3 +205,6 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ + +# Test results directory +.test-results/ diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..368506a --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,4 @@ +# Code owners for DocumentDB Functional Tests +# These owners will be automatically requested for review when PRs are opened + +* @documentdb/functional-tests-maintainers diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..b0ae8da --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,295 @@ +# Contributing to DocumentDB Functional Tests + +Thank you for your interest in contributing to the DocumentDB Functional Tests! This document provides guidelines for contributing to the project. + +## Getting Started + +### Prerequisites + +- Python 3.9 or higher +- Git +- Access to a DocumentDB or MongoDB instance for testing + +### Development Setup + +1. Fork the repository on GitHub +2. Clone your fork locally: + ```bash + git clone https://github.com/YOUR_USERNAME/functional-tests.git + cd functional-tests + ``` + +3. Install development dependencies: + ```bash + pip install -r requirements-dev.txt + ``` + +4. Create a branch for your changes: + ```bash + git checkout -b feature/your-feature-name + ``` + +## Writing Tests + +### Test File Organization + +- Place tests in the appropriate directory based on the operation being tested +- Use descriptive file names: `test_.py` +- Group related tests in the same file + +### Test Structure + +Every test should follow this structure: + +```python +import pytest +from tests.common.assertions import assert_document_match + +@pytest.mark. # Required: e.g., find, insert, aggregate +@pytest.mark. # Optional: e.g., rbac, decimal128 +@pytest.mark.documents([ # Optional: test data + {"name": "Alice", "age": 30} +]) +def test_descriptive_name(collection): + """ + Clear description of what this test validates. + + This should explain: + - What feature/behavior is being tested + - Expected outcome + - Any special conditions or edge cases + """ + # Setup (if needed beyond @pytest.mark.documents) + + # Execute the operation being tested + result = collection.find({"name": "Alice"}) + + # Assert expected behavior + assert len(list(result)) == 1 +``` + +### Naming Conventions + +- **Test functions**: `test_` + - Good: `test_find_with_gt_operator` + - Bad: `test_1`, `test_query` + +- **Test files**: `test_.py` + - Good: `test_query_operators.py` + - Bad: `tests.py`, `test.py` + +### Required Tags + +Every test MUST have at least one horizontal tag (operation): +- `@pytest.mark.find` +- `@pytest.mark.insert` +- `@pytest.mark.update` +- `@pytest.mark.delete` +- `@pytest.mark.aggregate` +- `@pytest.mark.index` +- `@pytest.mark.admin` +- `@pytest.mark.collection_mgmt` + +### Optional Tags + +Add vertical tags for cross-cutting features: +- `@pytest.mark.rbac` - Role-based access control +- `@pytest.mark.decimal128` - Decimal128 data type +- `@pytest.mark.collation` - Collation/sorting +- `@pytest.mark.transactions` - Transactions +- `@pytest.mark.geospatial` - Geospatial queries +- `@pytest.mark.text_search` - Text search +- `@pytest.mark.validation` - Schema validation +- `@pytest.mark.ttl` - Time-to-live indexes + +Add special tags when appropriate: +- `@pytest.mark.smoke` - Quick smoke tests +- `@pytest.mark.slow` - Tests taking > 5 seconds + +### Using Fixtures + +The framework provides three main fixtures: + +1. **engine_client**: Raw MongoDB client + ```python + def test_with_client(engine_client): + db = engine_client.test_db + collection = db.test_collection + # ... test code + ``` + +2. **database_client**: Database with automatic cleanup + ```python + def test_with_database(database_client): + collection = database_client.my_collection + # ... test code + # Database automatically dropped after test + ``` + +3. **collection**: Collection with automatic data setup and cleanup + ```python + @pytest.mark.documents([{"name": "Alice"}]) + def test_with_collection(collection): + # Documents already inserted + result = collection.find_one({"name": "Alice"}) + # ... assertions + # Collection automatically dropped after test + ``` + +### Custom Assertions + +Use the provided assertion helpers for common scenarios: + +```python +from tests.common.assertions import ( + assert_document_match, + assert_documents_match, + assert_field_exists, + assert_field_not_exists, + assert_count +) + +# Good: Use custom assertions +assert_document_match(actual, expected, ignore_id=True) +assert_count(collection, {"status": "active"}, 5) + +# Avoid: Manual comparison that's verbose +actual_doc = {k: v for k, v in actual.items() if k != "_id"} +expected_doc = {k: v for k, v in expected.items() if k != "_id"} +assert actual_doc == expected_doc +``` + +## Code Quality + +### Before Submitting + +Run these commands to ensure code quality: + +```bash +# Format code +black . + +# Sort imports +isort . + +# Run linter +flake8 + +# Type checking (optional but recommended) +mypy . + +# Run tests +pytest +``` + +### Code Style + +- Follow PEP 8 style guidelines +- Use meaningful variable names +- Add docstrings to test functions +- Keep test functions focused on a single behavior +- Avoid complex logic in tests + +## Testing Your Changes + +### Run Tests Locally + +```bash +# Run all tests +pytest + +# Run specific test file +pytest tests/find/test_basic_queries.py + +# Run specific test +pytest tests/find/test_basic_queries.py::test_find_all_documents + +# Run with your changes only +pytest -m "your_new_tag" +``` + +### Test Against Multiple Engines + +To test against multiple engines, run pytest separately for each engine: + +```bash +# Test against DocumentDB +pytest --connection-string mongodb://localhost:27017 --engine-name documentdb + +# Test against MongoDB +pytest --connection-string mongodb://mongo:27017 --engine-name mongodb +``` + +## Submitting Changes + +### Pull Request Process + +1. Ensure your code follows the style guidelines +2. Add tests for new functionality +3. Update documentation if needed +4. Commit with clear, descriptive messages: + ```bash + git commit -m "Add tests for $group stage with $avg operator" + ``` + +5. Push to your fork: + ```bash + git push origin feature/your-feature-name + ``` + +6. Create a Pull Request on GitHub + +### Pull Request Guidelines + +Your PR should: +- Have a clear title describing the change +- Include a description explaining: + - What the change does + - Why it's needed + - How to test it +- Reference any related issues +- Pass all CI checks + +### Commit Message Guidelines + +- Use present tense: "Add test" not "Added test" +- Be descriptive but concise +- Reference issues: "Fix #123: Add validation tests" + +## Adding New Test Categories + +If you're adding tests for a new feature area: + +1. Create a new directory under `tests/`: + ```bash + mkdir tests/update + ``` + +2. Add `__init__.py`: + ```python + """Update operation tests.""" + ``` + +3. Create test files following naming conventions + +4. Add appropriate markers to `pytest.ini`: + ```ini + markers = + update: Update operation tests + ``` + +5. Update documentation in README.md + +## Questions? + +- Open an issue for questions about contributing +- Check existing tests for examples +- Review the RFC document for framework design + +## Code of Conduct + +This project follows the Contributor Covenant Code of Conduct. By participating, you are expected to uphold this code. + +## License + +By contributing, you agree that your contributions will be licensed under the MIT License. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d1cc5ee --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +# Multi-stage build for DocumentDB Functional Tests + +# Stage 1: Build stage +FROM python:3.11-slim AS builder + +WORKDIR /build + +# Copy requirements +COPY requirements.txt . + +# Install dependencies +RUN pip install --no-cache-dir --user -r requirements.txt + +# Stage 2: Runtime stage +FROM python:3.11-slim + +# Set working directory +WORKDIR /app + +# Copy installed packages from builder +COPY --from=builder /root/.local /root/.local + +# Copy application code +COPY tests/ tests/ +COPY result_analyzer/ result_analyzer/ +COPY conftest.py . +COPY pytest.ini . +COPY setup.py . + +# Ensure scripts are in PATH +ENV PATH=/root/.local/bin:$PATH + +# Create directory for test results +RUN mkdir -p .test-results + +# Set Python to run unbuffered (so logs appear immediately) +ENV PYTHONUNBUFFERED=1 + +# Default command: run all tests +# Users can override with command line arguments +ENTRYPOINT ["pytest"] +CMD ["--help"] diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 0000000..7795279 --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,7 @@ +# Maintainers + +## Current Maintainers + +| Member | Organization | +| --- | --- | +| Nitin Ahuja | Amazon | diff --git a/README.md b/README.md index dcdb725..6d03189 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,413 @@ -# functional-tests -End-to-end functional testing framework for DocumentDB using pytest +# DocumentDB Functional Tests + +End-to-end functional testing framework for DocumentDB using pytest. This framework validates DocumentDB functionality against specifications through comprehensive test suites. + +## Overview + +This testing framework provides: + +- **Specification-based Testing**: Tests define explicit expected behavior for DocumentDB features +- **Multi-Engine Support**: Run the same tests against DocumentDB, MongoDB, and other compatible engines +- **Parallel Execution**: Fast test execution using pytest-xdist +- **Tag-Based Organization**: Flexible test filtering using pytest markers +- **Result Analysis**: Automatic categorization and reporting of test results + +## Quick Start + +### Prerequisites + +- Python 3.9 or higher +- Access to a DocumentDB or MongoDB instance +- pip package manager + +### Installation + +```bash +# Clone the repository +git clone https://github.com/documentdb/functional-tests.git +cd functional-tests + +# Install dependencies +pip install -r requirements.txt +``` + +### Running Tests + +#### Basic Usage + +```bash +# Run all tests against default localhost +pytest + +# Run against specific engine +pytest --connection-string mongodb://localhost:27017 --engine-name documentdb + +# Run with just connection string (engine-name defaults to "default") +pytest --connection-string mongodb://localhost:27017 +``` + +#### Filter by Tags + +```bash +# Run only find operation tests +pytest -m find + +# Run smoke tests +pytest -m smoke + +# Run find tests with RBAC +pytest -m "find and rbac" + +# Exclude slow tests +pytest -m "not slow" +``` + +#### Parallel Execution + +The framework supports parallel test execution using pytest-xdist, which can significantly reduce test execution time. + +```bash +# Run with 4 parallel processes +pytest -n 4 + +# Auto-detect number of CPUs (recommended) +pytest -n auto + +# Combine with other options +pytest -n auto -m smoke --connection-string mongodb://localhost:27017 --engine-name documentdb +``` + +**Performance Benefits:** +- Significantly faster test execution with multiple workers +- Scales with number of available CPU cores +- Particularly effective for large test suites + +**Best Practices:** +- Use `-n auto` to automatically detect optimal worker count +- Parallel execution works best with 4+ workers +- Each worker runs tests in isolation (separate database/collection) +- Safe for tests with automatic cleanup (all framework tests are safe) + +**When to Use:** +- Large test suites +- Local development for quick validation + +**Example with full options:** +```bash +pytest -n 4 \ + --connection-string mongodb://localhost:27017 \ + --engine-name documentdb \ + -m "find or aggregate" \ + -v \ + --json-report --json-report-file=.test-results/report.json +``` + +#### Output Formats + +```bash +# Generate JSON report +pytest --json-report --json-report-file=results.json + +# Generate JUnit XML +pytest --junitxml=results.xml + +# Verbose output +pytest -v + +# Show local variables on failure +pytest -l +``` + +## Docker Usage + +### Option 1: Use Pre-built Image (Recommended) + +Pull the latest image from GitHub Container Registry: + +```bash +# Pull latest version +docker pull ghcr.io/documentdb/functional-tests:latest + +# Or pull a specific version +docker pull ghcr.io/documentdb/functional-tests:v1.0.0 +``` + +Run tests with the pre-built image: + +```bash +# Run all tests +docker run --network host \ + ghcr.io/documentdb/functional-tests:latest \ + --connection-string "mongodb://user:pass@host:port/?tls=true&tlsAllowInvalidCertificates=true" \ + --engine-name documentdb + +# Run specific tags +docker run --network host \ + ghcr.io/documentdb/functional-tests:latest \ + -m smoke \ + --connection-string "mongodb://localhost:10260/?tls=true" + +# Run with parallel execution +docker run --network host \ + ghcr.io/documentdb/functional-tests:latest \ + -n 4 \ + --connection-string mongodb://localhost:27017 \ + --engine-name documentdb +``` + +### Option 2: Build Locally + +If you need to build from source: + +```bash +docker build -t documentdb/functional-tests . +``` + +### Run Tests in Container + +```bash +# Run against DocumentDB +docker run --network host \ + documentdb/functional-tests \ + --connection-string mongodb://localhost:27017 \ + --engine-name documentdb + +# Run specific tags +docker run documentdb/functional-tests \ + --connection-string mongodb://cluster.docdb.amazonaws.com:27017 \ + --engine-name documentdb \ + -m smoke + +# Run with parallel execution +docker run documentdb/functional-tests \ + --connection-string mongodb://localhost:27017 \ + --engine-name documentdb \ + -n 4 +``` + +## Test Organization + +Tests are organized by API operations with cross-cutting feature tags: + +``` +tests/ +├── find/ # Find operation tests +│ ├── test_basic_queries.py +│ ├── test_query_operators.py +│ └── test_projections.py +├── aggregate/ # Aggregation tests +│ ├── test_match_stage.py +│ └── test_group_stage.py +├── insert/ # Insert operation tests +│ └── test_insert_operations.py +└── common/ # Shared utilities + └── assertions.py +``` + +## Test Tags + +### Horizontal Tags (API Operations) +- `find`: Find operation tests +- `insert`: Insert operation tests +- `update`: Update operation tests +- `delete`: Delete operation tests +- `aggregate`: Aggregation pipeline tests +- `index`: Index management tests +- `admin`: Administrative command tests +- `collection_mgmt`: Collection management tests + +### Vertical Tags (Cross-cutting Features) +- `rbac`: Role-based access control tests +- `decimal128`: Decimal128 data type tests +- `collation`: Collation and sorting tests +- `transactions`: Transaction tests +- `geospatial`: Geospatial query tests +- `text_search`: Text search tests +- `validation`: Schema validation tests +- `ttl`: Time-to-live index tests + +### Special Tags +- `smoke`: Quick smoke tests for feature detection +- `slow`: Tests that take longer to execute + +## Writing Tests + +### Basic Test Structure + +```python +import pytest +from tests.common.assertions import assert_document_match + +@pytest.mark.find # Required: operation tag +@pytest.mark.smoke # Optional: additional tags +@pytest.mark.documents([ # Optional: test data + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25} +]) +def test_find_with_filter(collection): + """Test description.""" + # Execute operation + result = list(collection.find({"age": {"$gt": 25}})) + + # Verify results + assert len(result) == 1 + assert_document_match(result[0], {"name": "Alice", "age": 30}) +``` + +### Test Fixtures + +- `engine_client`: MongoDB client for the engine +- `database_client`: Database with automatic cleanup +- `collection`: Collection with automatic test data setup and cleanup + +### Custom Assertions + +```python +from tests.common.assertions import ( + assert_document_match, + assert_documents_match, + assert_field_exists, + assert_field_not_exists, + assert_count +) + +# Compare documents ignoring _id +assert_document_match(actual, expected, ignore_id=True) + +# Compare lists of documents +assert_documents_match(actual_list, expected_list, ignore_order=True) + +# Check field existence +assert_field_exists(document, "user.name") +assert_field_not_exists(document, "password") + +# Count documents matching filter +assert_count(collection, {"status": "active"}, 5) +``` + +## Result Analysis + +The framework includes a command-line tool to analyze test results and generate detailed reports categorized by feature tags. + +### Installation + +```bash +# Install the package to get the CLI tool +pip install -e . +``` + +### CLI Tool Usage + +```bash +# Analyze default report location +docdb-analyze + +# Analyze specific report +docdb-analyze --input custom-results.json + +# Generate text report +docdb-analyze --output report.txt --format text + +# Generate JSON analysis +docdb-analyze --output analysis.json --format json + +# Quiet mode (only write to file, no console output) +docdb-analyze --output report.txt --quiet + +# Get help +docdb-analyze --help +``` + +### Programmatic Usage + +You can also use the result analyzer as a Python library: + +```python +from result_analyzer import analyze_results, generate_report, print_summary + +# Analyze JSON report +analysis = analyze_results(".test-results/report.json") + +# Print summary to console +print_summary(analysis) + +# Generate text report +generate_report(analysis, "report.txt", format="text") + +# Generate JSON report +generate_report(analysis, "report.json", format="json") +``` + +### Failure Categories + +Tests are automatically categorized into: +- **PASS**: Test succeeded, behavior matches specification +- **FAIL**: Test failed, feature exists but behaves incorrectly +- **UNSUPPORTED**: Feature not implemented +- **INFRA_ERROR**: Infrastructure issue (connection, timeout, etc.) + +### Example Workflow + +```bash +# Run tests with JSON report +pytest --connection-string mongodb://localhost:27017 \ + --engine-name documentdb \ + --json-report --json-report-file=.test-results/report.json + +# Analyze results +docdb-analyze + +# Generate detailed text report +docdb-analyze --output detailed-report.txt --format text + +# Generate JSON for further processing +docdb-analyze --output analysis.json --format json +``` + +## Development + +### Install Development Dependencies + +```bash +pip install -r requirements-dev.txt +``` + +### Code Quality + +```bash +# Format code +black . + +# Sort imports +isort . + +# Lint +flake8 + +# Type checking +mypy . +``` + +### Run Tests with Coverage + +```bash +pytest --cov=. --cov-report=html +``` + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Write tests following the test structure guidelines +4. Ensure tests pass locally +5. Submit a pull request + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. + +## Support + +For issues and questions: +- GitHub Issues: https://github.com/documentdb/functional-tests/issues +- Documentation: https://github.com/documentdb/functional-tests/docs diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..a9b17e4 --- /dev/null +++ b/conftest.py @@ -0,0 +1,172 @@ +""" +Global pytest fixtures for functional testing framework. + +This module provides fixtures for: +- Engine parametrization +- Database connection management +- Test isolation +""" + + +import hashlib +import pytest +from pymongo import MongoClient + + +def pytest_addoption(parser): + """Add custom command-line options for pytest.""" + parser.addoption( + "--connection-string", + action="store", + default=None, + help="Database connection string. " + "Example: --connection-string mongodb://localhost:27017", + ) + parser.addoption( + "--engine-name", + action="store", + default="default", + help="Optional engine identifier for metadata. " + "Example: --engine-name documentdb", + ) + + +def pytest_configure(config): + """Configure pytest with custom settings.""" + # Get connection string and engine name + connection_string = config.getoption("--connection-string") + engine_name = config.getoption("--engine-name") + + # Store in config for access by fixtures + config.connection_string = connection_string + config.engine_name = engine_name + + # If no connection string specified, default to localhost + if not connection_string: + config.connection_string = "mongodb://localhost:27017" + if engine_name == "default": + # If using default connection and no engine name specified, set to "default" + config.engine_name = "default" + + +@pytest.fixture(scope="function") +def engine_client(request): + """ + Create a MongoDB client for the configured engine. + + Args: + request: pytest request object + + Yields: + MongoClient: Connected MongoDB client + + Raises: + ConnectionError: If unable to connect to the database + """ + connection_string = request.config.connection_string + engine_name = request.config.engine_name + + client = MongoClient(connection_string) + + # Verify connection + try: + client.admin.command("ping") + except Exception as e: + # Close the client before raising + client.close() + # Raise ConnectionError so analyzer categorizes as INFRA_ERROR + raise ConnectionError( + f"Cannot connect to {engine_name} at {connection_string}: {e}" + ) from e + + yield client + + # Cleanup: close connection + client.close() + + +@pytest.fixture(scope="function") +def database_client(engine_client, request): + """ + Provide a database client with automatic cleanup. + + Creates a test database with a collision-free name for parallel execution. + The name includes worker ID, hash, and abbreviated test name. + Automatically drops the database after the test completes. + + Args: + engine_client: MongoDB client from engine_client fixture + request: pytest request object + + Yields: + Database: MongoDB database object + """ + # Get worker ID for parallel execution (e.g., 'gw0', 'gw1', or 'main' for single process) + worker_id = getattr(request.config, 'workerinput', {}).get('workerid', 'main') + + # Get full test identifier (includes file path and test name) + full_test_id = request.node.nodeid + + # Create a short hash for uniqueness (first 8 chars of SHA256) + name_hash = hashlib.sha256(full_test_id.encode()).hexdigest()[:8] + + # Get abbreviated test name for readability (sanitize and truncate) + test_name = request.node.name.replace("[", "_").replace("]", "_") + abbreviated = test_name[:20] + + # Combine: test_{worker}_{hash}_{abbreviated} + # Example: test_gw0_a1b2c3d4_find_all_documents + db_name = f"test_{worker_id}_{name_hash}_{abbreviated}"[:63] # MongoDB limit + + db = engine_client[db_name] + + yield db + + # Cleanup: drop test database + try: + engine_client.drop_database(db_name) + except Exception: + pass # Best effort cleanup + + +@pytest.fixture(scope="function") +def collection(database_client, request): + """ + Provide an empty collection with automatic cleanup. + + Creates a collection with a collision-free name for parallel execution. + Tests should directly insert any required test data. + + Args: + database_client: Database from database_client fixture + request: pytest request object + + Yields: + Collection: Empty MongoDB collection object + """ + # Get worker ID for parallel execution (e.g., 'gw0', 'gw1', or 'main') + worker_id = getattr(request.config, 'workerinput', {}).get('workerid', 'main') + + # Get full test identifier + full_test_id = request.node.nodeid + + # Create a short hash for uniqueness (first 8 chars of SHA256) + name_hash = hashlib.sha256(full_test_id.encode()).hexdigest()[:8] + + # Get abbreviated test name for readability (sanitize and truncate) + test_name = request.node.name.replace("[", "_").replace("]", "_") + abbreviated = test_name[:25] + + # Combine: coll_{worker}_{hash}_{abbreviated} + # Example: coll_gw0_a1b2c3d4_find_all_documents + collection_name = f"coll_{worker_id}_{name_hash}_{abbreviated}"[:100] # Collection name limit + + coll = database_client[collection_name] + + yield coll + + # Cleanup: drop collection + try: + coll.drop() + except Exception: + pass # Best effort cleanup diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..905e866 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,36 @@ +[tool.black] +line-length = 100 +target-version = ['py39', 'py310', 'py311'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.venv + | venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +line_length = 100 +skip_gitignore = true + +[tool.mypy] +python_version = "3.9" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +ignore_missing_imports = true +exclude = [ + 'venv', + '.venv', + 'build', + 'dist', +] + +[tool.pytest.ini_options] +# Already in pytest.ini, keeping this for reference diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..f9c263e --- /dev/null +++ b/pytest.ini @@ -0,0 +1,50 @@ +[pytest] +# Test discovery +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* + +# Output options +addopts = + -v + --strict-markers + --tb=short + --color=yes + +# Markers for test categorization +markers = + # Horizontal tags (API Operations) + find: Find operation tests + insert: Insert operation tests + update: Update operation tests + delete: Delete operation tests + aggregate: Aggregation pipeline tests + index: Index management tests + admin: Administrative command tests + collection_mgmt: Collection management tests + + # Vertical tags (Cross-cutting Features) + rbac: Role-based access control tests + decimal128: Decimal128 data type tests + collation: Collation and sorting tests + transactions: Transaction tests + geospatial: Geospatial query tests + text_search: Text search tests + validation: Schema validation tests + ttl: Time-to-live index tests + + # Special markers + smoke: Quick smoke tests for feature detection + slow: Tests that take longer to execute + +# Timeout for tests (seconds) +timeout = 300 + +# Parallel execution settings +# Use with: pytest -n auto +# Or: pytest -n 4 (for 4 processes) + +# JSON report options +# These are command-line options, not config file options +# Use with: pytest --json-report --json-report-file=.test-results/report.json diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..e8f0654 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,11 @@ +# Development dependencies +-r requirements.txt + +# Code quality +black>=23.7.0 # Code formatting +flake8>=6.1.0 # Linting +mypy>=1.5.0 # Type checking +isort>=5.12.0 # Import sorting + +# Testing +pytest-cov>=4.1.0 # Coverage reporting diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0577522 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +# Core testing dependencies +pytest>=7.4.0 +pytest-xdist>=3.3.0 # Parallel test execution +pytest-json-report>=1.5.0 # JSON output format +pytest-timeout>=2.1.0 # Test timeout support + +# Database connectivity +pymongo>=4.5.0 + +# Configuration management +pyyaml>=6.0 + +# Reporting +jinja2>=3.1.2 # Template rendering for reports diff --git a/result_analyzer/README.md b/result_analyzer/README.md new file mode 100644 index 0000000..151882b --- /dev/null +++ b/result_analyzer/README.md @@ -0,0 +1,88 @@ +# Result Analyzer + +The Result Analyzer automatically processes pytest JSON reports and categorizes results by meaningful test markers. + +## Features + +- **Automatic Marker Detection**: Uses heuristics to identify meaningful test markers +- **Smart Filtering**: Automatically excludes test names, file names, and internal markers +- **Flexible**: Supports new markers without code changes +- **CLI Tool**: `docdb-analyze` command for quick analysis + +## Marker Detection + +The analyzer automatically identifies meaningful markers by excluding: + +### Excluded by Pattern +- **Test function names**: Any marker containing `[` or starting with `test_` +- **File names**: Any marker containing `.py` or `/` +- **Directory names**: Common names like `tests`, `functional-tests`, `src`, `lib` + +### Excluded by Name +- **Pytest internal**: `parametrize`, `usefixtures`, `filterwarnings`, `pytestmark` +- **Fixture markers**: `documents` (used for test data setup) +- **Engine names**: `documentdb`, `mongodb`, `cosmosdb`, `default` (from parametrization) + +### Result +Only meaningful categorization markers remain: +- Horizontal tags: `find`, `insert`, `update`, `delete`, `aggregate`, `index`, `admin`, `collection_mgmt` +- Vertical tags: `rbac`, `decimal128`, `collation`, `transactions`, `geospatial`, `text_search`, `validation`, `ttl` +- Special tags: `smoke`, `slow` + +## Adding New Markers + +Simply add the marker to `pytest.ini` and use it in your tests: + +```python +@pytest.mark.mynewfeature +def test_something(): + pass +``` + +The analyzer will automatically detect and report on it - no code changes needed! + +## Failure Categorization + +Tests are categorized into four types: + +1. **PASS**: Test succeeded +2. **FAIL**: Test failed, feature exists but behaves incorrectly +3. **UNSUPPORTED**: Feature not implemented (skipped tests) +4. **INFRA_ERROR**: Infrastructure issue (connection, timeout, etc.) + +## Usage + +### CLI +```bash +# Quick analysis +docdb-analyze + +# Custom input/output +docdb-analyze --input results.json --output report.txt +``` + +### Programmatic +```python +from result_analyzer import analyze_results, generate_report + +analysis = analyze_results("report.json") +generate_report(analysis, "report.txt", format="text") +``` + +## Maintenance + +The heuristic-based approach means: +- ✅ **No maintenance** for new test markers +- ✅ **Automatic adaptation** to new patterns +- ⚠️ **May need updates** if new fixture patterns emerge (e.g., new engine names) + +To add a new fixture marker or engine name to exclude, update the filter in `analyzer.py`: + +```python +# Skip fixture markers +if marker in {"documents", "mynewfixture"}: + continue + +# Skip engine names +if marker in {"documentdb", "mongodb", "mynewengine"}: + continue diff --git a/result_analyzer/__init__.py b/result_analyzer/__init__.py new file mode 100644 index 0000000..dd179b8 --- /dev/null +++ b/result_analyzer/__init__.py @@ -0,0 +1,11 @@ +""" +Result Analyzer for DocumentDB Functional Tests. + +This module provides tools for analyzing pytest test results and generating +reports categorized by feature tags. +""" + +from .analyzer import analyze_results, categorize_outcome +from .report_generator import generate_report, print_summary + +__all__ = ["analyze_results", "categorize_outcome", "generate_report", "print_summary"] diff --git a/result_analyzer/analyzer.py b/result_analyzer/analyzer.py new file mode 100644 index 0000000..156cf01 --- /dev/null +++ b/result_analyzer/analyzer.py @@ -0,0 +1,375 @@ +""" +Result analyzer for parsing and categorizing test results. + +This module provides functions to analyze pytest JSON output and categorize +test results by tags and failure types. +""" + +import json +import re +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, List + + +# Module-level constants and caches +INFRA_EXCEPTIONS = { + # Python built-in connection errors + "ConnectionError", + "ConnectionRefusedError", + "ConnectionResetError", + "ConnectionAbortedError", + # Python timeout errors + "TimeoutError", + "socket.timeout", + "socket.error", + # PyMongo connection errors + "pymongo.errors.ConnectionFailure", + "pymongo.errors.ServerSelectionTimeoutError", + "pymongo.errors.NetworkTimeout", + "pymongo.errors.AutoReconnect", + "pymongo.errors.ExecutionTimeout", + # Generic network/OS errors + "OSError", +} + +_REGISTERED_MARKERS_CACHE = None + + +# Mapping from TestOutcome to counter key names +OUTCOME_TO_KEY = { + "PASS": "passed", + "FAIL": "failed", + "UNSUPPORTED": "unsupported", + "SKIPPED": "skipped", + "INFRA_ERROR": "infra_error", +} + + +class TestOutcome: + """Enumeration of test outcomes.""" + + PASS = "PASS" + FAIL = "FAIL" + UNSUPPORTED = "UNSUPPORTED" # Error code 115 - feature not implemented + SKIPPED = "SKIPPED" # Environmental or conditional skip + INFRA_ERROR = "INFRA_ERROR" + + +def categorize_outcome(test_result: Dict[str, Any]) -> str: + """ + Categorize a test outcome based on test result information. + + Categories: + - PASS: Test passed + - UNSUPPORTED: Failed with error code 115 (unsupported feature) + - INFRA_ERROR: Infrastructure issues (connection, timeout, etc.) + - FAIL: Functional failure (bug/incorrect behavior) + - SKIPPED: Environmental or conditional skip + + Args: + test_result: Test result dictionary from pytest JSON report + + Returns: + One of: PASS, FAIL, UNSUPPORTED, SKIPPED, INFRA_ERROR + """ + outcome = test_result.get("outcome", "") + + if outcome == "passed": + return TestOutcome.PASS + + if outcome == "failed": + # Analyze the failure to determine the type + call_info = test_result.get("call", {}) + longrepr = call_info.get("longrepr", "") + + # Check for error code 115 (unsupported feature) - highest priority + if is_unsupported_error(longrepr): + return TestOutcome.UNSUPPORTED + + # Check for infrastructure-related errors (pass full test_result) + if is_infrastructure_error(test_result): + return TestOutcome.INFRA_ERROR + + # Otherwise, it's a functional failure + return TestOutcome.FAIL + + if outcome == "skipped": + # Environmental or conditional skip + return TestOutcome.SKIPPED + + # Unknown outcome, treat as infrastructure error + return TestOutcome.INFRA_ERROR + + +def is_unsupported_error(error_text: str) -> bool: + """ + Check if error contains code 115 (unsupported feature). + + DocumentDB returns error code 115 for unsupported features in the + error dictionary as 'code': 115 or "code": 115. + + Args: + error_text: Error message text from test failure + + Returns: + True if error contains code 115, False otherwise + """ + if not error_text: + return False + + # Match 'code': 115 or "code": 115 in error dict + return bool(re.search(r"['\"]code['\"]:\s*115", error_text)) + + +def extract_exception_type(crash_message: str) -> str: + """ + Extract exception type from pytest crash message. + + Args: + crash_message: Message like "module.Exception: error details" + + Returns: + Full exception type (e.g., "pymongo.errors.OperationFailure") + or empty string if not found + """ + # Match pattern: "module.exception.Type: message" + # Capture everything before the first colon + match = re.match(r'^([a-zA-Z0-9_.]+):\s', crash_message) + if match: + return match.group(1) + + return "" + + +def is_infrastructure_error(test_result: Dict[str, Any]) -> bool: + """ + Check if error is infrastructure-related based on exception type. + + This checks the actual exception type rather than keywords in error messages, + preventing false positives from error messages that happen to contain + infrastructure-related words (e.g., "host" in an assertion message). + + Args: + test_result: Full test result dict from pytest JSON + + Returns: + True if error is infrastructure-related, False otherwise + """ + # Get the crash info from call + call_info = test_result.get("call", {}) + crash_info = call_info.get("crash", {}) + crash_message = crash_info.get("message", "") + + if not crash_message: + return False + + # Extract exception type from "module.ExceptionClass: message" format + exception_type = extract_exception_type(crash_message) + + if not exception_type: + return False + + # Check against module-level constant + return exception_type in INFRA_EXCEPTIONS + + +def load_registered_markers(pytest_ini_path: str = "pytest.ini") -> set: + """ + Load registered markers from pytest.ini. + + Parses the markers section to extract marker names, ensuring we only + use markers that are explicitly registered in pytest configuration. + + Args: + pytest_ini_path: Path to pytest.ini file (defaults to "pytest.ini") + + Returns: + Set of registered marker names + """ + # Check if pytest.ini exists + if not Path(pytest_ini_path).exists(): + return set() + + registered_markers = set() + + try: + with open(pytest_ini_path, 'r') as f: + in_markers_section = False + + for line in f: + # Check if we're entering the markers section + if line.strip() == "markers =": + in_markers_section = True + continue + + if in_markers_section: + # Marker lines are indented, config keys are not + if line and not line[0].isspace(): + # Non-indented line means we left the markers section + break + + # Parse indented marker lines like " find: Find operation tests" + match = re.match(r'^\s+([a-zA-Z0-9_]+):', line) + if match: + registered_markers.add(match.group(1)) + + except Exception: + # If parsing fails, return empty set + pass + + return registered_markers + + +def extract_markers(test_result: Dict[str, Any]) -> List[str]: + """ + Extract pytest markers (tags) from a test result. + + Uses registered markers from pytest.ini as an allow list. + This ensures only intentional test categorization markers are included, + avoiding brittle heuristics that could break with future pytest versions. + + Markers are automatically loaded from pytest.ini, so they only need to be + defined in one place. + + Args: + test_result: Test result dictionary from pytest JSON report + + Returns: + List of marker names that match registered markers from pytest.ini + """ + global _REGISTERED_MARKERS_CACHE + + # Load registered markers from pytest.ini (cached for performance) + if _REGISTERED_MARKERS_CACHE is None: + _REGISTERED_MARKERS_CACHE = load_registered_markers() + + registered_markers = _REGISTERED_MARKERS_CACHE + + markers = [] + + # Extract from keywords + keywords = test_result.get("keywords", []) + if isinstance(keywords, list): + markers.extend(keywords) + + # Extract from markers field if present + test_markers = test_result.get("markers", []) + if isinstance(test_markers, list): + for marker in test_markers: + if isinstance(marker, dict): + markers.append(marker.get("name", "")) + else: + markers.append(str(marker)) + + # Filter to only registered markers + return [m for m in markers if m in registered_markers] + + +def analyze_results(json_report_path: str) -> Dict[str, Any]: + """ + Analyze pytest JSON report and generate categorized results. + + Args: + json_report_path: Path to pytest JSON report file + + Returns: + Dictionary containing analysis results with structure: + { + "summary": { + "total": int, + "passed": int, + "failed": int, + "unsupported": int, + "infra_error": int + }, + "by_tag": { + "tag_name": { + "passed": int, + "failed": int, + "unsupported": int, + "infra_error": int, + "pass_rate": float + } + }, + "tests": [ + { + "name": str, + "outcome": str, + "duration": float, + "tags": List[str], + "error": str (optional) + } + ] + } + """ + # Load JSON report + with open(json_report_path, "r") as f: + report = json.load(f) + + # Initialize counters + summary: Dict[str, Any] = { + "total": 0, + "passed": 0, + "failed": 0, + "unsupported": 0, + "skipped": 0, + "infra_error": 0, + } + + by_tag: Dict[str, Dict[str, int]] = defaultdict( + lambda: {"passed": 0, "failed": 0, "unsupported": 0, "skipped": 0, "infra_error": 0} + ) + + tests_details = [] + + # Process each test + tests = report.get("tests", []) + for test in tests: + summary["total"] += 1 + + # Categorize the outcome + test_outcome = categorize_outcome(test) + + # Extract tags + tags = extract_markers(test) + + # Update summary counters using mapping + counter_key = OUTCOME_TO_KEY.get(test_outcome) + if counter_key: + summary[counter_key] += 1 + + # Update tag-specific counters + if counter_key: + for tag in tags: + by_tag[tag][counter_key] += 1 + + # Store test details + test_detail = { + "name": test.get("nodeid", ""), + "outcome": test_outcome, + "duration": test.get("duration", 0), + "tags": tags, + } + + # Add error information if failed + if test_outcome in [TestOutcome.FAIL, TestOutcome.INFRA_ERROR]: + call_info = test.get("call", {}) + test_detail["error"] = call_info.get("longrepr", "") + + tests_details.append(test_detail) + + # Calculate pass rates for each tag + by_tag_with_rates = {} + for tag, counts in by_tag.items(): + total = counts["passed"] + counts["failed"] + counts["unsupported"] + counts["infra_error"] + pass_rate = (counts["passed"] / total * 100) if total > 0 else 0 + + by_tag_with_rates[tag] = {**counts, "total": total, "pass_rate": round(pass_rate, 2)} + + # Calculate overall pass rate + summary["pass_rate"] = round( + (summary["passed"] / summary["total"] * 100) if summary["total"] > 0 else 0, 2 + ) + + return {"summary": summary, "by_tag": by_tag_with_rates, "tests": tests_details} diff --git a/result_analyzer/cli.py b/result_analyzer/cli.py new file mode 100644 index 0000000..99ccad7 --- /dev/null +++ b/result_analyzer/cli.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +""" +DocumentDB Functional Test Results Analyzer - CLI + +Command-line interface for analyzing pytest JSON reports and generating +categorized results by feature tags. +""" + +import argparse +import sys +from pathlib import Path + +from .analyzer import analyze_results +from .report_generator import generate_report, print_summary + + +def main(): + parser = argparse.ArgumentParser( + description="Analyze DocumentDB functional test results and generate reports", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Analyze default report location + %(prog)s + + # Analyze specific report + %(prog)s --input my-results.json + + # Generate text report + %(prog)s --output report.txt --format text + + # Generate JSON analysis + %(prog)s --output analysis.json --format json + + # Quiet mode (no console output) + %(prog)s --output report.txt --quiet + """, + ) + + parser.add_argument( + "-i", + "--input", + default=".test-results/report.json", + help="Path to pytest JSON report file (default: .test-results/report.json)", + ) + + parser.add_argument( + "-o", "--output", help="Path to output report file (if not specified, prints to console)" + ) + + parser.add_argument( + "-f", + "--format", + choices=["text", "json"], + default="text", + help="Output format: text or json (default: text)", + ) + + parser.add_argument( + "-q", + "--quiet", + action="store_true", + help="Suppress console output (only write to output file)", + ) + + parser.add_argument( + "--no-summary", action="store_true", help="Skip printing summary to console" + ) + + args = parser.parse_args() + + # Check if input file exists + input_path = Path(args.input) + if not input_path.exists(): + print(f"Error: Input file not found: {args.input}", file=sys.stderr) + print("\nMake sure to run pytest with --json-report first:", file=sys.stderr) + print(f" pytest --json-report --json-report-file={args.input}", file=sys.stderr) + return 1 + + try: + # Analyze the results + if not args.quiet: + print(f"Analyzing test results from: {args.input}") + + analysis = analyze_results(args.input) + + # Print summary to console (unless quiet or no-summary) + if not args.quiet and not args.no_summary: + print_summary(analysis) + + # Generate output file if specified + if args.output: + generate_report(analysis, args.output, format=args.format) + if not args.quiet: + print(f"\nReport saved to: {args.output}") + + # If no output file and quiet mode, print to stdout + elif not args.quiet: + print("\nResults by Tag:") + print("-" * 60) + for tag, stats in sorted( + analysis["by_tag"].items(), key=lambda x: x[1]["pass_rate"], reverse=True + ): + passed = stats["passed"] + total = stats["total"] + rate = stats["pass_rate"] + print(f"{tag:30s} | {passed:3d}/{total:3d} passed ({rate:5.1f}%)") + + # Return exit code based on test results + if analysis["summary"]["failed"] > 0: + return 1 + return 0 + + except Exception as e: + print(f"Error analyzing results: {e}", file=sys.stderr) + if not args.quiet: + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/result_analyzer/report_generator.py b/result_analyzer/report_generator.py new file mode 100644 index 0000000..ed6b44c --- /dev/null +++ b/result_analyzer/report_generator.py @@ -0,0 +1,137 @@ +""" +Report generator for creating human-readable reports from analysis results. + +This module provides functions to generate various report formats from +analyzed test results. +""" + +import json +from datetime import datetime +from typing import Any, Dict + + +def generate_report(analysis: Dict[str, Any], output_path: str, format: str = "json"): + """ + Generate a report from analysis results. + + Args: + analysis: Analysis results from analyze_results() + output_path: Path to write the report + format: Report format ("json" or "text") + """ + if format == "json": + generate_json_report(analysis, output_path) + elif format == "text": + generate_text_report(analysis, output_path) + else: + raise ValueError(f"Unsupported report format: {format}") + + +def generate_json_report(analysis: Dict[str, Any], output_path: str): + """ + Generate a JSON report. + + Args: + analysis: Analysis results from analyze_results() + output_path: Path to write the JSON report + """ + report = { + "generated_at": datetime.now().isoformat(), + "summary": analysis["summary"], + "by_tag": analysis["by_tag"], + "tests": analysis["tests"], + } + + with open(output_path, "w") as f: + json.dump(report, f, indent=2) + + +def generate_text_report(analysis: Dict[str, Any], output_path: str): + """ + Generate a human-readable text report. + + Args: + analysis: Analysis results from analyze_results() + output_path: Path to write the text report + """ + lines = [] + + # Header + lines.append("=" * 80) + lines.append("DocumentDB Functional Test Results") + lines.append("=" * 80) + lines.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + lines.append("") + + # Summary + summary = analysis["summary"] + lines.append("SUMMARY") + lines.append("-" * 80) + lines.append(f"Total Tests: {summary['total']}") + lines.append(f"Passed: {summary['passed']} ({summary['pass_rate']}%)") + lines.append(f"Failed: {summary['failed']}") + lines.append(f"Unsupported: {summary['unsupported']}") + lines.append(f"Skipped: {summary['skipped']}") + lines.append(f"Infrastructure Errors: {summary['infra_error']}") + lines.append("") + + # Results by tag + lines.append("RESULTS BY TAG") + lines.append("-" * 80) + + if analysis["by_tag"]: + # Sort tags by pass rate (ascending) to highlight problematic areas + sorted_tags = sorted(analysis["by_tag"].items(), key=lambda x: x[1]["pass_rate"]) + + for tag, stats in sorted_tags: + lines.append(f"\n{tag}:") + lines.append(f" Total: {stats['total']}") + lines.append(f" Passed: {stats['passed']} ({stats['pass_rate']}%)") + lines.append(f" Failed: {stats['failed']}") + lines.append(f" Unsupported: {stats['unsupported']}") + lines.append(f" Skipped: {stats['skipped']}") + lines.append(f" Infra Error: {stats['infra_error']}") + else: + lines.append("No tags found in test results.") + + lines.append("") + + # Failed tests details + failed_tests = [t for t in analysis["tests"] if t["outcome"] == "FAIL"] + if failed_tests: + lines.append("FAILED TESTS") + lines.append("-" * 80) + for test in failed_tests: + lines.append(f"\n{test['name']}") + lines.append(f" Tags: {', '.join(test['tags'])}") + lines.append(f" Duration: {test['duration']:.2f}s") + if "error" in test: + error_preview = test["error"][:200] + lines.append(f" Error: {error_preview}...") + + lines.append("") + lines.append("=" * 80) + + # Write report + with open(output_path, "w") as f: + f.write("\n".join(lines)) + + +def print_summary(analysis: Dict[str, Any]): + """ + Print a brief summary to console. + + Args: + analysis: Analysis results from analyze_results() + """ + summary = analysis["summary"] + print("\n" + "=" * 60) + print("Test Results Summary") + print("=" * 60) + print(f"Total: {summary['total']}") + print(f"Passed: {summary['passed']} ({summary['pass_rate']}%)") + print(f"Failed: {summary['failed']}") + print(f"Unsupported: {summary['unsupported']}") + print(f"Skipped: {summary['skipped']}") + print(f"Infra Error: {summary['infra_error']}") + print("=" * 60 + "\n") diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..22194cf --- /dev/null +++ b/setup.py @@ -0,0 +1,39 @@ +from setuptools import find_packages, setup + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +setup( + name="documentdb-functional-tests", + version="0.1.0", + author="DocumentDB Contributors", + description="End-to-end functional testing framework for DocumentDB", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/documentdb/functional-tests", + packages=find_packages(), + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + ], + python_requires=">=3.9", + install_requires=[ + "pytest>=7.4.0", + "pytest-xdist>=3.3.0", + "pytest-json-report>=1.5.0", + "pytest-timeout>=2.1.0", + "pymongo>=4.5.0", + "pyyaml>=6.0", + "jinja2>=3.1.2", + ], + entry_points={ + "console_scripts": [ + "docdb-analyze=result_analyzer.cli:main", + ], + }, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..3492497 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,5 @@ +""" +DocumentDB Functional Tests + +End-to-end functional testing suite for DocumentDB. +""" diff --git a/tests/aggregate/__init__.py b/tests/aggregate/__init__.py new file mode 100644 index 0000000..1103a4a --- /dev/null +++ b/tests/aggregate/__init__.py @@ -0,0 +1 @@ +"""Aggregation pipeline tests.""" diff --git a/tests/aggregate/test_group_stage.py b/tests/aggregate/test_group_stage.py new file mode 100644 index 0000000..3af0710 --- /dev/null +++ b/tests/aggregate/test_group_stage.py @@ -0,0 +1,128 @@ +""" +Aggregation $group stage tests. + +Tests for the $group stage in aggregation pipelines. +""" + +import pytest + + +@pytest.mark.aggregate +@pytest.mark.smoke +def test_group_with_count(collection): + """Test $group stage with count aggregation.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "department": "Engineering", "salary": 100000}, + {"name": "Bob", "department": "Engineering", "salary": 90000}, + {"name": "Charlie", "department": "Sales", "salary": 80000}, + {"name": "David", "department": "Sales", "salary": 75000}, + ]) + + # Act - Execute aggregation to count documents by department + pipeline = [{"$group": {"_id": "$department", "count": {"$sum": 1}}}] + result = list(collection.aggregate(pipeline)) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 departments" + + # Convert to dict for easier verification + dept_counts = {doc["_id"]: doc["count"] for doc in result} + assert dept_counts["Engineering"] == 2, "Expected 2 employees in Engineering" + assert dept_counts["Sales"] == 2, "Expected 2 employees in Sales" + + +@pytest.mark.aggregate +def test_group_with_sum(collection): + """Test $group stage with sum aggregation.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "department": "Engineering", "salary": 100000}, + {"name": "Bob", "department": "Engineering", "salary": 90000}, + {"name": "Charlie", "department": "Sales", "salary": 80000}, + ]) + + # Act - Execute aggregation to sum salaries by department + pipeline = [{"$group": {"_id": "$department", "totalSalary": {"$sum": "$salary"}}}] + result = list(collection.aggregate(pipeline)) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 departments" + + # Convert to dict for easier verification + dept_salaries = {doc["_id"]: doc["totalSalary"] for doc in result} + assert dept_salaries["Engineering"] == 190000, "Expected total Engineering salary of 190000" + assert dept_salaries["Sales"] == 80000, "Expected total Sales salary of 80000" + + +@pytest.mark.aggregate +def test_group_with_avg(collection): + """Test $group stage with average aggregation.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "department": "Engineering", "salary": 100000}, + {"name": "Bob", "department": "Engineering", "salary": 90000}, + {"name": "Charlie", "department": "Sales", "salary": 80000}, + ]) + + # Act - Execute aggregation to calculate average salary by department + pipeline = [{"$group": {"_id": "$department", "avgSalary": {"$avg": "$salary"}}}] + result = list(collection.aggregate(pipeline)) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 departments" + + # Convert to dict for easier verification + dept_avg = {doc["_id"]: doc["avgSalary"] for doc in result} + assert dept_avg["Engineering"] == 95000, "Expected average Engineering salary of 95000" + assert dept_avg["Sales"] == 80000, "Expected average Sales salary of 80000" + + +@pytest.mark.aggregate +def test_group_with_min_max(collection): + """Test $group stage with min and max aggregations.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "department": "Engineering", "salary": 100000}, + {"name": "Bob", "department": "Engineering", "salary": 90000}, + {"name": "Charlie", "department": "Sales", "salary": 80000}, + ]) + + # Act - Execute aggregation to find min and max salary by department + pipeline = [ + { + "$group": { + "_id": "$department", + "minSalary": {"$min": "$salary"}, + "maxSalary": {"$max": "$salary"}, + } + } + ] + result = list(collection.aggregate(pipeline)) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 departments" + + # Verify Engineering department + eng_dept = next(doc for doc in result if doc["_id"] == "Engineering") + assert eng_dept["minSalary"] == 90000, "Expected min Engineering salary of 90000" + assert eng_dept["maxSalary"] == 100000, "Expected max Engineering salary of 100000" + + +@pytest.mark.aggregate +def test_group_all_documents(collection): + """Test $group stage grouping all documents (using null as _id).""" + # Arrange - Insert test data + collection.insert_many([ + {"item": "A", "quantity": 5}, + {"item": "B", "quantity": 10}, + {"item": "A", "quantity": 3}, + ]) + + # Act - Execute aggregation to sum quantities across all documents + pipeline = [{"$group": {"_id": None, "totalQuantity": {"$sum": "$quantity"}}}] + result = list(collection.aggregate(pipeline)) + + # Assert - Verify results + assert len(result) == 1, "Expected single result grouping all documents" + assert result[0]["totalQuantity"] == 18, "Expected total quantity of 18" diff --git a/tests/aggregate/test_match_stage.py b/tests/aggregate/test_match_stage.py new file mode 100644 index 0000000..0643fea --- /dev/null +++ b/tests/aggregate/test_match_stage.py @@ -0,0 +1,86 @@ +""" +Aggregation $match stage tests. + +Tests for the $match stage in aggregation pipelines. +""" + +import pytest + + +@pytest.mark.aggregate +@pytest.mark.smoke +def test_match_simple_filter(collection): + """Test $match stage with simple equality filter.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30, "status": "active"}, + {"name": "Bob", "age": 25, "status": "active"}, + {"name": "Charlie", "age": 35, "status": "inactive"}, + ]) + + # Act - Execute aggregation with $match stage + pipeline = [{"$match": {"status": "active"}}] + result = list(collection.aggregate(pipeline)) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 active users" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Bob"}, "Expected Alice and Bob" + + +@pytest.mark.aggregate +def test_match_with_comparison_operator(collection): + """Test $match stage with comparison operators.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ]) + + # Act - Execute aggregation with $match using $gt + pipeline = [{"$match": {"age": {"$gt": 25}}}] + result = list(collection.aggregate(pipeline)) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 users with age > 25" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Charlie"}, "Expected Alice and Charlie" + + +@pytest.mark.aggregate +def test_match_multiple_conditions(collection): + """Test $match stage with multiple filter conditions.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30, "city": "NYC"}, + {"name": "Bob", "age": 25, "city": "SF"}, + {"name": "Charlie", "age": 35, "city": "NYC"}, + ]) + + # Act - Execute aggregation with multiple conditions in $match + pipeline = [{"$match": {"city": "NYC", "age": {"$gte": 30}}}] + result = list(collection.aggregate(pipeline)) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 users from NYC with age >= 30" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Charlie"}, "Expected Alice and Charlie" + + +@pytest.mark.aggregate +@pytest.mark.find +def test_match_empty_result(collection): + """Test $match stage that matches no documents.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "status": "active"}, + {"name": "Bob", "status": "active"}, + ]) + + # Act - Execute aggregation with $match that matches nothing + pipeline = [{"$match": {"status": "inactive"}}] + result = list(collection.aggregate(pipeline)) + + # Assert - Verify empty result + assert result == [], "Expected empty result when no documents match" diff --git a/tests/collection/__init__.py b/tests/collection/__init__.py new file mode 100644 index 0000000..b73cf50 --- /dev/null +++ b/tests/collection/__init__.py @@ -0,0 +1 @@ +"""Collection management tests.""" diff --git a/tests/collection/test_capped_collections.py b/tests/collection/test_capped_collections.py new file mode 100644 index 0000000..0f1acd4 --- /dev/null +++ b/tests/collection/test_capped_collections.py @@ -0,0 +1,43 @@ +""" +Tests for capped collection operations. + +NOTE: This test was added by design to demonstrate how the Result Analyzer +automatically detects and categorizes unsupported features (error code 115). +When run against DocumentDB, this test will fail with error code 115, which +the analyzer will correctly categorize as UNSUPPORTED rather than FAIL. +""" + +import pytest +from pymongo.errors import OperationFailure + + +@pytest.mark.collection_mgmt +def test_create_capped_collection(database_client): + """ + Test creating a capped collection. + + Capped collections are fixed-size collections that maintain insertion order. + This feature is not supported in DocumentDB and should return error code 115. + + Expected behavior: + - Creates a capped collection successfully + """ + collection_name = "capped_test_collection" + + try: + # Attempt to create capped collection + database_client.create_collection( + collection_name, + capped=True, + size=100000 + ) + + # Verify it's actually capped + collection_info = database_client[collection_name].options() + assert collection_info.get("capped") is True, "Collection should be capped" + + # Cleanup + database_client.drop_collection(collection_name) + + except OperationFailure as e: + raise diff --git a/tests/common/__init__.py b/tests/common/__init__.py new file mode 100644 index 0000000..b34d9ac --- /dev/null +++ b/tests/common/__init__.py @@ -0,0 +1 @@ +"""Common utilities and helpers for functional tests.""" diff --git a/tests/common/assertions.py b/tests/common/assertions.py new file mode 100644 index 0000000..c6d9e89 --- /dev/null +++ b/tests/common/assertions.py @@ -0,0 +1,103 @@ +""" +Custom assertion helpers for functional tests. + +Provides convenient assertion methods for common test scenarios. +""" + +from typing import Dict, List + + +def assert_document_match(actual: Dict, expected: Dict, ignore_id: bool = True): + """ + Assert that a document matches the expected structure and values. + + Args: + actual: The actual document from the database + expected: The expected document structure + ignore_id: If True, ignore _id field in comparison (default: True) + """ + if ignore_id: + actual = {k: v for k, v in actual.items() if k != "_id"} + expected = {k: v for k, v in expected.items() if k != "_id"} + + assert actual == expected, f"Document mismatch.\nExpected: {expected}\nActual: {actual}" + + +def assert_documents_match( + actual: List[Dict], expected: List[Dict], ignore_id: bool = True, ignore_order: bool = False +): + """ + Assert that a list of documents matches the expected list. + + Args: + actual: List of actual documents from the database + expected: List of expected documents + ignore_id: If True, ignore _id field in comparison (default: True) + ignore_order: If True, sort both lists before comparison (default: False) + """ + if ignore_id: + actual = [{k: v for k, v in doc.items() if k != "_id"} for doc in actual] + expected = [{k: v for k, v in doc.items() if k != "_id"} for doc in expected] + + assert len(actual) == len( + expected + ), f"Document count mismatch. Expected {len(expected)}, got {len(actual)}" + + if ignore_order: + # Sort for comparison + actual = sorted(actual, key=lambda x: str(x)) + expected = sorted(expected, key=lambda x: str(x)) + + for i, (act, exp) in enumerate(zip(actual, expected)): + assert act == exp, f"Document at index {i} does not match.\nExpected: {exp}\nActual: {act}" + + +def assert_field_exists(document: Dict, field_path: str): + """ + Assert that a field exists in a document (supports nested paths). + + Args: + document: The document to check + field_path: Dot-notation field path (e.g., "user.name") + """ + parts = field_path.split(".") + current = document + + for part in parts: + assert part in current, f"Field '{field_path}' does not exist in document" + current = current[part] + + +def assert_field_not_exists(document: Dict, field_path: str): + """ + Assert that a field does not exist in a document (supports nested paths). + + Args: + document: The document to check + field_path: Dot-notation field path (e.g., "user.name") + """ + parts = field_path.split(".") + current = document + + for i, part in enumerate(parts): + if part not in current: + return # Field doesn't exist, assertion passes + if i == len(parts) - 1: + raise AssertionError(f"Field '{field_path}' exists in document but should not") + current = current[part] + + +def assert_count(collection, filter_query: Dict, expected_count: int): + """ + Assert that a collection contains the expected number of documents matching a filter. + + Args: + collection: MongoDB collection object + filter_query: Query filter + expected_count: Expected number of matching documents + """ + actual_count = collection.count_documents(filter_query) + assert actual_count == expected_count, ( + f"Document count mismatch for filter {filter_query}. " + f"Expected {expected_count}, got {actual_count}" + ) diff --git a/tests/find/__init__.py b/tests/find/__init__.py new file mode 100644 index 0000000..0bdc7e7 --- /dev/null +++ b/tests/find/__init__.py @@ -0,0 +1 @@ +"""Find operation tests.""" diff --git a/tests/find/test_basic_queries.py b/tests/find/test_basic_queries.py new file mode 100644 index 0000000..3de9c70 --- /dev/null +++ b/tests/find/test_basic_queries.py @@ -0,0 +1,134 @@ +""" +Basic find operation tests. + +Tests for fundamental find() and findOne() operations. +""" + +import pytest + +from tests.common.assertions import assert_document_match + + +@pytest.mark.find +@pytest.mark.smoke +def test_find_all_documents(collection): + """Test finding all documents in a collection.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30, "status": "active"}, + {"name": "Bob", "age": 25, "status": "active"}, + {"name": "Charlie", "age": 35, "status": "inactive"}, + ]) + + # Act - Execute find operation + result = list(collection.find()) + + # Assert - Verify all documents are returned + assert len(result) == 3, "Expected to find 3 documents" + + # Verify document content + names = {doc["name"] for doc in result} + assert names == {"Alice", "Bob", "Charlie"}, "Expected to find all three users" + + +@pytest.mark.find +@pytest.mark.smoke +def test_find_with_filter(collection): + """Test find operation with a simple equality filter.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30, "status": "active"}, + {"name": "Bob", "age": 25, "status": "active"}, + {"name": "Charlie", "age": 35, "status": "inactive"}, + ]) + + # Act - Execute find with filter + result = list(collection.find({"status": "active"})) + + # Assert - Verify only active users are returned + assert len(result) == 2, "Expected to find 2 active users" + + # Verify all returned documents have status "active" + for doc in result: + assert doc["status"] == "active", "All returned documents should have status 'active'" + + # Verify correct users are returned + names = {doc["name"] for doc in result} + assert names == {"Alice", "Bob"}, "Expected to find Alice and Bob" + + +@pytest.mark.find +def test_find_one(collection): + """Test findOne operation returns a single document.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + ]) + + # Act - Execute findOne + result = collection.find_one({"name": "Alice"}) + + # Assert - Verify document is returned and content matches + assert result is not None, "Expected to find a document" + assert_document_match(result, {"name": "Alice", "age": 30}) + + +@pytest.mark.find +def test_find_one_not_found(collection): + """Test findOne returns None when no document matches.""" + # Arrange - Insert test data + collection.insert_one({"name": "Alice", "age": 30}) + + # Act - Execute findOne with non-matching filter + result = collection.find_one({"name": "NonExistent"}) + + # Assert - Verify None is returned + assert result is None, "Expected None when no document matches" + + +@pytest.mark.find +def test_find_empty_collection(collection): + """Test find on an empty collection returns empty result.""" + # Arrange - Collection is already empty (no insertion needed) + + # Act - Execute find on empty collection + result = list(collection.find()) + + # Assert - Verify empty result + assert result == [], "Expected empty result for empty collection" + + +@pytest.mark.find +def test_find_with_multiple_conditions(collection): + """Test find with multiple filter conditions (implicit AND).""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30, "city": "NYC"}, + {"name": "Bob", "age": 25, "city": "SF"}, + {"name": "Charlie", "age": 35, "city": "NYC"}, + ]) + + # Act - Execute find with multiple conditions + result = list(collection.find({"city": "NYC", "age": 30})) + + # Assert - Verify only matching document is returned + assert len(result) == 1, "Expected to find 1 document" + assert_document_match(result[0], {"name": "Alice", "age": 30, "city": "NYC"}) + + +@pytest.mark.find +def test_find_nested_field(collection): + """Test find with nested field query using dot notation.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "profile": {"age": 30, "city": "NYC"}}, + {"name": "Bob", "profile": {"age": 25, "city": "SF"}}, + ]) + + # Act - Execute find with nested field + result = list(collection.find({"profile.city": "NYC"})) + + # Assert - Verify correct document is returned + assert len(result) == 1, "Expected to find 1 document" + assert result[0]["name"] == "Alice", "Expected to find Alice" diff --git a/tests/find/test_projections.py b/tests/find/test_projections.py new file mode 100644 index 0000000..ab941bd --- /dev/null +++ b/tests/find/test_projections.py @@ -0,0 +1,117 @@ +""" +Projection tests for find operations. + +Tests for field inclusion, exclusion, and projection operators. +""" + +import pytest + +from tests.common.assertions import assert_field_exists, assert_field_not_exists + + +@pytest.mark.find +def test_find_with_field_inclusion(collection): + """Test find with explicit field inclusion.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30, "email": "alice@example.com", "city": "NYC"}, + {"name": "Bob", "age": 25, "email": "bob@example.com", "city": "SF"}, + ]) + + # Act - Find with projection to include only name and age + result = list(collection.find({}, {"name": 1, "age": 1})) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 documents" + + # Verify included fields exist + for doc in result: + assert_field_exists(doc, "_id") # _id is included by default + assert_field_exists(doc, "name") + assert_field_exists(doc, "age") + + # Verify excluded fields don't exist + assert_field_not_exists(doc, "email") + assert_field_not_exists(doc, "city") + + +@pytest.mark.find +def test_find_with_field_exclusion(collection): + """Test find with explicit field exclusion.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30, "email": "alice@example.com", "city": "NYC"}, + {"name": "Bob", "age": 25, "email": "bob@example.com", "city": "SF"}, + ]) + + # Act - Find with projection to exclude email and city + result = list(collection.find({}, {"email": 0, "city": 0})) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 documents" + + # Verify included fields exist + for doc in result: + assert_field_exists(doc, "_id") + assert_field_exists(doc, "name") + assert_field_exists(doc, "age") + + # Verify excluded fields don't exist + assert_field_not_exists(doc, "email") + assert_field_not_exists(doc, "city") + + +@pytest.mark.find +def test_find_exclude_id(collection): + """Test find with _id exclusion.""" + # Arrange - Insert test data + collection.insert_one({"name": "Alice", "age": 30, "email": "alice@example.com"}) + + # Act - Find with projection to exclude _id + result = list(collection.find({}, {"_id": 0, "name": 1, "age": 1})) + + # Assert - Verify results + assert len(result) == 1, "Expected 1 document" + + # Verify _id is excluded + assert_field_not_exists(result[0], "_id") + assert_field_exists(result[0], "name") + assert_field_exists(result[0], "age") + + +@pytest.mark.find +def test_find_nested_field_projection(collection): + """Test find with nested field projection.""" + # Arrange - Insert test data + collection.insert_one({ + "name": "Alice", + "profile": {"age": 30, "city": "NYC", "email": "alice@example.com"} + }) + + # Act - Find with projection for nested field + result = list(collection.find({}, {"name": 1, "profile.age": 1})) + + # Assert - Verify results + assert len(result) == 1, "Expected 1 document" + assert_field_exists(result[0], "name") + assert_field_exists(result[0], "profile.age") + + # Verify other nested fields are not included + assert "city" not in result[0].get("profile", {}), "city should not be in profile" + assert "email" not in result[0].get("profile", {}), "email should not be in profile" + + +@pytest.mark.find +@pytest.mark.smoke +def test_find_empty_projection(collection): + """Test find with empty projection returns all fields.""" + # Arrange - Insert test data + collection.insert_one({"name": "Alice", "age": 30}) + + # Act - Find with empty projection + result = collection.find_one({}, {}) + + # Assert - Verify all fields are present + assert_field_exists(result, "_id") + assert_field_exists(result, "name") + assert_field_exists(result, "age") diff --git a/tests/find/test_query_operators.py b/tests/find/test_query_operators.py new file mode 100644 index 0000000..9cd6bd7 --- /dev/null +++ b/tests/find/test_query_operators.py @@ -0,0 +1,159 @@ +""" +Query operator tests for find operations. + +Tests for comparison operators: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin +""" + +import pytest + + +@pytest.mark.find +def test_find_gt_operator(collection): + """Test find with $gt (greater than) operator.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ]) + + # Act - Find documents where age > 25 + result = list(collection.find({"age": {"$gt": 25}})) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 documents with age > 25" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Charlie"}, "Expected Alice and Charlie" + + +@pytest.mark.find +def test_find_gte_operator(collection): + """Test find with $gte (greater than or equal) operator.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ]) + + # Act - Find documents where age >= 30 + result = list(collection.find({"age": {"$gte": 30}})) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 documents with age >= 30" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Charlie"}, "Expected Alice and Charlie" + + +@pytest.mark.find +def test_find_lt_operator(collection): + """Test find with $lt (less than) operator.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ]) + + # Act - Find documents where age < 30 + result = list(collection.find({"age": {"$lt": 30}})) + + # Assert - Verify results + assert len(result) == 1, "Expected 1 document with age < 30" + assert result[0]["name"] == "Bob", "Expected Bob" + + +@pytest.mark.find +def test_find_lte_operator(collection): + """Test find with $lte (less than or equal) operator.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ]) + + # Act - Find documents where age <= 30 + result = list(collection.find({"age": {"$lte": 30}})) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 documents with age <= 30" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Bob"}, "Expected Alice and Bob" + + +@pytest.mark.find +def test_find_ne_operator(collection): + """Test find with $ne (not equal) operator.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ]) + + # Act - Find documents where age != 30 + result = list(collection.find({"age": {"$ne": 30}})) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 documents with age != 30" + names = {doc["name"] for doc in result} + assert names == {"Bob", "Charlie"}, "Expected Bob and Charlie" + + +@pytest.mark.find +def test_find_in_operator(collection): + """Test find with $in operator.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "status": "active"}, + {"name": "Bob", "status": "inactive"}, + {"name": "Charlie", "status": "pending"}, + {"name": "David", "status": "active"}, + ]) + + # Act - Find documents where status is in ["active", "pending"] + result = list(collection.find({"status": {"$in": ["active", "pending"]}})) + + # Assert - Verify results + assert len(result) == 3, "Expected 3 documents with status in [active, pending]" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Charlie", "David"}, "Expected Alice, Charlie, and David" + + +@pytest.mark.find +def test_find_nin_operator(collection): + """Test find with $nin (not in) operator.""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "status": "active"}, + {"name": "Bob", "status": "inactive"}, + {"name": "Charlie", "status": "pending"}, + ]) + + # Act - Find documents where status is not in ["active", "pending"] + result = list(collection.find({"status": {"$nin": ["active", "pending"]}})) + + # Assert - Verify results + assert len(result) == 1, "Expected 1 document with status not in [active, pending]" + assert result[0]["name"] == "Bob", "Expected Bob" + + +@pytest.mark.find +@pytest.mark.smoke +def test_find_range_query(collection): + """Test find with range query (combining $gte and $lte).""" + # Arrange - Insert test data + collection.insert_many([ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ]) + + # Act - Find documents where 25 <= age <= 30 + result = list(collection.find({"age": {"$gte": 25, "$lte": 30}})) + + # Assert - Verify results + assert len(result) == 2, "Expected 2 documents with age between 25 and 30" + names = {doc["name"] for doc in result} + assert names == {"Alice", "Bob"}, "Expected Alice and Bob" diff --git a/tests/insert/__init__.py b/tests/insert/__init__.py new file mode 100644 index 0000000..47ae134 --- /dev/null +++ b/tests/insert/__init__.py @@ -0,0 +1 @@ +"""Insert operation tests.""" diff --git a/tests/insert/test_insert_operations.py b/tests/insert/test_insert_operations.py new file mode 100644 index 0000000..659d26b --- /dev/null +++ b/tests/insert/test_insert_operations.py @@ -0,0 +1,145 @@ +""" +Insert operation tests. + +Tests for insertOne, insertMany, and related operations. +""" + +import pytest +from bson import ObjectId + +from tests.common.assertions import assert_count + + +@pytest.mark.insert +@pytest.mark.smoke +def test_insert_one_document(collection): + """Test inserting a single document.""" + # Insert one document + document = {"name": "Alice", "age": 30} + result = collection.insert_one(document) + + # Verify insert was acknowledged + assert result.acknowledged, "Insert should be acknowledged" + assert result.inserted_id is not None, "Should return inserted _id" + assert isinstance(result.inserted_id, ObjectId), "Inserted _id should be ObjectId" + + # Verify document exists in collection + assert_count(collection, {}, 1) + + # Verify document content + found = collection.find_one({"name": "Alice"}) + assert found is not None, "Document should exist" + assert found["name"] == "Alice" + assert found["age"] == 30 + + +@pytest.mark.insert +@pytest.mark.smoke +def test_insert_many_documents(collection): + """Test inserting multiple documents.""" + # Insert multiple documents + documents = [ + {"name": "Alice", "age": 30}, + {"name": "Bob", "age": 25}, + {"name": "Charlie", "age": 35}, + ] + result = collection.insert_many(documents) + + # Verify insert was acknowledged + assert result.acknowledged, "Insert should be acknowledged" + assert len(result.inserted_ids) == 3, "Should return 3 inserted IDs" + + # Verify all IDs are ObjectIds + for inserted_id in result.inserted_ids: + assert isinstance(inserted_id, ObjectId), "Each inserted _id should be ObjectId" + + # Verify all documents exist + assert_count(collection, {}, 3) + + +@pytest.mark.insert +def test_insert_with_custom_id(collection): + """Test inserting document with custom _id.""" + # Insert document with custom _id + custom_id = "custom_123" + document = {"_id": custom_id, "name": "Alice"} + result = collection.insert_one(document) + + # Verify custom _id is used + assert result.inserted_id == custom_id, "Should use custom _id" + + # Verify document can be retrieved by custom _id + found = collection.find_one({"_id": custom_id}) + assert found is not None, "Document should exist" + assert found["name"] == "Alice" + + +@pytest.mark.insert +def test_insert_duplicate_id_fails(collection): + """Test that inserting duplicate _id raises error.""" + # Insert first document + document = {"_id": "duplicate_id", "name": "Alice"} + collection.insert_one(document) + + # Try to insert document with same _id + duplicate = {"_id": "duplicate_id", "name": "Bob"} + + # Should raise exception + with pytest.raises(Exception): # DuplicateKeyError or similar + collection.insert_one(duplicate) + + # Verify only first document exists + assert_count(collection, {}, 1) + found = collection.find_one({"_id": "duplicate_id"}) + assert found["name"] == "Alice", "Should have first document" + + +@pytest.mark.insert +def test_insert_nested_document(collection): + """Test inserting document with nested structure.""" + # Insert document with nested fields + document = { + "name": "Alice", + "profile": {"age": 30, "address": {"city": "NYC", "country": "USA"}}, + } + result = collection.insert_one(document) + + # Verify insert + assert result.inserted_id is not None + + # Verify nested structure is preserved + found = collection.find_one({"name": "Alice"}) + assert found["profile"]["age"] == 30 + assert found["profile"]["address"]["city"] == "NYC" + + +@pytest.mark.insert +def test_insert_array_field(collection): + """Test inserting document with array fields.""" + # Insert document with array + document = {"name": "Alice", "tags": ["python", "mongodb", "testing"], "scores": [95, 87, 92]} + result = collection.insert_one(document) + + # Verify insert + assert result.inserted_id is not None + + # Verify array fields are preserved + found = collection.find_one({"name": "Alice"}) + assert found["tags"] == ["python", "mongodb", "testing"] + assert found["scores"] == [95, 87, 92] + + +@pytest.mark.insert +def test_insert_empty_document(collection): + """Test inserting an empty document.""" + # Insert empty document + result = collection.insert_one({}) + + # Verify insert was successful + assert result.inserted_id is not None + + # Verify document exists (only has _id) + found = collection.find_one({"_id": result.inserted_id}) + assert found is not None + assert len(found) == 1 # Only _id field + assert "_id" in found