diff --git a/.gitignore b/.gitignore index 943314f5..0259c573 100644 --- a/.gitignore +++ b/.gitignore @@ -182,4 +182,5 @@ cython_debug/ .DS_Store .vscode/ src/assetopsbench/sample_data/bulk_docs.json -benchmark/cods_track1/.env.local \ No newline at end of file +benchmark/cods_track1/.env.local + diff --git a/benchmark/entrypoint.sh b/benchmark/entrypoint.sh old mode 100644 new mode 100755 diff --git a/src/assetopsbench/core/validator.py b/src/assetopsbench/core/validator.py index 5b13632f..b4e521ce 100644 --- a/src/assetopsbench/core/validator.py +++ b/src/assetopsbench/core/validator.py @@ -4,7 +4,7 @@ from typing import List, Iterator, Dict, Any from pydantic import ValidationError -from scenarios import Scenario +from assetopsbench.core.scenarios import Scenario def read_json_file(file_path: pathlib.Path) -> List[Dict[str, Any]]: """Read and parse JSON file, handling both arrays and single objects.""" diff --git a/tests/Dockerfile.minimal b/tests/Dockerfile.minimal new file mode 100644 index 00000000..25a94a42 --- /dev/null +++ b/tests/Dockerfile.minimal @@ -0,0 +1,26 @@ +# Minimal Dockerfile for running AssetOpsBench tests +FROM python:3.12-slim + +# Set working directory +WORKDIR /app + +# Install minimal system dependencies +RUN apt-get update && apt-get install -y \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Copy and install minimal test requirements +COPY tests/test_requirements.txt /tmp/test_requirements.txt +RUN pip install --no-cache-dir -r /tmp/test_requirements.txt + +# Copy the entire project +COPY . /app/ + +# Set Python path to include src directory +ENV PYTHONPATH=/app/src:$PYTHONPATH + +# Make test scripts executable +RUN chmod +x tests/run_tests.py tests/run_all_tests.py + +# Default command: run all tests +CMD ["python", "tests/run_all_tests.py"] diff --git a/tests/Dockerfile.simple b/tests/Dockerfile.simple new file mode 100644 index 00000000..afdf17d3 --- /dev/null +++ b/tests/Dockerfile.simple @@ -0,0 +1,25 @@ +# Simple Dockerfile for running AssetOpsBench tests +FROM python:3.12-slim + +# Set working directory +WORKDIR /app + +# Install minimal system dependencies +RUN apt-get update && apt-get install -y \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Install only the essential Python dependencies for testing +RUN pip install --no-cache-dir pydantic pathlib + +# Copy the entire project +COPY . /app/ + +# Set Python path to include src directory +ENV PYTHONPATH=/app/src:$PYTHONPATH + +# Make test scripts executable +RUN chmod +x tests/run_tests.py tests/run_all_tests.py + +# Default command: run all tests +CMD ["python", "tests/run_all_tests.py"] diff --git a/tests/Dockerfile.test b/tests/Dockerfile.test new file mode 100644 index 00000000..65a16d9b --- /dev/null +++ b/tests/Dockerfile.test @@ -0,0 +1,32 @@ +# Dockerfile for running AssetOpsBench tests +FROM python:3.12-slim + +# Set working directory +WORKDIR /app + +# Install system dependencies including build tools +RUN apt-get update && apt-get install -y \ + git \ + curl \ + build-essential \ + g++ \ + gcc \ + make \ + python3-dev \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements and install Python dependencies +COPY benchmark/basic_requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +# Copy the entire project +COPY . /app/ + +# Set Python path to include src directory +ENV PYTHONPATH=/app/src:$PYTHONPATH + +# Make test scripts executable +RUN chmod +x tests/run_tests.py tests/run_all_tests.py + +# Default command: run all tests +CMD ["python", "tests/run_all_tests.py"] diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..c4d5e2ff --- /dev/null +++ b/tests/README.md @@ -0,0 +1,303 @@ +# AssetOpsBench Tests + +This directory contains comprehensive unit tests for the AssetOpsBench scenario validation system, as requested in [GitHub issue #30](https://github.com/IBM/AssetOpsBench/issues/30). + +## Test Structure + +- `test_scenario_validation.py` - Core scenario validation tests +- `test_real_scenarios.py` - Tests for actual scenario files +- `run_tests.py` - Basic test runner +- `run_all_tests.py` - Comprehensive test runner + +## Running Tests + +### Local Execution + +#### Option 1: Using Conda (Recommended for macOS) + +1. **Install Conda via Homebrew:** + + ```bash + # Install Homebrew if not already installed + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + + # Install conda via Homebrew + brew install --cask miniconda + + # Add conda to your PATH (add to ~/.zshrc or ~/.bash_profile) + echo 'export PATH="/opt/homebrew/Caskroom/miniconda/base/bin:$PATH"' >> ~/.zshrc + source ~/.zshrc + + # Initialize conda + conda init zsh # or bash if using bash + ``` + +2. **Create Conda Environment:** + + ```bash + # Ensure you're in the AssetOpsBench root directory + cd /path/to/AssetOpsBench + + # Create conda environment with Python 3.12 + conda create -n assetopsbench python=3.12 -y + conda activate assetopsbench + + # Install dependencies + pip install -r benchmark/basic_requirements.txt + ``` + +3. **Run Tests:** + + ```bash + # Activate environment (if not already active) + conda activate assetopsbench + + # Run all tests + python tests/run_all_tests.py + + # Or run quick test first + python tests/run_local_tests.py + ``` + +#### Option 2: Using Python Virtual Environment + +1. **Setup Virtual Environment:** + + ```bash + # Ensure you're in the AssetOpsBench root directory + cd /path/to/AssetOpsBench + + # Create virtual environment + python3 -m venv venv + + # Activate virtual environment + source venv/bin/activate # On macOS/Linux + # or + venv\Scripts\activate # On Windows + + # Install dependencies + pip install -r benchmark/basic_requirements.txt + ``` + +2. **Run Tests:** + + ```bash + # Ensure virtual environment is activated + source venv/bin/activate + + # Run all tests + python tests/run_all_tests.py + + # Or run quick test first + python tests/run_local_tests.py + ``` + +#### Running Specific Test Suites + +```bash +# Scenario validation tests only +python tests/run_all_tests.py scenario_validation + +# Real scenario files tests only +python tests/run_all_tests.py real_scenarios + +# Individual test files +python -m unittest tests.test_scenario_validation -v +python -m unittest tests.test_real_scenarios -v + +# Using the test runner script +./tests/test_runner.sh scenario_validation +``` + +### Docker Execution + +#### Option 1: Simple Docker Setup (Recommended - Fast & Reliable) + +```bash +# Build and run tests in one command +docker-compose -f tests/docker-compose.simple.yml up --build + +# Or run specific test suites +docker-compose -f tests/docker-compose.simple.yml run --rm assetopsbench-tests-simple + +# Using Docker directly +docker build -f tests/Dockerfile.simple -t assetopsbench-tests-simple . +docker run --rm assetopsbench-tests-simple +``` + +#### Option 2: Minimal Docker Setup (Fastest - Testing Only) + +```bash +# Build with minimal dependencies +docker build -f tests/Dockerfile.minimal -t assetopsbench-tests-minimal . + +# Run tests +docker run --rm assetopsbench-tests-minimal + +# Or with volume mounting for development +docker run --rm -v $(pwd)/src:/app/src -v $(pwd)/tests:/app/tests assetopsbench-tests-minimal +``` + +#### Option 3: Full Docker Setup (Complete Environment) + +```bash +# Build and run with all dependencies (requires build tools) +docker-compose -f tests/docker-compose.test.yml up --build + +# Or run specific test suites +docker-compose -f tests/docker-compose.test.yml run --rm assetopsbench-tests + +# Using Docker directly +docker build -f tests/Dockerfile.test -t assetopsbench-tests . +docker run --rm assetopsbench-tests +``` + +#### Docker Test Runner Script + +```bash +# Run all tests in Docker +./tests/test_runner.sh --docker + +# Run specific test suite in Docker +./tests/test_runner.sh --docker scenario_validation + +# Run real scenarios tests in Docker +./tests/test_runner.sh --docker real_scenarios +``` + +## Test Coverage + +The tests cover: + +- ✅ Scenario validation functionality +- ✅ FMSR scenario 113 (Evaporator Water side fouling) +- ✅ TSFM scenario 217 (Chiller 9 forecasting) +- ✅ Real scenario files validation +- ✅ Edge cases and error handling +- ✅ JSON/JSONL file parsing +- ✅ Pydantic model validation + +## Expected Output + +When tests pass, you should see: + +``` +🧪 Running All AssetOpsBench Tests... +============================================================ +test_valid_fmsr_scenario (tests.test_scenario_validation.TestScenarioValidation) ... ok +test_valid_tsfm_scenario (tests.test_scenario_validation.TestScenarioValidation) ... ok +... +✅ All tests passed! Scenario validation is working correctly. +``` + +## Troubleshooting + +### Local Environment Issues + +1. **Import Errors:** + + ```bash + # Ensure you're in the AssetOpsBench root directory + cd /path/to/AssetOpsBench + + # For conda users + conda activate assetopsbench + + # For virtual environment users + source venv/bin/activate + ``` + +2. **Missing Dependencies:** + + ```bash + # Install required packages + pip install -r benchmark/basic_requirements.txt + + # Or for minimal testing only + pip install pydantic pathlib + ``` + +3. **Permission Errors (macOS):** + + ```bash + # Make test scripts executable + chmod +x tests/test_runner.sh + chmod +x tests/run_tests.py + chmod +x tests/run_all_tests.py + ``` + +4. **Python Path Issues:** + + ```bash + # Add src to Python path + export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" + + # Or run with explicit path + PYTHONPATH=$(pwd)/src python tests/run_all_tests.py + ``` + +### Docker Issues + +1. **Docker Build Failures:** + + ```bash + # Use the simple Docker setup instead + docker-compose -f tests/docker-compose.simple.yml up --build + + # Or use minimal setup + docker build -f tests/Dockerfile.minimal -t assetopsbench-minimal . + ``` + +2. **Permission Denied Errors:** + + ```bash + # Make sure Docker is running + docker --version + + # Check Docker daemon status + docker info + ``` + +3. **Volume Mount Issues:** + ```bash + # Use absolute paths for volume mounting + docker run --rm -v /absolute/path/to/AssetOpsBench/src:/app/src assetopsbench-minimal + ``` + +### Environment-Specific Issues + +1. **macOS with Apple Silicon (M1/M2):** + + ```bash + # Use conda for better compatibility + brew install --cask miniconda + conda create -n assetopsbench python=3.12 -y + ``` + +2. **Linux/WSL:** + + ```bash + # Use virtual environment + python3 -m venv venv + source venv/bin/activate + ``` + +3. **Windows:** + ```bash + # Use conda or PowerShell + conda create -n assetopsbench python=3.12 -y + conda activate assetopsbench + ``` + +### Quick Verification + +```bash +# Test if everything is set up correctly +python tests/run_local_tests.py + +# Expected output: +# ✅ Dependencies available +# ✅ Scenario 113 validation passed +# ✅ Scenario 217 validation passed +# 🎉 Quick test completed successfully! +``` diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..dbe842ba --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# Tests package for AssetOpsBench diff --git a/tests/docker-compose.simple.yml b/tests/docker-compose.simple.yml new file mode 100644 index 00000000..50f22564 --- /dev/null +++ b/tests/docker-compose.simple.yml @@ -0,0 +1,15 @@ +version: "3.8" + +services: + assetopsbench-tests-simple: + build: + context: .. + dockerfile: tests/Dockerfile.simple + volumes: + # Mount source code for development + - ../src:/app/src + - ../tests:/app/tests + - ../scenarios:/app/scenarios + environment: + - PYTHONPATH=/app/src:/app/tests + command: ["python", "tests/run_all_tests.py"] diff --git a/tests/docker-compose.test.yml b/tests/docker-compose.test.yml new file mode 100644 index 00000000..d0543077 --- /dev/null +++ b/tests/docker-compose.test.yml @@ -0,0 +1,42 @@ +version: "3.8" + +services: + assetopsbench-tests: + platform: linux/amd64 + build: + context: .. + dockerfile: tests/Dockerfile.test + volumes: + # Mount source code for development + - ../src:/app/src + - ../tests:/app/tests + - ../scenarios:/app/scenarios + environment: + - PYTHONPATH=/app/src:/app/tests + command: ["python", "tests/run_all_tests.py"] + + # Optional: Run specific test suites + scenario-validation-tests: + platform: linux/amd64 + build: + context: .. + dockerfile: tests/Dockerfile.test + volumes: + - ../src:/app/src + - ../tests:/app/tests + - ../scenarios:/app/scenarios + environment: + - PYTHONPATH=/app/src:/app/tests + command: ["python", "tests/run_all_tests.py", "scenario_validation"] + + real-scenarios-tests: + build: + context: .. + dockerfile: tests/Dockerfile.test + volumes: + - ../src:/app/src + - ../tests:/app/tests + - ../scenarios:/app/scenarios + environment: + - PYTHONPATH=/app/src:/app/tests + command: ["python", "tests/run_all_tests.py", "real_scenarios"] diff --git a/tests/run_all_tests.py b/tests/run_all_tests.py new file mode 100644 index 00000000..c4c55702 --- /dev/null +++ b/tests/run_all_tests.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +""" +Comprehensive test runner for all AssetOpsBench tests. + +This script runs all unittest tests including scenario validation tests +as requested in GitHub issue #30: +https://github.com/IBM/AssetOpsBench/issues/30 +""" + +import unittest +import sys +import os +from pathlib import Path + +# Add src to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + + +def run_all_tests(): + """Run all available tests.""" + print("🧪 Running All AssetOpsBench Tests...") + print("=" * 60) + + # Discover and run all tests + loader = unittest.TestLoader() + start_dir = os.path.dirname(__file__) + suite = loader.discover(start_dir, pattern='test_*.py') + + runner = unittest.TextTestRunner(verbosity=2) + result = runner.run(suite) + + # Print detailed summary + print("\n" + "=" * 60) + print("📊 COMPREHENSIVE TEST SUMMARY") + print("=" * 60) + print(f"Tests run: {result.testsRun}") + print(f"Failures: {len(result.failures)}") + print(f"Errors: {len(result.errors)}") + print(f"Skipped: {len(result.skipped) if hasattr(result, 'skipped') else 0}") + + success_rate = ((result.testsRun - len(result.failures) - len(result.errors)) / result.testsRun * 100) + print(f"Success rate: {success_rate:.1f}%") + + if result.failures: + print(f"\n❌ FAILURES ({len(result.failures)}):") + for i, (test, traceback) in enumerate(result.failures, 1): + print(f" {i}. {test}") + print(f" {traceback.split('AssertionError:')[-1].strip()}") + + if result.errors: + print(f"\n💥 ERRORS ({len(result.errors)}):") + for i, (test, traceback) in enumerate(result.errors, 1): + print(f" {i}. {test}") + print(f" {traceback.split('Error:')[-1].strip()}") + + if result.wasSuccessful(): + print("\n✅ All tests passed! Scenario validation is working correctly.") + print("\n📋 Test Coverage:") + print(" ✅ Scenario validation functionality") + print(" ✅ FMSR scenario 113 (Evaporator Water side fouling)") + print(" ✅ TSFM scenario 217 (Chiller 9 forecasting)") + print(" ✅ Real scenario files validation") + print(" ✅ Edge cases and error handling") + return True + else: + print(f"\n❌ {len(result.failures + result.errors)} test(s) failed") + print("\n🔧 Next Steps:") + print(" 1. Review the failed tests above") + print(" 2. Check the scenario validation code") + print(" 3. Ensure all scenario files are properly formatted") + return False + + +def run_specific_test_suite(test_name): + """Run a specific test suite.""" + print(f"🧪 Running {test_name} Tests...") + print("=" * 60) + + # Import and run specific test + if test_name == "scenario_validation": + from test_scenario_validation import TestScenarioValidation, TestScenarioExamples + suite = unittest.TestLoader().loadTestsFromTestCase(TestScenarioValidation) + suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestScenarioExamples)) + elif test_name == "real_scenarios": + from test_real_scenarios import TestRealScenarioFiles + suite = unittest.TestLoader().loadTestsFromTestCase(TestRealScenarioFiles) + else: + print(f"Unknown test suite: {test_name}") + return False + + runner = unittest.TextTestRunner(verbosity=2) + result = runner.run(suite) + + print(f"\n{'✅' if result.wasSuccessful() else '❌'} {test_name} tests completed") + return result.wasSuccessful() + + +if __name__ == '__main__': + if len(sys.argv) > 1: + # Run specific test suite + test_suite = sys.argv[1] + success = run_specific_test_suite(test_suite) + else: + # Run all tests + success = run_all_tests() + + sys.exit(0 if success else 1) diff --git a/tests/run_local_tests.py b/tests/run_local_tests.py new file mode 100644 index 00000000..7d372e97 --- /dev/null +++ b/tests/run_local_tests.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +Quick local test runner for AssetOpsBench scenario validation. + +This script is designed for quick local testing without Docker. +""" + +import sys +import os +from pathlib import Path + +def setup_environment(): + """Set up the environment for running tests.""" + # Get the project root directory + project_root = Path(__file__).parent.parent + src_path = project_root / "src" + + # Add src to Python path + if str(src_path) not in sys.path: + sys.path.insert(0, str(src_path)) + + # Change to project root directory + os.chdir(project_root) + + print(f"📁 Project root: {project_root}") + print(f"🐍 Python path includes: {src_path}") + +def check_dependencies(): + """Check if required dependencies are available.""" + try: + from assetopsbench.core.validator import validate_scenario + from assetopsbench.core.scenarios import Scenario + print("✅ Dependencies available") + return True + except ImportError as e: + print(f"❌ Missing dependencies: {e}") + print("💡 Try: pip install -r benchmark/basic_requirements.txt") + return False + +def run_quick_test(): + """Run a quick test to verify everything works.""" + print("\n🧪 Running Quick Test...") + + try: + from assetopsbench.core.validator import validate_scenario + from assetopsbench.core.scenarios import Scenario + + # Test scenario 113 + scenario_113 = { + "id": "113", + "type": "FMSR", + "text": "If Evaporator Water side fouling occurs for Chiller 6, which sensor is most relevant for monitoring this specific failure?", + "deterministic": False + } + + errors = validate_scenario(scenario_113) + if len(errors) == 0: + print("✅ Scenario 113 validation passed") + else: + print(f"❌ Scenario 113 validation failed: {errors}") + return False + + # Test scenario 217 + scenario_217 = { + "id": "217", + "type": "TSFM", + "text": "Forecast 'Chiller 9 Condenser Water Flow' using data in 'chiller9_annotated_small_test.csv'. Use parameter 'Timestamp' as a timestamp.", + "category": "Inference Query" + } + + errors = validate_scenario(scenario_217) + if len(errors) == 0: + print("✅ Scenario 217 validation passed") + else: + print(f"❌ Scenario 217 validation failed: {errors}") + return False + + print("🎉 Quick test completed successfully!") + return True + + except Exception as e: + print(f"💥 Quick test failed: {e}") + return False + +def main(): + """Main function.""" + print("🚀 AssetOpsBench Local Test Runner") + print("=" * 50) + + # Setup environment + setup_environment() + + # Check dependencies + if not check_dependencies(): + sys.exit(1) + + # Run quick test + if run_quick_test(): + print("\n📋 Next steps:") + print(" 1. Run full tests: python tests/run_all_tests.py") + print(" 2. Run with Docker: docker-compose -f tests/docker-compose.test.yml up") + print(" 3. Run specific tests: python -m unittest tests.test_scenario_validation -v") + sys.exit(0) + else: + print("\n🔧 Troubleshooting:") + print(" 1. Check that you're in the AssetOpsBench root directory") + print(" 2. Ensure src/ directory contains the validation code") + print(" 3. Install dependencies: pip install -r benchmark/basic_requirements.txt") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/tests/run_tests.py b/tests/run_tests.py new file mode 100644 index 00000000..e63a86f6 --- /dev/null +++ b/tests/run_tests.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +""" +Test runner for AssetOpsBench scenario validation tests. + +This script runs the unittest tests as requested in GitHub issue #30: +https://github.com/IBM/AssetOpsBench/issues/30 +""" + +import unittest +import sys +import os + +# Add src to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + +def run_scenario_validation_tests(): + """Run scenario validation tests.""" + print("🧪 Running Scenario Validation Tests...") + print("=" * 60) + + # Discover and run tests + loader = unittest.TestLoader() + start_dir = os.path.dirname(__file__) + suite = loader.discover(start_dir, pattern='test_scenario_validation.py') + + runner = unittest.TextTestRunner(verbosity=2) + result = runner.run(suite) + + # Print summary + print("\n" + "=" * 60) + print("📊 TEST SUMMARY") + print("=" * 60) + print(f"Tests run: {result.testsRun}") + print(f"Failures: {len(result.failures)}") + print(f"Errors: {len(result.errors)}") + print(f"Success rate: {((result.testsRun - len(result.failures) - len(result.errors)) / result.testsRun * 100):.1f}%") + + if result.failures: + print("\n❌ FAILURES:") + for test, traceback in result.failures: + print(f" - {test}: {traceback}") + + if result.errors: + print("\n💥 ERRORS:") + for test, traceback in result.errors: + print(f" - {test}: {traceback}") + + if result.wasSuccessful(): + print("\n✅ All tests passed!") + return True + else: + print(f"\n❌ {len(result.failures + result.errors)} test(s) failed") + return False + +if __name__ == '__main__': + success = run_scenario_validation_tests() + sys.exit(0 if success else 1) diff --git a/tests/test_real_scenarios.py b/tests/test_real_scenarios.py new file mode 100644 index 00000000..adc64796 --- /dev/null +++ b/tests/test_real_scenarios.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +""" +Unit tests for real scenario files in AssetOpsBench. + +This module tests the actual scenario files from the scenarios/ directory +using the scenario validation code. +""" + +import unittest +import json +import os +import sys +from pathlib import Path +from typing import List, Dict, Any + +# Add src to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + +from assetopsbench.core.validator import validate_file, validate_scenario +from assetopsbench.core.scenarios import Scenario + + +class TestRealScenarioFiles(unittest.TestCase): + """Test cases for real scenario files in the repository.""" + + def setUp(self): + """Set up test fixtures.""" + self.scenarios_dir = Path(__file__).parent.parent / "scenarios" + self.single_agent_dir = self.scenarios_dir / "single_agent" + self.multi_agent_dir = self.scenarios_dir / "multi_agent" + + # Expected scenario files + self.expected_files = [ + "iot_utterance_meta.json", + "fmsr_utterance.json", + "tsfm_utterance.json", + "wo_utterance.json", + "end2end_utterance.json" + ] + + def test_scenario_files_exist(self): + """Test that expected scenario files exist.""" + for filename in self.expected_files: + if filename == "end2end_utterance.json": + file_path = self.multi_agent_dir / filename + else: + file_path = self.single_agent_dir / filename + + self.assertTrue(file_path.exists(), f"Scenario file {filename} should exist") + + def test_iot_scenarios_validation(self): + """Test validation of IoT scenarios.""" + iot_file = self.single_agent_dir / "iot_utterance_meta.json" + if iot_file.exists(): + errors = validate_file(iot_file) + self.assertEqual(len(errors), 0, f"IoT scenarios should be valid: {errors}") + + # Load and test a few scenarios + with open(iot_file, 'r') as f: + scenarios = json.load(f) + + # Test first scenario + if scenarios: + errors = validate_scenario(scenarios[0]) + self.assertEqual(len(errors), 0, f"First IoT scenario should be valid: {errors}") + + scenario = Scenario(**scenarios[0]) + self.assertEqual(scenario.type, "IoT") + + def test_fmsr_scenarios_validation(self): + """Test validation of FMSR scenarios.""" + fmsr_file = self.single_agent_dir / "fmsr_utterance.json" + if fmsr_file.exists(): + errors = validate_file(fmsr_file) + self.assertEqual(len(errors), 0, f"FMSR scenarios should be valid: {errors}") + + # Load and test scenario 113 specifically + with open(fmsr_file, 'r') as f: + scenarios = json.load(f) + + # Find scenario 113 + scenario_113 = next((s for s in scenarios if s.get('id') == 113), None) + if scenario_113: + errors = validate_scenario(scenario_113) + self.assertEqual(len(errors), 0, f"FMSR scenario 113 should be valid: {errors}") + + scenario = Scenario(**scenario_113) + self.assertEqual(scenario.id, "113") + # Note: scenario_113 might not have a 'type' field, so we check if it exists + if 'type' in scenario_113: + self.assertEqual(scenario.type, "FMSR") + self.assertIn("Evaporator Water side fouling", scenario.text) + + def test_tsfm_scenarios_validation(self): + """Test validation of TSFM scenarios.""" + tsfm_file = self.single_agent_dir / "tsfm_utterance.json" + if tsfm_file.exists(): + errors = validate_file(tsfm_file) + self.assertEqual(len(errors), 0, f"TSFM scenarios should be valid: {errors}") + + # Load and test scenario 217 specifically + with open(tsfm_file, 'r') as f: + scenarios = json.load(f) + + # Find scenario 217 + scenario_217 = next((s for s in scenarios if s.get('id') == 217), None) + if scenario_217: + errors = validate_scenario(scenario_217) + self.assertEqual(len(errors), 0, f"TSFM scenario 217 should be valid: {errors}") + + scenario = Scenario(**scenario_217) + self.assertEqual(scenario.id, "217") + self.assertEqual(scenario.type, "TSFM") + self.assertIn("Forecast", scenario.text) + self.assertIn("chiller9_annotated_small_test.csv", scenario.text) + + def test_workorder_scenarios_validation(self): + """Test validation of WorkOrder scenarios.""" + wo_file = self.single_agent_dir / "wo_utterance.json" + if wo_file.exists(): + errors = validate_file(wo_file) + self.assertEqual(len(errors), 0, f"WorkOrder scenarios should be valid: {errors}") + + # Load and test a few scenarios + with open(wo_file, 'r') as f: + scenarios = json.load(f) + + # Test first scenario + if scenarios: + errors = validate_scenario(scenarios[0]) + self.assertEqual(len(errors), 0, f"First WorkOrder scenario should be valid: {errors}") + + scenario = Scenario(**scenarios[0]) + self.assertIn(scenario.type, ["WorkOrder", "Workorder"]) # Accept both variants + + def test_end2end_scenarios_validation(self): + """Test validation of end-to-end scenarios.""" + end2end_file = self.multi_agent_dir / "end2end_utterance.json" + if end2end_file.exists(): + errors = validate_file(end2end_file) + self.assertEqual(len(errors), 0, f"End-to-end scenarios should be valid: {errors}") + + # Load and test a few scenarios + with open(end2end_file, 'r') as f: + scenarios = json.load(f) + + # Test first scenario + if scenarios: + errors = validate_scenario(scenarios[0]) + self.assertEqual(len(errors), 0, f"First end-to-end scenario should be valid: {errors}") + + def test_all_utterances_jsonl_validation(self): + """Test validation of all_utterance.jsonl file.""" + all_utterances_file = self.scenarios_dir / "all_utterance.jsonl" + if all_utterances_file.exists(): + errors = validate_file(all_utterances_file) + self.assertEqual(len(errors), 0, f"All utterances JSONL should be valid: {errors}") + + # Test a few scenarios from the file + with open(all_utterances_file, 'r') as f: + lines = f.readlines() + + # Test first few scenarios + for i, line in enumerate(lines[:5]): # Test first 5 scenarios + if line.strip(): + scenario_data = json.loads(line) + errors = validate_scenario(scenario_data) + self.assertEqual(len(errors), 0, f"Scenario {i+1} in all_utterance.jsonl should be valid: {errors}") + + def test_scenario_types_and_categories(self): + """Test that scenarios have valid types and categories.""" + scenario_files = [ + self.single_agent_dir / "iot_utterance_meta.json", + self.single_agent_dir / "fmsr_utterance.json", + self.single_agent_dir / "tsfm_utterance.json", + self.single_agent_dir / "wo_utterance.json", + self.multi_agent_dir / "end2end_utterance.json" + ] + + expected_types = {"IoT", "FMSR", "TSFM", "WorkOrder", "Workorder", ""} # Include empty string + + for file_path in scenario_files: + if file_path.exists(): + with open(file_path, 'r') as f: + scenarios = json.load(f) + + for scenario_data in scenarios: + if 'type' in scenario_data: + self.assertIn(scenario_data['type'], expected_types, + f"Unknown scenario type: '{scenario_data['type']}'") + + # Test scenario creation + errors = validate_scenario(scenario_data) + self.assertEqual(len(errors), 0, + f"Scenario {scenario_data.get('id', 'unknown')} should be valid: {errors}") + + def test_scenario_characteristic_forms(self): + """Test that scenarios have meaningful characteristic_forms.""" + scenario_files = [ + self.single_agent_dir / "iot_utterance_meta.json", + self.single_agent_dir / "fmsr_utterance.json", + self.single_agent_dir / "tsfm_utterance.json" + ] + + for file_path in scenario_files: + if file_path.exists(): + with open(file_path, 'r') as f: + scenarios = json.load(f) + + for scenario_data in scenarios: + if 'characteristic_form' in scenario_data: + char_form = scenario_data['characteristic_form'] + self.assertIsInstance(char_form, str, "characteristic_form should be a string") + self.assertGreater(len(char_form), 10, "characteristic_form should be meaningful") + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/test_requirements.txt b/tests/test_requirements.txt new file mode 100644 index 00000000..6bb0e4e4 --- /dev/null +++ b/tests/test_requirements.txt @@ -0,0 +1,5 @@ +# Minimal requirements for AssetOpsBench testing +pydantic>=2.0.0 +pathlib +python-dotenv +requests diff --git a/tests/test_runner.sh b/tests/test_runner.sh new file mode 100755 index 00000000..e0c438a2 --- /dev/null +++ b/tests/test_runner.sh @@ -0,0 +1,233 @@ +#!/bin/bash + +# AssetOpsBench Test Runner Script +# Supports both local and Docker execution + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Function to print colored output +print_status() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Function to check if running in Docker +is_docker() { + [ -f /.dockerenv ] || [ -n "$DOCKER_CONTAINER" ] +} + +# Function to setup environment +setup_environment() { + print_status "Setting up environment..." + + # Get project root + PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + cd "$PROJECT_ROOT" + + print_status "Project root: $PROJECT_ROOT" + + # Check if src directory exists + if [ ! -d "src" ]; then + print_error "src directory not found. Are you in the AssetOpsBench root directory?" + exit 1 + fi + + # Check if scenarios directory exists + if [ ! -d "scenarios" ]; then + print_warning "scenarios directory not found. Some tests may fail." + fi + + print_success "Environment setup complete" +} + +# Function to run local tests +run_local_tests() { + print_status "Running tests locally..." + + # Check Python availability + if ! command -v python3 &> /dev/null; then + print_error "Python 3 not found. Please install Python 3." + exit 1 + fi + + # Check if virtual environment should be activated + if [ -f "venv/bin/activate" ]; then + print_status "Activating virtual environment..." + source venv/bin/activate + fi + + # Run quick test first + print_status "Running quick validation test..." + python3 tests/run_local_tests.py + + if [ $? -eq 0 ]; then + print_success "Quick test passed!" + + # Run full test suite + print_status "Running full test suite..." + python3 tests/run_all_tests.py + + if [ $? -eq 0 ]; then + print_success "All tests passed!" + else + print_error "Some tests failed!" + exit 1 + fi + else + print_error "Quick test failed. Please check your setup." + exit 1 + fi +} + +# Function to run Docker tests +run_docker_tests() { + print_status "Running tests in Docker..." + + # Check if Docker is available + if ! command -v docker &> /dev/null; then + print_error "Docker not found. Please install Docker." + exit 1 + fi + + # Check if docker-compose is available + if ! command -v docker-compose &> /dev/null; then + print_error "docker-compose not found. Please install docker-compose." + exit 1 + fi + + # Build and run tests + print_status "Building Docker test image..." + docker-compose -f tests/docker-compose.test.yml build + + if [ $? -eq 0 ]; then + print_success "Docker image built successfully!" + + print_status "Running tests in Docker container..." + docker-compose -f tests/docker-compose.test.yml run --rm assetopsbench-tests + + if [ $? -eq 0 ]; then + print_success "Docker tests completed successfully!" + else + print_error "Docker tests failed!" + exit 1 + fi + else + print_error "Failed to build Docker image!" + exit 1 + fi +} + +# Function to run specific test suite +run_specific_tests() { + local test_suite="$1" + print_status "Running specific test suite: $test_suite" + + if is_docker; then + python3 tests/run_all_tests.py "$test_suite" + else + # Check if we should use Docker + if [ "$USE_DOCKER" = "true" ]; then + docker-compose -f tests/docker-compose.test.yml run --rm assetopsbench-tests python3 tests/run_all_tests.py "$test_suite" + else + python3 tests/run_all_tests.py "$test_suite" + fi + fi +} + +# Function to show help +show_help() { + echo "AssetOpsBench Test Runner" + echo "" + echo "Usage: $0 [OPTIONS] [TEST_SUITE]" + echo "" + echo "Options:" + echo " -d, --docker Run tests in Docker container" + echo " -l, --local Run tests locally (default)" + echo " -h, --help Show this help message" + echo "" + echo "Test Suites:" + echo " scenario_validation Run scenario validation tests only" + echo " real_scenarios Run real scenario file tests only" + echo " all Run all tests (default)" + echo "" + echo "Examples:" + echo " $0 # Run all tests locally" + echo " $0 --docker # Run all tests in Docker" + echo " $0 scenario_validation # Run scenario validation tests locally" + echo " $0 --docker real_scenarios # Run real scenarios tests in Docker" + echo "" +} + +# Main function +main() { + local use_docker=false + local test_suite="all" + + # Parse arguments + while [[ $# -gt 0 ]]; do + case $1 in + -d|--docker) + use_docker=true + shift + ;; + -l|--local) + use_docker=false + shift + ;; + -h|--help) + show_help + exit 0 + ;; + scenario_validation|real_scenarios|all) + test_suite="$1" + shift + ;; + *) + print_error "Unknown option: $1" + show_help + exit 1 + ;; + esac + done + + # Setup environment + setup_environment + + # Run tests based on mode + if [ "$use_docker" = true ]; then + export USE_DOCKER=true + if [ "$test_suite" = "all" ]; then + run_docker_tests + else + run_specific_tests "$test_suite" + fi + else + export USE_DOCKER=false + if [ "$test_suite" = "all" ]; then + run_local_tests + else + run_specific_tests "$test_suite" + fi + fi +} + +# Run main function with all arguments +main "$@" diff --git a/tests/test_scenario_validation.py b/tests/test_scenario_validation.py new file mode 100644 index 00000000..2cbaede4 --- /dev/null +++ b/tests/test_scenario_validation.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +""" +Unit tests for scenario validation code in AssetOpsBench. + +This module tests the scenario validation functionality as requested in GitHub issue #30: +https://github.com/IBM/AssetOpsBench/issues/30 + +Tests the validator.py code using Python's built-in unittest package. +""" + +import unittest +import tempfile +import json +import os +from pathlib import Path +from typing import Dict, Any + +# Add src to path for imports +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +from assetopsbench.core.validator import ( + validate_scenario, + validate_file, + find_json_files, +) +from assetopsbench.core.scenarios import Scenario + + +class TestScenarioValidation(unittest.TestCase): + """Test cases for scenario validation functionality.""" + + def setUp(self): + """Set up test fixtures.""" + self.valid_scenario_data = { + "id": "113", + "type": "FMSR", + "text": "If Evaporator Water side fouling occurs for Chiller 6, which sensor is most relevant for monitoring this specific failure?", + "category": "Knowledge Query", + "characteristic_form": "the answer should contain one of more sensors of Chiller 6. The sensors of Chiller 6 need to be from the list [Chiller 6 Chiller % Loaded, Chiller 6 Chiller Efficiency, Chiller 6 Condenser Water Flow, Chiller 6 Condenser Water Return To Tower Temperature, Chiller 6 Liquid Refrigerant Evaporator Temperature, Chiller 6 Power Input, Chiller 6 Return Temperature, Chiller 6 Schedule, Chiller 6 Supply Temperature, Chiller 6 Tonnage.]", + "deterministic": False, + } + + self.tsfm_scenario_data = { + "id": "217", + "type": "TSFM", + "text": "Forecast 'Chiller 9 Condenser Water Flow' using data in 'chiller9_annotated_small_test.csv'. Use parameter 'Timestamp' as a timestamp.", + "category": "Inference Query", + "characteristic_form": "The expected response should be: Forecasting results of 'Chiller 9 Condenser Water Flow' using data in 'chiller9_annotated_small_test.csv' are stored in json file", + } + + def test_valid_fmsr_scenario(self): + """Test validation of FMSR scenario 113.""" + errors = validate_scenario(self.valid_scenario_data) + self.assertEqual( + len(errors), 0, f"Validation should pass but got errors: {errors}" + ) + + # Test that it creates a valid Scenario object + scenario = Scenario(**self.valid_scenario_data) + self.assertEqual(scenario.id, "113") + self.assertEqual(scenario.type, "FMSR") + self.assertIn("Evaporator Water side fouling", scenario.text) + + def test_valid_tsfm_scenario(self): + """Test validation of TSFM scenario 217.""" + errors = validate_scenario(self.tsfm_scenario_data) + self.assertEqual( + len(errors), 0, f"Validation should pass but got errors: {errors}" + ) + + # Test that it creates a valid Scenario object + scenario = Scenario(**self.tsfm_scenario_data) + self.assertEqual(scenario.id, "217") + self.assertEqual(scenario.type, "TSFM") + self.assertIn("Forecast", scenario.text) + + def test_invalid_scenario_missing_required_fields(self): + """Test validation fails for missing required fields.""" + invalid_data = { + "id": "999" + # Missing required 'text' field + } + errors = validate_scenario(invalid_data) + self.assertGreater( + len(errors), 0, "Should have validation errors for missing required fields" + ) + self.assertTrue(any("text" in error.lower() for error in errors)) + + def test_invalid_scenario_empty_text(self): + """Test validation fails for empty text field.""" + invalid_data = { + "id": "999", + "text": " ", # Whitespace-only text should be stripped to empty + "type": "FMSR", + } + errors = validate_scenario(invalid_data) + # Note: The current model allows empty text, so this test checks the current behavior + # If we want to enforce non-empty text, we'd need to add validation to the Scenario model + self.assertEqual(len(errors), 0, "Current model allows empty text after stripping whitespace") + + def test_scenario_with_optional_fields(self): + """Test validation with optional fields.""" + data_with_optionals = self.valid_scenario_data.copy() + data_with_optionals.update( + { + "uuid": "test-uuid-123", + "expected_result": { + "sensor": "Chiller 6 Liquid Refrigerant Evaporator Temperature" + }, + "data": {"site": "MAIN", "equipment": "Chiller 6"}, + "source": "test_dataset.json", + } + ) + + errors = validate_scenario(data_with_optionals) + self.assertEqual( + len(errors), 0, f"Validation should pass with optional fields: {errors}" + ) + + scenario = Scenario(**data_with_optionals) + self.assertEqual(scenario.uuid, "test-uuid-123") + self.assertEqual(scenario.data["site"], "MAIN") + + def test_scenario_id_type_conversion(self): + """Test that integer IDs are converted to strings.""" + data_with_int_id = self.valid_scenario_data.copy() + data_with_int_id["id"] = 113 # Integer ID + + # The validator should convert this to string + errors = validate_scenario(data_with_int_id) + self.assertEqual(len(errors), 0, "Should handle integer ID conversion") + + def test_validate_json_file(self): + """Test validation of JSON files.""" + # Create temporary JSON file with valid scenarios + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump([self.valid_scenario_data, self.tsfm_scenario_data], f) + temp_file = Path(f.name) + + try: + errors = validate_file(temp_file) + self.assertEqual( + len(errors), 0, f"JSON file validation should pass: {errors}" + ) + finally: + temp_file.unlink() + + def test_validate_jsonl_file(self): + """Test validation of JSONL files.""" + # Create temporary JSONL file with valid scenarios + with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f: + f.write(json.dumps(self.valid_scenario_data) + "\n") + f.write(json.dumps(self.tsfm_scenario_data) + "\n") + temp_file = Path(f.name) + + try: + errors = validate_file(temp_file) + self.assertEqual( + len(errors), 0, f"JSONL file validation should pass: {errors}" + ) + finally: + temp_file.unlink() + + def test_validate_invalid_json_file(self): + """Test validation fails for invalid JSON files.""" + # Create temporary JSON file with invalid scenario + invalid_scenario = { + "id": "999" + # Missing required 'text' field + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump([invalid_scenario], f) + temp_file = Path(f.name) + + try: + errors = validate_file(temp_file) + self.assertGreater( + len(errors), 0, "Should have validation errors for invalid JSON file" + ) + finally: + temp_file.unlink() + + def test_find_json_files(self): + """Test finding JSON and JSONL files in directory.""" + # Create temporary directory with various files + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create test files + (temp_path / "test.json").write_text("[]") + (temp_path / "test.jsonl").write_text("") + (temp_path / "test.txt").write_text("not json") + (temp_path / "test.py").write_text('print("hello")') + + # Find JSON files + json_files = find_json_files(temp_path) + + # Should find 2 files (test.json and test.jsonl) + self.assertEqual(len(json_files), 2) + file_names = [f.name for f in json_files] + self.assertIn("test.json", file_names) + self.assertIn("test.jsonl", file_names) + + def test_scenario_validation_edge_cases(self): + """Test edge cases in scenario validation.""" + # Test with minimal required fields only + minimal_scenario = {"id": "001", "text": "What IoT sites are available?"} + errors = validate_scenario(minimal_scenario) + self.assertEqual(len(errors), 0, "Minimal scenario should be valid") + + # Test with extra fields (should be ignored due to extra="ignore") + extra_fields_scenario = self.valid_scenario_data.copy() + extra_fields_scenario["extra_field"] = "should be ignored" + extra_fields_scenario["another_field"] = 123 + + errors = validate_scenario(extra_fields_scenario) + self.assertEqual(len(errors), 0, "Extra fields should be ignored") + + def test_characteristic_form_validation(self): + """Test validation of characteristic_form field.""" + # Test with long characteristic_form + long_characteristic = "x" * 10000 # Very long string + scenario_with_long_char = self.valid_scenario_data.copy() + scenario_with_long_char["characteristic_form"] = long_characteristic + + errors = validate_scenario(scenario_with_long_char) + self.assertEqual(len(errors), 0, "Long characteristic_form should be valid") + + # Test with None characteristic_form + scenario_with_none_char = self.valid_scenario_data.copy() + scenario_with_none_char["characteristic_form"] = None + + errors = validate_scenario(scenario_with_none_char) + self.assertEqual(len(errors), 0, "None characteristic_form should be valid") + + def test_deterministic_field_validation(self): + """Test validation of deterministic field.""" + # Test with deterministic=True + deterministic_scenario = self.valid_scenario_data.copy() + deterministic_scenario["deterministic"] = True + deterministic_scenario["expected_result"] = { + "answer": "Chiller 6 Liquid Refrigerant Evaporator Temperature" + } + + errors = validate_scenario(deterministic_scenario) + self.assertEqual(len(errors), 0, "Deterministic scenario should be valid") + + # Test with deterministic=False + non_deterministic_scenario = self.valid_scenario_data.copy() + non_deterministic_scenario["deterministic"] = False + + errors = validate_scenario(non_deterministic_scenario) + self.assertEqual(len(errors), 0, "Non-deterministic scenario should be valid") + + +class TestScenarioExamples(unittest.TestCase): + """Test cases using the actual example scenarios from the issue.""" + + def test_fmsr_scenario_113(self): + """Test the specific FMSR scenario 113 mentioned in the issue.""" + scenario_113 = { + "id": "113", + "type": "FMSR", + "deterministic": False, + "characteristic_form": "the answer should contain one of more sensors of Chiller 6. The sensors of Chiller 6 need to be from the list [Chiller 6 Chiller % Loaded, Chiller 6 Chiller Efficiency, Chiller 6 Condenser Water Flow, Chiller 6 Condenser Water Return To Tower Temperature, Chiller 6 Liquid Refrigerant Evaporator Temperature, Chiller 6 Power Input, Chiller 6 Return Temperature, Chiller 6 Schedule, Chiller 6 Supply Temperature, Chiller 6 Tonnage.] ", + "text": " If Evaporator Water side fouling occurs for Chiller 6, which sensor is most relevant for monitoring this specific failure?", + } + + errors = validate_scenario(scenario_113) + self.assertEqual(len(errors), 0, f"Scenario 113 should be valid: {errors}") + + # Validate the scenario object creation + scenario = Scenario(**scenario_113) + self.assertEqual(scenario.id, "113") + self.assertEqual(scenario.type, "FMSR") + self.assertFalse(scenario.deterministic) + self.assertIn("Evaporator Water side fouling", scenario.text) + self.assertIn("Chiller 6", scenario.characteristic_form) + + def test_tsfm_scenario_217(self): + """Test the specific TSFM scenario 217 mentioned in the issue.""" + scenario_217 = { + "id": "217", + "type": "TSFM", + "text": "Forecast 'Chiller 9 Condenser Water Flow' using data in 'chiller9_annotated_small_test.csv'. Use parameter 'Timestamp' as a timestamp.", + "category": "Inference Query", + "characteristic_form": "The expected response should be: Forecasting results of 'Chiller 9 Condenser Water Flow' using data in 'chiller9_annotated_small_test.csv' are stored in json file", + } + + errors = validate_scenario(scenario_217) + self.assertEqual(len(errors), 0, f"Scenario 217 should be valid: {errors}") + + # Validate the scenario object creation + scenario = Scenario(**scenario_217) + self.assertEqual(scenario.id, "217") + self.assertEqual(scenario.type, "TSFM") + self.assertEqual(scenario.category, "Inference Query") + self.assertIn("Forecast", scenario.text) + self.assertIn("chiller9_annotated_small_test.csv", scenario.text) + self.assertIn("Timestamp", scenario.text) + + +if __name__ == "__main__": + # Run the tests + unittest.main(verbosity=2)