diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..f737ac9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,53 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true + +# Python files +[*.py] +indent_style = space +indent_size = 4 +max_line_length = 120 + +# Markdown files +[*.md] +trim_trailing_whitespace = false +max_line_length = off + +# YAML files +[*.{yml,yaml}] +indent_style = space +indent_size = 2 + +# JSON files +[*.json] +indent_style = space +indent_size = 2 + +# TOML files +[*.toml] +indent_style = space +indent_size = 4 + +# reStructuredText files +[*.rst] +indent_style = space +indent_size = 3 +max_line_length = off + +# Makefile +[Makefile] +indent_style = tab + +# Shell scripts +[*.sh] +indent_style = space +indent_size = 2 + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index da82bd1..a87a67e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,14 +21,17 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" - name: Set up Python 3.10 - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install dependencies run: | - python -m pip install --upgrade pip - python -m pip install -e .[test] + uv pip install --system -e .[test,html] - name: Test with unittest run: | coverage run -m unittest discover tests \ No newline at end of file diff --git a/.gitignore b/.gitignore index dd12a8c..1753c21 100644 --- a/.gitignore +++ b/.gitignore @@ -134,12 +134,17 @@ celerybeat.pid # Environments .env .venv +.venv/ env/ venv/ ENV/ env.bak/ venv.bak/ +# uv +uv.lock +.python-version + # Spyder project settings .spyderproject .spyproject diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..dba1cad --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,52 @@ +# Pre-commit hooks for SAES +# Install with: pre-commit install +# Run manually with: pre-commit run --all-files + +repos: + # General file checks + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-toml + - id: check-json + - id: check-added-large-files + args: ['--maxkb=1000'] + - id: check-merge-conflict + - id: check-case-conflict + - id: mixed-line-ending + args: ['--fix=lf'] + + # Python code formatting + - repo: https://github.com/psf/black + rev: 24.1.1 + hooks: + - id: black + language_version: python3.10 + args: ['--line-length=120'] + + # Import sorting + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + args: ['--profile=black', '--line-length=120'] + + # Linting + - repo: https://github.com/PyCQA/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + args: ['--max-line-length=120', '--extend-ignore=E203,W503'] + additional_dependencies: [flake8-docstrings] + + # Type checking (optional - may need adjustments) + # - repo: https://github.com/pre-commit/mirrors-mypy + # rev: v1.8.0 + # hooks: + # - id: mypy + # additional_dependencies: [types-all] + # args: ['--ignore-missing-imports'] + diff --git a/CHANGELOG.md b/CHANGELOG.md index a143c67..a039bd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,141 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixed +- Corrected 'frtom' typo to 'from' in all plot module docstrings (23 instances across 5 files) + ## [Released] -## [RELEASE_VERSION] - Released on RELEASE_DATE by COMMIT_AUTHOR +## [1.3.6] - 2025-03-18 + +### Added +- HistoPlot visualization for algorithm performance distribution +- Violin plot for enhanced performance distribution visualization + +### Fixed +- Various bug fixes and added code comments for better maintainability + +## [1.3.5] - 2025-03-13 + +### Added +- Anova and T-test tables for parametric statistical analysis +- Extra Friedman test variations (aligned-rank, Quade) for non-parametric analysis +- ML notebook example demonstrating library usage with machine learning algorithms +- Comprehensive tests for new features + +### Changed +- Mean/median now used as estimators of best and second-best performance in LaTeX tables +- Improved documentation across the entire library +- Updated Bayesian notebook with better examples + +### Fixed +- Bug in MeanMedian table show() function + +## [1.3.4] - 2025-03-06 + +### Added +- Frequency graph to the Bayesian posterior plot (Pplot) +- Article references for statistical test implementations + +### Fixed +- Fixed dependency issues (v2) +- Fixed tests to accommodate new changes + +## [1.3.2] - 2025-03-06 + +### Changed +- Updated internal dependencies and configurations + +## [1.3.1] - 2025-03-05 + +### Fixed +- Minor bug fixes and improvements + +## [1.3.0] - 2025-03-05 + +### Added +- Bayesian posterior plot (Pplot) for probabilistic algorithm comparison +- HTML module for generating interactive analysis reports + +### Changed +- Updated multi-objective fronts notebook +- Updated sphinx documentation + +## [1.2.0] - 2025-03-04 + +### Added +- Reference fronts support in 2D and 3D for multi-objective optimization module +- Parallel coordinates visualization for multi-objective analysis +- Fronts notebook with comprehensive examples + +### Changed +- Updated all SAES fstring documentation format + +### Fixed +- Bug fixed in pareto_front.py + +## [1.1.0] - 2025-02-26 + +### Changed +- Updated Sphinx documentation to v1.1.0 +- Updated README.md with improved examples and instructions + +## [1.0.3] - 2025-02-07 + +### Changed +- Documentation improvements and README updates + +## [1.0.2] - 2025-02-06 + +### Changed +- Minor improvements and documentation updates + +## [1.0.1] - 2025-02-06 + +### Fixed +- Initial post-release bug fixes + +## [1.0.0] - 2025-02-05 + +### Added +- First stable release +- Core statistical analysis features (Friedman test, Wilcoxon signed-rank test) +- LaTeX table generation (Median, Friedman, Wilcoxon tables) +- Visualization tools (Boxplot, Critical Distance plot) +- Multi-objective optimization support (Pareto front visualization) +- Command-line interface +- Comprehensive documentation + +## [0.6.0] - 2025-02-03 + +### Added +- Pre-release version with core features +- Initial multi-objective optimization module + +## [0.5.1] - 2025-01-21 + +### Added +- Initial beta release +- Basic statistical testing framework +- CSV data processing utilities + +[Unreleased]: https://github.com/jMetal/SAES/compare/v1.3.6...HEAD +[1.3.6]: https://github.com/jMetal/SAES/compare/v1.3.5...v1.3.6 +[1.3.5]: https://github.com/jMetal/SAES/compare/v1.3.4...v1.3.5 +[1.3.4]: https://github.com/jMetal/SAES/compare/v1.3.2...v1.3.4 +[1.3.2]: https://github.com/jMetal/SAES/compare/v1.3.1...v1.3.2 +[1.3.1]: https://github.com/jMetal/SAES/compare/v1.3.0...v1.3.1 +[1.3.0]: https://github.com/jMetal/SAES/compare/v1.2.0...v1.3.0 +[1.2.0]: https://github.com/jMetal/SAES/compare/v1.1.0...v1.2.0 +[1.1.0]: https://github.com/jMetal/SAES/compare/v1.0.3...v1.1.0 +[1.0.3]: https://github.com/jMetal/SAES/compare/v1.0.2...v1.0.3 +[1.0.2]: https://github.com/jMetal/SAES/compare/v1.0.1...v1.0.2 +[1.0.1]: https://github.com/jMetal/SAES/compare/v1.0.0...v1.0.1 +[1.0.0]: https://github.com/jMetal/SAES/compare/v0.6.0...v1.0.0 +[0.6.0]: https://github.com/jMetal/SAES/compare/v0.5.1...v0.6.0 +[0.5.1]: https://github.com/jMetal/SAES/releases/tag/v0.5.1 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..d3f5723 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,136 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at: + +- Antonio J. Nebro: ajnebro@uma.es +- Emilio Rodrigo Carreira Villalta: emiliorodrigo.ecr@gmail.com + +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f29c4c0..c74079c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,27 +1,360 @@ # Contributing to SAES -Thank you for considering contributing to the **SAES**! We welcome contributions that improve the tool, whether it's new features, bug fixes, or enhancements. By contributing, you help make this project better for everyone. Here's how you can get involved. +Thank you for considering contributing to **SAES** (Stochastic Algorithm Evaluation Suite)! We welcome contributions that improve the tool, whether it's new features, bug fixes, documentation improvements, or enhancements. By contributing, you help make this project better for everyone. -## 🎯 **How to Contribute** +## Table of Contents -1. Fork the repository -2. Create a new branch (`git checkout -b feature-branch`) -3. Commit your changes (`git commit -m 'Add new feature'`) -4. Push to the branch (`git push origin feature-branch`) -5. Create a new Pull Request +- [Code of Conduct](#code-of-conduct) +- [Getting Started](#getting-started) +- [Development Setup](#development-setup) +- [How to Contribute](#how-to-contribute) +- [Pull Request Guidelines](#pull-request-guidelines) +- [Coding Standards](#coding-standards) +- [Testing](#testing) +- [Documentation](#documentation) +- [Reporting Issues](#reporting-issues) +- [Community](#community) -## πŸ’¬ **Pull Request Guidelines** +## Code of Conduct -- **Clear and Descriptive Commit Messages**: Make sure your commit messages are clear and descriptive of the changes you've made. -- **One Change Per PR**: Try to limit your pull request to a single feature or fix to keep things simple and easy to review. -- **Follow Code Style**: Ensure that your code follows the style and conventions of the existing project. -- **Test Your Changes**: If possible, add tests for new functionality or ensure existing tests still pass. +This project adheres to a [Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to the maintainers. -## πŸ“ **Reporting Issues** +## Getting Started -If you encounter bugs or have suggestions for improvements, please open an issue in the [Issues tab](https://github.com/jMetal/SAES). -We look forward to your contributions! +1. **Fork the Repository**: Click the "Fork" button at the top right of the repository page. +2. **Clone Your Fork**: + ```bash + git clone https://github.com/YOUR-USERNAME/SAES.git + cd SAES + ``` +3. **Add Upstream Remote**: + ```bash + git remote add upstream https://github.com/jMetal/SAES.git + ``` -## πŸ™Œ **Thank You for Contributing!** +## Development Setup -Your contributions help make this tool better for everyone. We appreciate your time and effort in helping to improve the **SAES**! \ No newline at end of file +### Prerequisites + +- Python >= 3.10 +- pip (Python package installer) +- git + +### Setting Up Your Development Environment + +We use [uv](https://docs.astral.sh/uv/) for fast, reliable Python package management. + +1. **Install uv** (if not already installed): + ```bash + # On macOS and Linux + curl -LsSf https://astral.sh/uv/install.sh | sh + + # On Windows + powershell -c "irm https://astral.sh/uv/install.ps1 | iex" + + # Or with pip + pip install uv + ``` + +2. **Create a Virtual Environment and Install Dependencies**: + ```bash + # Create virtual environment + uv venv + + # Activate it + source .venv/bin/activate # On Windows: .venv\Scripts\activate + + # Install the package with all dev dependencies + uv pip install -e ".[dev]" + ``` + +3. **Install Pre-commit Hooks** (Optional but Recommended): + ```bash + pre-commit install + ``` + +#### Alternative: Traditional pip/venv + +If you prefer traditional tools: + ```bash + python3 -m venv venv + source venv/bin/activate # On Windows: venv\Scripts\activate + pip install -e ".[dev]" + ``` + +### Running Tests + +```bash +# Run all tests +python -m unittest discover tests + +# Run specific test file +python -m unittest tests.plots.test_boxplot + +# Run with coverage +coverage run -m unittest discover tests +coverage report +coverage html # Generate HTML coverage report +``` + +### Building Documentation + +```bash +cd docs +make html +# Open docs/_build/html/index.html in your browser +``` + +## How to Contribute + +### Types of Contributions + +1. **Bug Fixes**: Fix issues reported in the issue tracker. +2. **New Features**: Implement new statistical tests, visualizations, or utilities. +3. **Documentation**: Improve or add documentation, examples, and tutorials. +4. **Tests**: Add or improve test coverage. +5. **Performance**: Optimize existing code for better performance. + +### Contribution Workflow + +1. **Create a New Branch**: + ```bash + git checkout -b feature/your-feature-name + # or + git checkout -b bug/issue-number-description + ``` + +2. **Make Your Changes**: Write your code, following the [Coding Standards](#coding-standards). + +3. **Write Tests**: Add tests for your changes in the `tests/` directory. + +4. **Update Documentation**: Update relevant documentation if needed. + +5. **Run Tests**: Ensure all tests pass. + ```bash + python -m unittest discover tests + ``` + +6. **Commit Your Changes**: Use clear, descriptive commit messages. + ```bash + git commit -m "feat: add new Bayesian posterior analysis feature" + ``` + + Follow [Conventional Commits](https://www.conventionalcommits.org/): + - `feat:` for new features + - `fix:` for bug fixes + - `docs:` for documentation changes + - `test:` for adding tests + - `refactor:` for code refactoring + - `style:` for formatting changes + - `perf:` for performance improvements + +7. **Push to Your Fork**: + ```bash + git push origin feature/your-feature-name + ``` + +8. **Create a Pull Request**: Go to the original repository and click "New Pull Request". + +## Pull Request Guidelines + +### Before Submitting + +- [ ] Tests pass locally +- [ ] Code follows the project's style guidelines +- [ ] Documentation is updated +- [ ] Commit messages are clear and follow conventions +- [ ] Branch is up to date with main branch + +### PR Description Should Include + +1. **Summary**: Brief description of what the PR does +2. **Motivation**: Why is this change needed? +3. **Changes**: List of main changes made +4. **Testing**: How was this tested? +5. **Screenshots** (if applicable): For UI/visualization changes +6. **Breaking Changes**: Clearly note any breaking changes +7. **Related Issues**: Link to related issue(s) using `#issue-number` + +### Example PR Title + +``` +feat: add support for custom color schemes in boxplots +``` + +### Review Process + +- Maintainers will review your PR +- Address any feedback or requested changes +- Once approved, your PR will be merged +- **One Change Per PR**: Keep PRs focused on a single feature or fix + +## Coding Standards + +### Python Style Guide + +- Follow [PEP 8](https://pep8.org/) style guide +- Use meaningful variable and function names +- Maximum line length: 120 characters +- Use type hints where appropriate +- Write docstrings for all public functions and classes + +### Docstring Format + +Use Google-style docstrings: + +```python +def function_name(param1: str, param2: int) -> bool: + """ + Brief description of the function. + + Detailed description if necessary. + + Args: + param1: Description of param1. + param2: Description of param2. + + Returns: + Description of return value. + + Raises: + ValueError: When invalid input is provided. + + Example: + >>> from SAES.module import function_name + >>> result = function_name("test", 42) + >>> print(result) + True + """ + pass +``` + +### Code Formatting + +The project uses the following tools (automatically applied if pre-commit is installed): + +- **black**: Code formatting +- **isort**: Import sorting +- **flake8**: Linting + +To manually format code: +```bash +black SAES tests +isort SAES tests +flake8 SAES tests +``` + +## Testing + +### Writing Tests + +- Place tests in the `tests/` directory +- Mirror the package structure in test files +- Use descriptive test names: `test__` +- Test both expected behavior and edge cases +- Include tests for error conditions + +### Test Example + +```python +import unittest +from SAES.statistical_tests.non_parametrical import friedman + +class TestFriedman(unittest.TestCase): + def setUp(self): + self.data = ... # Test data + + def test_friedman_basic_functionality(self): + """Test basic friedman test execution.""" + result = friedman(self.data, maximize=True) + self.assertIsInstance(result, pd.DataFrame) + # More assertions... + + def test_friedman_invalid_input(self): + """Test friedman with invalid input.""" + with self.assertRaises(ValueError): + friedman(invalid_data, maximize=True) +``` + +## Documentation + +### Documentation Standards + +- Document all public APIs +- Include usage examples +- Update relevant `.rst` files in `docs/` +- Add entries to appropriate documentation sections + +### Building and Viewing Documentation Locally + +```bash +cd docs +make html +open _build/html/index.html # macOS +# or +xdg-open _build/html/index.html # Linux +# or +start _build/html/index.html # Windows +``` + +## Reporting Issues + +### Before Creating an Issue + +- Search existing issues to avoid duplicates +- Check the documentation +- Try to reproduce the issue + +### Creating a Good Issue + +Include: + +1. **Clear Title**: Descriptive and specific +2. **Environment**: + - OS and version + - Python version + - SAES version +3. **Steps to Reproduce**: Minimal code example +4. **Expected Behavior**: What should happen +5. **Actual Behavior**: What actually happens +6. **Additional Context**: Screenshots, error messages, etc. + +### Issue Templates + +Use the appropriate template when available: +- Bug Report +- Feature Request +- Documentation Improvement + +## Community + +### Getting Help + +- **Documentation**: Check the [official documentation](https://jMetal.github.io/SAES/) +- **Issues**: Search or create an issue +- **Discussions**: Join GitHub Discussions for questions and ideas + +### Stay Updated + +- Watch the repository for updates +- Follow release notes in CHANGELOG.md +- Check the project roadmap + +## Recognition + +Contributors will be acknowledged in: +- Release notes +- CHANGELOG.md +- GitHub contributors page + +## License + +By contributing to SAES, you agree that your contributions will be licensed under the GNU General Public License v3.0. + +--- + +## Thank You! πŸŽ‰ + +Your contributions help make SAES better for everyone. We appreciate your time and effort in helping to improve the project! + +For questions about contributing, please open an issue or contact the maintainers directly. diff --git a/README.md b/README.md index e6194c8..0f04417 100644 --- a/README.md +++ b/README.md @@ -95,15 +95,33 @@ The SAES library offers a range of functions categorized into three groups, corr - **Python**: >= 3.10 ## πŸ“¦ Installation -Before installing the project, we recommend creating a virtual environment to avoid conflicts with other Python projects: + +### Using pip (Recommended for Users) ```sh -python3 -m venv venv -source venv/bin/activate # On Windows, use `venv\Scripts\activate` +pip install SAES +``` + +### For Development + +We recommend using [uv](https://docs.astral.sh/uv/) for fast dependency management: + +```sh +# Install uv +curl -LsSf https://astral.sh/uv/install.sh | sh + +# Clone and setup +git clone https://github.com/jMetal/SAES.git +cd SAES +uv venv +source .venv/bin/activate # On Windows: .venv\Scripts\activate +uv pip install -e ".[dev]" ``` -Once you have activated the virtual environment, you can install the project with its dependencies using the following command: +### Traditional venv Setup ```sh -pip install SAES +python3 -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate +pip install -e ".[dev]" ``` ## 🀝 Contributors diff --git a/SAES/__main__.py b/SAES/__main__.py index ca5398f..4470c49 100644 --- a/SAES/__main__.py +++ b/SAES/__main__.py @@ -6,7 +6,7 @@ from SAES.latex_generation.stats_table import Wilcoxon from SAES.plots.boxplot import Boxplot -from SAES.plots.CDplot import CDplot +from SAES.plots.cdplot import CDplot from SAES.multiobjective.pareto_front import Front2D from SAES.multiobjective.pareto_front import Front3D diff --git a/SAES/html/notebooks/bayesian_posterior.ipynb b/SAES/html/notebooks/bayesian_posterior.ipynb index 81f1c3b..eb7b8a2 100644 --- a/SAES/html/notebooks/bayesian_posterior.ipynb +++ b/SAES/html/notebooks/bayesian_posterior.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "from SAES.plots.Pplot import Pplot" + "from SAES.plots.pplot import Pplot" ] }, { @@ -120,4 +120,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/SAES/html/notebooks/multiobjective_fronts2D.ipynb b/SAES/html/notebooks/multiobjective_fronts2D.ipynb index ab6b86a..5549082 100644 --- a/SAES/html/notebooks/multiobjective_fronts2D.ipynb +++ b/SAES/html/notebooks/multiobjective_fronts2D.ipynb @@ -9,9 +9,9 @@ "from SAES.latex_generation.stats_table import Friedman\n", "from SAES.latex_generation.stats_table import Wilcoxon\n", "from SAES.plots.boxplot import Boxplot\n", - "from SAES.plots.CDplot import CDplot\n", + "from SAES.plots.cdplot import CDplot\n", "from SAES.plots.violin import Violin\n", - "from SAES.plots.HistoPlot import HistoPlot\n", + "from SAES.plots.histoplot import HistoPlot\n", "from SAES.multiobjective.pareto_front import Front2D" ] }, @@ -543,4 +543,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/SAES/html/notebooks/multiobjective_fronts3D.ipynb b/SAES/html/notebooks/multiobjective_fronts3D.ipynb index 06d715a..1235f8d 100644 --- a/SAES/html/notebooks/multiobjective_fronts3D.ipynb +++ b/SAES/html/notebooks/multiobjective_fronts3D.ipynb @@ -9,9 +9,9 @@ "from SAES.latex_generation.stats_table import Friedman\n", "from SAES.latex_generation.stats_table import Wilcoxon\n", "from SAES.plots.boxplot import Boxplot\n", - "from SAES.plots.CDplot import CDplot\n", + "from SAES.plots.cdplot import CDplot\n", "from SAES.plots.violin import Violin\n", - "from SAES.plots.HistoPlot import HistoPlot\n", + "from SAES.plots.histoplot import HistoPlot\n", "from SAES.multiobjective.pareto_front import Front3D" ] }, @@ -543,4 +543,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/SAES/html/notebooks/multiobjective_frontsND.ipynb b/SAES/html/notebooks/multiobjective_frontsND.ipynb index 5c71206..6af62cd 100644 --- a/SAES/html/notebooks/multiobjective_frontsND.ipynb +++ b/SAES/html/notebooks/multiobjective_frontsND.ipynb @@ -9,9 +9,9 @@ "from SAES.latex_generation.stats_table import Friedman\n", "from SAES.latex_generation.stats_table import Wilcoxon\n", "from SAES.plots.boxplot import Boxplot\n", - "from SAES.plots.CDplot import CDplot\n", + "from SAES.plots.cdplot import CDplot\n", "from SAES.plots.violin import Violin\n", - "from SAES.plots.HistoPlot import HistoPlot\n", + "from SAES.plots.histoplot import HistoPlot\n", "from SAES.multiobjective.pareto_front import FrontND" ] }, @@ -543,4 +543,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/SAES/html/notebooks/multiobjective_optimization.ipynb b/SAES/html/notebooks/multiobjective_optimization.ipynb index f74086f..a83459b 100644 --- a/SAES/html/notebooks/multiobjective_optimization.ipynb +++ b/SAES/html/notebooks/multiobjective_optimization.ipynb @@ -9,10 +9,10 @@ "from SAES.latex_generation.stats_table import Friedman\n", "from SAES.latex_generation.stats_table import Wilcoxon\n", "from SAES.plots.boxplot import Boxplot\n", - "from SAES.plots.CDplot import CDplot\n", + "from SAES.plots.cdplot import CDplot\n", "from SAES.plots.violin import Violin\n", - "from SAES.plots.Pplot import Pplot\n", - "from SAES.plots.HistoPlot import HistoPlot" + "from SAES.plots.pplot import Pplot\n", + "from SAES.plots.histoplot import HistoPlot" ] }, { @@ -475,4 +475,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/SAES/plots/boxplot.py b/SAES/plots/boxplot.py index 261086a..4c04611 100644 --- a/SAES/plots/boxplot.py +++ b/SAES/plots/boxplot.py @@ -59,7 +59,7 @@ def __init__(self, data: pd.DataFrame, metrics: pd.DataFrame, metric: str) -> No None Example: - >>> frtom SAES.plots.boxplot import Boxplot + >>> from SAES.plots.boxplot import Boxplot >>> >>> data = pd.read_csv("data.csv") >>> metrics = pd.read_csv("metrics.csv") @@ -93,7 +93,7 @@ def save_instance(self, instance: str, output_path: str, file_name: str = None, None Example: - >>> frtom SAES.plots.boxplot import Boxplot + >>> from SAES.plots.boxplot import Boxplot >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -128,7 +128,7 @@ def save_all_instances(self, output_path: str, file_name: str = None, width: int None Example: - >>> frtom SAES.plots.boxplot import Boxplot + >>> from SAES.plots.boxplot import Boxplot >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -160,7 +160,7 @@ def show_instance(self, instance: str, width: int = 8) -> None: None Example: - >>> frtom SAES.plots.boxplot import Boxplot + >>> from SAES.plots.boxplot import Boxplot >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -185,7 +185,7 @@ def show_all_instances(self, width: int = 30) -> None: None Example: - >>> frtom SAES.plots.boxplot import Boxplot + >>> from SAES.plots.boxplot import Boxplot >>> import os >>> >>> data = pd.read_csv("data.csv") diff --git a/SAES/plots/CDplot.py b/SAES/plots/cdplot.py similarity index 98% rename from SAES/plots/CDplot.py rename to SAES/plots/cdplot.py index 6544025..85f4db9 100644 --- a/SAES/plots/CDplot.py +++ b/SAES/plots/cdplot.py @@ -53,7 +53,7 @@ def __init__(self, data: pd.DataFrame, metrics: pd.DataFrame, metric: str) -> No None Example: - >>> frtom SAES.plots.CDplot import CDplot + >>> from SAES.plots.cdplot import CDplot >>> >>> data = pd.read_csv("data.csv") >>> metrics = pd.read_csv("metrics.csv") @@ -86,7 +86,7 @@ def save(self, output_path: str, file_name: str = None, width: int = 9) -> None: None Example: - >>> frtom SAES.plots.CDplot import CDplot + >>> from SAES.plots.cdplot import CDplot >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -115,7 +115,7 @@ def show(self, width: int = 9) -> None: None Example: - >>> frtom SAES.plots.CDplot import CDplot + >>> from SAES.plots.cdplot import CDplot >>> import os >>> >>> data = pd.read_csv("data.csv") diff --git a/SAES/plots/HistoPlot.py b/SAES/plots/histoplot.py similarity index 94% rename from SAES/plots/HistoPlot.py rename to SAES/plots/histoplot.py index 6800e95..a9855aa 100644 --- a/SAES/plots/HistoPlot.py +++ b/SAES/plots/histoplot.py @@ -59,7 +59,7 @@ def __init__(self, data: pd.DataFrame, metrics: pd.DataFrame, metric: str) -> No None Example: - >>> frtom SAES.plots.Histoplot import Histoplot + >>> from SAES.plots.histoplot import HistoPlot >>> >>> data = pd.read_csv("data.csv") >>> metrics = pd.read_csv("metrics.csv") @@ -94,7 +94,7 @@ def save_instance(self, instance: str, output_path: str, file_name: str = None, None Example: - >>> frtom SAES.plots.Histoplot import Histoplot + >>> from SAES.plots.histoplot import HistoPlot >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -129,7 +129,7 @@ def save_all_instances(self, output_path: str, file_name: str = None, width: int None Example: - >>> frtom SAES.plots.Histoplot import Histoplot + >>> from SAES.plots.histoplot import HistoPlot >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -161,7 +161,7 @@ def show_instance(self, instance: str, width: int = 8) -> None: None Example: - >>> frtom SAES.plots.Histoplot import Histoplot + >>> from SAES.plots.histoplot import HistoPlot >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -186,7 +186,7 @@ def show_all_instances(self, width: int = 30) -> None: None Example: - >>> frtom SAES.plots.Histoplot import Histoplot + >>> from SAES.plots.histoplot import HistoPlot >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -272,11 +272,4 @@ def _plot_all_instances(self, width: int) -> None: for j in range(i + 1, len(axes)): fig.delaxes(axes[j]) - plt.subplots_adjust(left=0.1, right=0.9, bottom=0.1, top=0.9, wspace=0.35, hspace=0.45) - -if __name__ == "__main__": - data = "/home/khaosdev/SAES/notebooks/swarmIntelligence.csv" - metrics = "/home/khaosdev/SAES/notebooks/multiobjectiveMetrics.csv" - metric = "HV" - histoPlot = HistoPlot(data, metrics, metric) - histoPlot.save_instance("ZDT1", "/home/khaosdev/SAES/notebooks/plots", width=10) \ No newline at end of file + plt.subplots_adjust(left=0.1, right=0.9, bottom=0.1, top=0.9, wspace=0.35, hspace=0.45) \ No newline at end of file diff --git a/SAES/plots/Pplot.py b/SAES/plots/pplot.py similarity index 98% rename from SAES/plots/Pplot.py rename to SAES/plots/pplot.py index 2cc6774..d65b5a4 100644 --- a/SAES/plots/Pplot.py +++ b/SAES/plots/pplot.py @@ -65,7 +65,7 @@ def __init__(self, data: pd.DataFrame, metrics: pd.DataFrame, metric: str, bayes None Example: - >>> frtom SAES.plots.Pplot import Pplot + >>> from SAES.plots.pplot import Pplot >>> >>> data = pd.read_csv("data.csv") >>> metrics = pd.read_csv("metrics.csv") @@ -109,7 +109,7 @@ def save(self, alg1, alg2, output_path: str, file_name: str = None, width: int = None Example: - >>> frtom SAES.plots.Pplot import Pplot + >>> from SAES.plots.pplot import Pplot >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -147,7 +147,7 @@ def show(self, alg1: str, alg2: str, width: int = 5, sample_size: int = 2500) -> None Example: - >>> frtom SAES.plots.Pplot import Pplot + >>> from SAES.plots.pplot import Pplot >>> >>> data = pd.read_csv("data.csv") >>> metrics = pd.read_csv("metrics.csv") @@ -190,7 +190,7 @@ def save_pivot(self, algorithm: str, output_path: str, file_name: str = None, None Example: - >>> frtom SAES.plots.Pplot import Pplot + >>> from SAES.plots.pplot import Pplot >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -227,7 +227,7 @@ def show_pivot(self, algorithm: str, width: int = 30, heigth: int = 15, sample_s None Example: - >>> frtom SAES.plots.Pplot import Pplot + >>> from SAES.plots.pplot import Pplot >>> >>> data = pd.read_csv("data.csv") >>> metrics = pd.read_csv("metrics.csv") diff --git a/SAES/plots/violin.py b/SAES/plots/violin.py index c81de67..470a464 100644 --- a/SAES/plots/violin.py +++ b/SAES/plots/violin.py @@ -62,7 +62,7 @@ def __init__(self, data: pd.DataFrame, metrics: pd.DataFrame, metric: str) -> No None Example: - >>> frtom SAES.plots.violin import Violin + >>> from SAES.plots.violin import Violin >>> >>> data = pd.read_csv("data.csv") >>> metrics = pd.read_csv("metrics.csv") @@ -97,7 +97,7 @@ def save_instance(self, instance: str, output_path: str, file_name: str = None, None Example: - >>> frtom SAES.plots.violin import Violin + >>> from SAES.plots.violin import Violin >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -132,7 +132,7 @@ def save_all_instances(self, output_path: str, file_name: str = None, width: int None Example: - >>> frtom SAES.plots.violin import Violin + >>> from SAES.plots.violin import Violin >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -164,7 +164,7 @@ def show_instance(self, instance: str, width: int = 8) -> None: None Example: - >>> frtom SAES.plots.violin import Violin + >>> from SAES.plots.violin import Violin >>> import os >>> >>> data = pd.read_csv("data.csv") @@ -189,7 +189,7 @@ def show_all_instances(self, width: int = 30) -> None: None Example: - >>> frtom SAES.plots.violin import Violin + >>> from SAES.plots.violin import Violin >>> import os >>> >>> data = pd.read_csv("data.csv") diff --git a/SAES/statistical_tests/apv_procedures.py b/SAES/statistical_tests/apv_procedures.py index 3c61609..75cb8e6 100644 --- a/SAES/statistical_tests/apv_procedures.py +++ b/SAES/statistical_tests/apv_procedures.py @@ -1,11 +1,34 @@ +""" +Advanced Post-Hoc Analysis with Adjusted P-Values (APV) Procedures. + +This module contains implementations of various post-hoc tests with adjusted p-value +procedures for multiple comparisons. These are advanced statistical methods that extend +the basic non-parametric tests provided in non_parametrical.py. + +Current Status: + These procedures are fully implemented but not yet integrated into the main SAES API. + They are available for advanced users who need fine-grained control over post-hoc + analysis with specific APV correction methods. + +Future Integration: + A future release will integrate these procedures into the main statistical test + functions, providing a unified API with optional parameters for APV procedures. + +Usage: + These functions can be imported and used directly for advanced statistical analysis. + See individual function docstrings for detailed usage information. + +Supported APV Procedures: + - One vs. All: Bonferroni, Holm, Hochberg, Holland, Finner, Li + - All vs. All: Shaffer, Holm, Nemenyi +""" + import numpy as np import pandas as pd from SAES.statistical_tests.non_parametrical import _ranks from scipy.stats import rankdata, mannwhitneyu, chi2, binom, f, norm -## TODO: Integrate the APV procedures into the main functions of the SAES package. - def friedman_ph_test(data: pd.DataFrame, maximize: bool, control=None, apv_procedure=None) -> pd.DataFrame: """Friedman post-hoc test. diff --git a/docs/conf.py b/docs/conf.py index 1801cbf..a2610c6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,10 +11,20 @@ sys.path.insert(0, os.path.abspath('..')) +# Read version from version.txt +def get_version(): + version_file = os.path.join(os.path.dirname(__file__), '..', 'version.txt') + try: + with open(version_file, 'r') as f: + return f.read().strip() + except FileNotFoundError: + return 'dev' + project = 'SAES' copyright = '2025, Emilio Rodrigo Carreira Villalta and Antonio J. Nebro' author = 'Emilio Rodrigo Carreira Villalta and Antonio J. Nebro' -release = 'dev' +release = get_version() +version = release # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/docs/usage/bayesian.rst b/docs/usage/bayesian.rst index 50faca4..df33834 100644 --- a/docs/usage/bayesian.rst +++ b/docs/usage/bayesian.rst @@ -9,7 +9,7 @@ This module provides the functionality to generate a posterior plot for the Baye .. code-block:: python - from SAES.plots.Pplot import Pplot + from SAES.plots.pplot import Pplot # Load the data and metrics from the CSV files data = "swarmIntelligence.csv" diff --git a/docs/usage/cd.rst b/docs/usage/cd.rst index 51d284f..4d65a67 100644 --- a/docs/usage/cd.rst +++ b/docs/usage/cd.rst @@ -5,7 +5,7 @@ The last feauture of the library is the ability to generate critical distance gr .. code-block:: python - from SAES.plots.CDplot import CDplot + from SAES.plots.cdplot import CDplot # Load the data and metrics from the CSV files data = "swarmIntelligence.csv" diff --git a/docs/usage/histoplot.rst b/docs/usage/histoplot.rst index 0ccfed1..c89b672 100644 --- a/docs/usage/histoplot.rst +++ b/docs/usage/histoplot.rst @@ -9,7 +9,7 @@ nother feature of the library is the ability to generate histogram plots of the .. code-block:: python - from SAES.plots.HistoPlot import HistoPlot + from SAES.plots.histoplot import HistoPlot # Load the data and metrics from the CSV files data = "swarmIntelligence.csv" diff --git a/notebooks/SAES_tutorial.ipynb b/notebooks/SAES_tutorial.ipynb index e99b285..d866ff1 100644 --- a/notebooks/SAES_tutorial.ipynb +++ b/notebooks/SAES_tutorial.ipynb @@ -35,9 +35,9 @@ "source": [ "from SAES.latex_generation.stats_table import Friedman\n", "from SAES.plots.boxplot import Boxplot\n", - "from SAES.plots.CDplot import CDplot\n", + "from SAES.plots.cdplot import CDplot\n", "from SAES.plots.violin import Violin\n", - "from SAES.plots.HistoPlot import HistoPlot\n", + "from SAES.plots.histoplot import HistoPlot\n", "\n", "import matplotlib.pyplot as plt\n", "from PIL import Image\n", diff --git a/notebooks/bayesian_posterior.ipynb b/notebooks/bayesian_posterior.ipynb index 8c5ed07..6e8da70 100644 --- a/notebooks/bayesian_posterior.ipynb +++ b/notebooks/bayesian_posterior.ipynb @@ -29,11 +29,11 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "from SAES.plots.Pplot import Pplot\n", + "from SAES.plots.pplot import Pplot\n", "import pandas as pd\n", "import os" ] diff --git a/notebooks/ml/ml_study.ipynb b/notebooks/ml/ml_study.ipynb index cb2c4d1..a573eff 100644 --- a/notebooks/ml/ml_study.ipynb +++ b/notebooks/ml/ml_study.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook demonstrates how to use the **SAES** library in a machine learning context. As a case study, we will analyze the performance of different machine learning modelsβ€”SVM (Support Vector Machine), RandomForest, KNN (K-Nearest Neighbors), and MLP (Multi-Layer Perceptron)β€”against different versions of the Iris dataset. In machine learning, performance evaluation typically involves metrics like **accuracy** and **loss**. We'll compare the models across different Iris dataset versions using these metrics to identify the best-performing model for various versions of the dataset.\n", + "This notebook demonstrates how to use the **SAES** library in a machine learning context. As a case study, we will analyze the performance of different machine learning models\u2014SVM (Support Vector Machine), RandomForest, KNN (K-Nearest Neighbors), and MLP (Multi-Layer Perceptron)\u2014against different versions of the Iris dataset. In machine learning, performance evaluation typically involves metrics like **accuracy** and **loss**. We'll compare the models across different Iris dataset versions using these metrics to identify the best-performing model for various versions of the dataset.\n", "\n", "## **Iris Dataset Versions**\n", "\n", @@ -45,7 +45,7 @@ "We will use the following metrics to assess the performance of the models:\n", "\n", "- **Accuracy**: The percentage of correct predictions made by the model. It is one of the most common metrics used in classification tasks.\n", - "- **Loss**: The loss function quantifies how well the model’s predictions match the actual labels. We will use categorical cross-entropy loss for classification tasks, which measures the difference between the predicted probability distribution and the true distribution.\n", + "- **Loss**: The loss function quantifies how well the model\u2019s predictions match the actual labels. We will use categorical cross-entropy loss for classification tasks, which measures the difference between the predicted probability distribution and the true distribution.\n", "\n", "## **Why the Iris Dataset and These Models?**\n", "\n", @@ -69,10 +69,10 @@ "from SAES.latex_generation.stats_table import Friedman\n", "from SAES.latex_generation.stats_table import Wilcoxon\n", "from SAES.plots.boxplot import Boxplot\n", - "from SAES.plots.CDplot import CDplot\n", - "from SAES.plots.Pplot import Pplot\n", + "from SAES.plots.cdplot import CDplot\n", + "from SAES.plots.pplot import Pplot\n", "from SAES.plots.violin import Violin\n", - "from SAES.plots.HistoPlot import HistoPlot\n", + "from SAES.plots.histoplot import HistoPlot\n", "\n", "import pandas as pd" ] @@ -521,4 +521,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/notebooks/multiobjective/multiobjective_fronts.ipynb b/notebooks/multiobjective/multiobjective_fronts.ipynb index 41c7a15..4660373 100644 --- a/notebooks/multiobjective/multiobjective_fronts.ipynb +++ b/notebooks/multiobjective/multiobjective_fronts.ipynb @@ -9,9 +9,9 @@ "from SAES.latex_generation.stats_table import Friedman\n", "from SAES.latex_generation.stats_table import Wilcoxon\n", "from SAES.plots.boxplot import Boxplot\n", - "from SAES.plots.CDplot import CDplot\n", + "from SAES.plots.cdplot import CDplot\n", "from SAES.plots.violin import Violin\n", - "from SAES.plots.HistoPlot import HistoPlot\n", + "from SAES.plots.histoplot import HistoPlot\n", "from SAES.multiobjective.pareto_front import Front2D\n", "from SAES.multiobjective.pareto_front import Front3D\n", "from SAES.multiobjective.pareto_front import FrontND\n", @@ -293,27 +293,27 @@ "Each algorithm's front data should be organized in the following folder structure:\n", "\n", "```\n", - "πŸ“‚ fronts_folder \n", - "β”œβ”€β”€ πŸ“‚ algorithm-1/ \n", - "β”‚ β”œβ”€β”€ πŸ“‚ instance-1 \n", - "| | β”œβ”€β”€ BEST_metric-1_FUN.csv\n", - "| | β”œβ”€β”€ MEDIAN_metric-1_FUN.csv\n", + "\ud83d\udcc2 fronts_folder \n", + "\u251c\u2500\u2500 \ud83d\udcc2 algorithm-1/ \n", + "\u2502 \u251c\u2500\u2500 \ud83d\udcc2 instance-1 \n", + "| | \u251c\u2500\u2500 BEST_metric-1_FUN.csv\n", + "| | \u251c\u2500\u2500 MEDIAN_metric-1_FUN.csv\n", "| | .\n", "| | .\n", - "| | β”œβ”€β”€ BEST_metric-k_FUN.csv\n", - "| | β”œβ”€β”€ MEDIAN_metric-k_FUN.csv\n", - "β”‚ β”œβ”€β”€ πŸ“‚ instance-2\n", + "| | \u251c\u2500\u2500 BEST_metric-k_FUN.csv\n", + "| | \u251c\u2500\u2500 MEDIAN_metric-k_FUN.csv\n", + "\u2502 \u251c\u2500\u2500 \ud83d\udcc2 instance-2\n", "| .\n", "| .\n", - "| └── πŸ“‚ instance-m\n", - "β”œβ”€β”€ πŸ“‚ algorithm-2/ \n", + "| \u2514\u2500\u2500 \ud83d\udcc2 instance-m\n", + "\u251c\u2500\u2500 \ud83d\udcc2 algorithm-2/ \n", ".\n", ".\n", - "β”œβ”€β”€ πŸ“‚ algorithm-n/ \n", + "\u251c\u2500\u2500 \ud83d\udcc2 algorithm-n/ \n", "```\n", "\n", "- Each **algorithm** has its own directory inside `fronts_folder`. \n", - "- Within each algorithm’s folder, **instances** are stored as subdirectories. \n", + "- Within each algorithm\u2019s folder, **instances** are stored as subdirectories. \n", "- Each instance contains multiple CSV files representing Pareto fronts, following the format: \n", " - `BEST_metric-x_FUN.csv`: The best Pareto front based on metric `x`. \n", " - `MEDIAN_metric-x_FUN.csv`: The median Pareto front based on metric `x`.\n", @@ -323,12 +323,12 @@ "The reference front data should be stored in a separate folder with the following structure: \n", "\n", "```\n", - "πŸ“‚ references_folder \n", - "β”œβ”€β”€ instance-1.csv\n", - "β”œβ”€β”€ instance-2.csv \n", + "\ud83d\udcc2 references_folder \n", + "\u251c\u2500\u2500 instance-1.csv\n", + "\u251c\u2500\u2500 instance-2.csv \n", ".\n", ".\n", - "β”œβ”€β”€ instance-m.csv \n", + "\u251c\u2500\u2500 instance-m.csv \n", "```\n", "\n", "- Each **instance** has a corresponding reference front file. \n", @@ -1554,4 +1554,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/notebooks/multiobjective/multiobjective_optimization.ipynb b/notebooks/multiobjective/multiobjective_optimization.ipynb index 01af293..6684996 100644 --- a/notebooks/multiobjective/multiobjective_optimization.ipynb +++ b/notebooks/multiobjective/multiobjective_optimization.ipynb @@ -9,10 +9,10 @@ "from SAES.latex_generation.stats_table import Friedman\n", "from SAES.latex_generation.stats_table import Wilcoxon\n", "from SAES.plots.boxplot import Boxplot\n", - "from SAES.plots.CDplot import CDplot\n", - "from SAES.plots.Pplot import Pplot\n", + "from SAES.plots.cdplot import CDplot\n", + "from SAES.plots.pplot import Pplot\n", "from SAES.plots.violin import Violin\n", - "from SAES.plots.HistoPlot import HistoPlot\n", + "from SAES.plots.histoplot import HistoPlot\n", "\n", "import pandas as pd" ] @@ -517,4 +517,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index cb63d74..815ae73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ exclude = ["*.pyc", "*.log", ".git", "dist", "venv", "*__pycache__*"] [project] name = "SAES" version = "1.3.6" +dynamic = [] description = "Python library designed to analyze and compare the performance of different algorithms across multiple problems using non-parametric statistical tests" readme = "README.md" @@ -17,16 +18,14 @@ dependencies = [ "matplotlib==3.9.2", "Jinja2==3.1.6", "scikit-posthocs==0.10.0", - "coverage==7.6.8", - "sphinx==8.1.3", - "sphinx_rtd_theme==3.0.2", - "ipython<=8.30.0", - "nbconvert==7.16.6", - "papermill==2.6.0", - "ipykernel<=6.29.5" + "scipy>=1.7.0", + "pandas>=1.3.0", + "seaborn>=0.11.0", + "papermill==2.6.0" ] requires-python = ">=3.10" + authors = [ {name = "Antonio J. Nebro", email = "ajnebro@uma.es"}, {name = "Emilio Rodrigo Carreira Villalta", email = "emiliorodrigo.ecr@gmail.com"} @@ -44,3 +43,33 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", "Programming Language :: Python :: 3.10" ] + +[project.optional-dependencies] +dev = [ + "coverage==7.6.8", + "sphinx==8.1.3", + "sphinx_rtd_theme==3.0.2", + "pytest>=7.0.0", + "pytest-cov>=3.0.0", + "black>=22.0.0", + "flake8>=4.0.0", + "mypy>=0.950", + "isort>=5.10.0", + "pre-commit>=3.0.0" +] +html = [ + "ipython<=8.30.0", + "nbconvert==7.16.6", + "papermill==2.6.0", + "ipykernel<=6.29.5" +] +test = [ + "coverage==7.6.8", + "pytest>=7.0.0", + "pytest-cov>=3.0.0", + "Pillow>=9.0.0" +] +docs = [ + "sphinx==8.1.3", + "sphinx_rtd_theme==3.0.2" +] diff --git a/tests/htmls/bayesian.ipynb b/tests/htmls/bayesian.ipynb new file mode 100644 index 0000000..3bef08a --- /dev/null +++ b/tests/htmls/bayesian.ipynb @@ -0,0 +1,223 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "ebd39d2e", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "data = \"tests/test_data/swarmIntelligence.csv\"\n", + "metrics = \"tests/test_data/multiobjectiveMetrics.csv\"\n", + "metric = \"HV\"\n", + "pivot = \"NSGAII\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a37ca2f0", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from SAES.plots.pplot import Pplot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1ca39f1b", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "In order to generate the HTML code from the notebook, write this command in the terminal:\n", + "jupyter nbconvert --to html --no-input multiobjective_optimization.ipynb\n", + "\n", + "If you do not have jupyter installed, you can use the python terminal:\n", + "python -m nbconvert --to html --no-input multiobjective_optimization.ipynb\n", + "\"\"\";" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aac610b3", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# Load the experiment data\n", + "experimentData = data\n", + "metrics = metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e578e373", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "print(\"Metric: \", metric)" + ] + }, + { + "cell_type": "markdown", + "id": "46d636d1", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Posterior Plot using Bayesian Sign Test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "803f7afc", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "Pplot(experimentData, metrics, metric, bayesian_test=\"sign\").show_pivot(pivot)" + ] + }, + { + "cell_type": "markdown", + "id": "ba6bfa6e", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Posterior Plot using Bayesian Signed Rank Test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d7c5f7b", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "Pplot(experimentData, metrics, metric, bayesian_test=\"rank\").show_pivot(pivot)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + }, + "papermill": { + "default_parameters": {}, + "duration": 0.022066, + "end_time": "2025-11-08T18:27:09.819556", + "environment_variables": {}, + "exception": null, + "input_path": "/Users/rorro6787/Desktop/SAES/SAES/html/notebooks/bayesian_posterior.ipynb", + "output_path": "tests/htmls/bayesian.ipynb", + "parameters": { + "data": "tests/test_data/swarmIntelligence.csv", + "metric": "HV", + "metrics": "tests/test_data/multiobjectiveMetrics.csv", + "pivot": "NSGAII" + }, + "start_time": "2025-11-08T18:27:09.797490", + "version": "2.6.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/tests/htmls/fronts2D.ipynb b/tests/htmls/fronts2D.ipynb new file mode 100644 index 0000000..bf6137f --- /dev/null +++ b/tests/htmls/fronts2D.ipynb @@ -0,0 +1,301 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "957c9e95", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "data = \"tests/test_data/swarmIntelligence.csv\"\n", + "metrics = \"tests/test_data/multiobjectiveMetrics.csv\"\n", + "metric = \"HV\"\n", + "fronts = \"tests/test_data/fronts\"\n", + "references = \"tests/test_data/references\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "538b4d35", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from SAES.latex_generation.stats_table import Friedman\n", + "from SAES.latex_generation.stats_table import Wilcoxon\n", + "from SAES.plots.boxplot import Boxplot\n", + "from SAES.plots.cdplot import CDplot\n", + "from SAES.plots.violin import Violin\n", + "from SAES.plots.histoplot import HistoPlot\n", + "from SAES.multiobjective.pareto_front import Front2D" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37038ffa", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "In order to generate the HTML code from the notebook, write this command in the terminal:\n", + "jupyter nbconvert --to html --no-input multiobjective_optimization.ipynb\n", + "\n", + "If you do not have jupyter installed, you can use the python terminal:\n", + "python -m nbconvert --to html --no-input multiobjective_optimization.ipynb\n", + "\"\"\";" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b2f6f96", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# Load the experiment data\n", + "experiment_data = data\n", + "metrics = metrics\n", + "fronts_path = fronts\n", + "references_path = references" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95be5f23", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "print(\"Metric: \", metric)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91c32437", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "friedman = Friedman(experiment_data, metrics, metric)\n", + "friedman.show()" + ] + }, + { + "cell_type": "markdown", + "id": "fa35b774", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Wilcoxon rank-sum test table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8257d3d6", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "wilcoxon = Wilcoxon(experiment_data, metrics, metric)\n", + "wilcoxon.show()" + ] + }, + { + "cell_type": "markdown", + "id": "0701d3b2", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Critical distance ranking" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a89b6b54", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "cdplot = CDplot(experiment_data, metrics, metric)\n", + "cdplot.show()" + ] + }, + { + "cell_type": "markdown", + "id": "619672a8", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Boxplot + Violin Plot + HistoPlot + Pareto Fronts for each problem (2D projection of the first two objectives)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e9aaa58", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "front = Front2D(fronts_path, references_path, metric)\n", + "boxplot = Boxplot(experiment_data, metrics, metric)\n", + "violin = Violin(experiment_data, metrics, metric)\n", + "histoplot = HistoPlot(experiment_data, metrics, metric)\n", + "\n", + "for instance in front.instances:\n", + " boxplot.show_instance(instance)\n", + " violin.show_instance(instance)\n", + " histoplot.show_instance(instance)\n", + " front.show(instance, median=True)\n", + " print(\"\\n\"*6)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + }, + "papermill": { + "default_parameters": {}, + "duration": 0.002144, + "end_time": "2025-11-08T18:27:09.869755", + "environment_variables": {}, + "exception": null, + "input_path": "/Users/rorro6787/Desktop/SAES/SAES/html/notebooks/multiobjective_fronts2D.ipynb", + "output_path": "tests/htmls/fronts2D.ipynb", + "parameters": { + "data": "tests/test_data/swarmIntelligence.csv", + "fronts": "tests/test_data/fronts", + "metric": "HV", + "metrics": "tests/test_data/multiobjectiveMetrics.csv", + "references": "tests/test_data/references" + }, + "start_time": "2025-11-08T18:27:09.867611", + "version": "2.6.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/tests/htmls/fronts3D.ipynb b/tests/htmls/fronts3D.ipynb new file mode 100644 index 0000000..ede6a3a --- /dev/null +++ b/tests/htmls/fronts3D.ipynb @@ -0,0 +1,301 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "ad13ebbd", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "data = \"tests/test_data/swarmIntelligence.csv\"\n", + "metrics = \"tests/test_data/multiobjectiveMetrics.csv\"\n", + "metric = \"HV\"\n", + "fronts = \"tests/test_data/fronts\"\n", + "references = \"tests/test_data/references\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2bba7a2", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from SAES.latex_generation.stats_table import Friedman\n", + "from SAES.latex_generation.stats_table import Wilcoxon\n", + "from SAES.plots.boxplot import Boxplot\n", + "from SAES.plots.cdplot import CDplot\n", + "from SAES.plots.violin import Violin\n", + "from SAES.plots.histoplot import HistoPlot\n", + "from SAES.multiobjective.pareto_front import Front3D" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e343abd8", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "In order to generate the HTML code from the notebook, write this command in the terminal:\n", + "jupyter nbconvert --to html --no-input multiobjective_optimization.ipynb\n", + "\n", + "If you do not have jupyter installed, you can use the python terminal:\n", + "python -m nbconvert --to html --no-input multiobjective_optimization.ipynb\n", + "\"\"\";" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03238fbb", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# Load the experiment data\n", + "experiment_data = data\n", + "metrics = metrics\n", + "fronts_path = fronts\n", + "references_path = references" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1079af49", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "print(\"Metric: \", metric)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1fa1a174", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "friedman = Friedman(experiment_data, metrics, metric)\n", + "friedman.show()" + ] + }, + { + "cell_type": "markdown", + "id": "cb2a4c75", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Wilcoxon rank-sum test table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90bef33f", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "wilcoxon = Wilcoxon(experiment_data, metrics, metric)\n", + "wilcoxon.show()" + ] + }, + { + "cell_type": "markdown", + "id": "5f7845c3", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Critical distance ranking" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ab349d32", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "cdplot = CDplot(experiment_data, metrics, metric)\n", + "cdplot.show()" + ] + }, + { + "cell_type": "markdown", + "id": "46e4fd9b", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Boxplot + Violin Plot + HistoPlot + Pareto Fronts for each problem (3D)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48b4c8f0", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "front = Front3D(fronts_path, references_path, metric)\n", + "boxplot = Boxplot(experiment_data, metrics, metric)\n", + "violin = Violin(experiment_data, metrics, metric)\n", + "histoplot = HistoPlot(experiment_data, metrics, metric)\n", + "\n", + "for instance in front.instances:\n", + " boxplot.show_instance(instance)\n", + " violin.show_instance(instance)\n", + " histoplot.show_instance(instance)\n", + " front.show(instance, median=True)\n", + " print(\"\\n\"*6)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + }, + "papermill": { + "default_parameters": {}, + "duration": 0.003249, + "end_time": "2025-11-08T18:27:09.892689", + "environment_variables": {}, + "exception": null, + "input_path": "/Users/rorro6787/Desktop/SAES/SAES/html/notebooks/multiobjective_fronts3D.ipynb", + "output_path": "tests/htmls/fronts3D.ipynb", + "parameters": { + "data": "tests/test_data/swarmIntelligence.csv", + "fronts": "tests/test_data/fronts", + "metric": "HV", + "metrics": "tests/test_data/multiobjectiveMetrics.csv", + "references": "tests/test_data/references" + }, + "start_time": "2025-11-08T18:27:09.889440", + "version": "2.6.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/tests/htmls/frontsND.ipynb b/tests/htmls/frontsND.ipynb new file mode 100644 index 0000000..52ae382 --- /dev/null +++ b/tests/htmls/frontsND.ipynb @@ -0,0 +1,303 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0713ffd5", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "data = \"tests/test_data/swarmIntelligence.csv\"\n", + "metrics = \"tests/test_data/multiobjectiveMetrics.csv\"\n", + "metric = \"HV\"\n", + "fronts = \"tests/test_data/fronts\"\n", + "references = \"tests/test_data/references\"\n", + "dimensions = 3\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "183e2e9e", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from SAES.latex_generation.stats_table import Friedman\n", + "from SAES.latex_generation.stats_table import Wilcoxon\n", + "from SAES.plots.boxplot import Boxplot\n", + "from SAES.plots.cdplot import CDplot\n", + "from SAES.plots.violin import Violin\n", + "from SAES.plots.histoplot import HistoPlot\n", + "from SAES.multiobjective.pareto_front import FrontND" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd72ca07", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "In order to generate the HTML code from the notebook, write this command in the terminal:\n", + "jupyter nbconvert --to html --no-input multiobjective_optimization.ipynb\n", + "\n", + "If you do not have jupyter installed, you can use the python terminal:\n", + "python -m nbconvert --to html --no-input multiobjective_optimization.ipynb\n", + "\"\"\";" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2a0af258", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# Load the experiment data\n", + "experiment_data = data\n", + "metrics = metrics\n", + "fronts_path = fronts\n", + "references_path = references" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "641a16d0", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "print(\"Metric: \", metric)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd593de9", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "friedman = Friedman(experiment_data, metrics, metric)\n", + "friedman.show()" + ] + }, + { + "cell_type": "markdown", + "id": "a54f8785", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Wilcoxon rank-sum test table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13d894f3", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "wilcoxon = Wilcoxon(experiment_data, metrics, metric)\n", + "wilcoxon.show()" + ] + }, + { + "cell_type": "markdown", + "id": "36c49b5f", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Critical distance ranking" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd3a8ba9", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "cdplot = CDplot(experiment_data, metrics, metric)\n", + "cdplot.show()" + ] + }, + { + "cell_type": "markdown", + "id": "3a2b31c0", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Boxplot + Violin Plot + HistoPlot + Parallel Coordinates Graph (ND)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f690096f", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "front = FrontND(fronts_path, references_path, metric, dimensions=dimensions)\n", + "boxplot = Boxplot(experiment_data, metrics, metric)\n", + "violin = Violin(experiment_data, metrics, metric)\n", + "histoplot = HistoPlot(experiment_data, metrics, metric)\n", + "\n", + "for instance in front.instances:\n", + " boxplot.show_instance(instance)\n", + " violin.show_instance(instance)\n", + " histoplot.show_instance(instance)\n", + " front.show(instance, median=True)\n", + " print(\"\\n\"*6)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + }, + "papermill": { + "default_parameters": {}, + "duration": 0.002787, + "end_time": "2025-11-08T18:27:09.917859", + "environment_variables": {}, + "exception": null, + "input_path": "/Users/rorro6787/Desktop/SAES/SAES/html/notebooks/multiobjective_frontsND.ipynb", + "output_path": "tests/htmls/frontsND.ipynb", + "parameters": { + "data": "tests/test_data/swarmIntelligence.csv", + "dimensions": 3, + "fronts": "tests/test_data/fronts", + "metric": "HV", + "metrics": "tests/test_data/multiobjectiveMetrics.csv", + "references": "tests/test_data/references" + }, + "start_time": "2025-11-08T18:27:09.915072", + "version": "2.6.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/tests/htmls/no_fronts.ipynb b/tests/htmls/no_fronts.ipynb new file mode 100644 index 0000000..b0c462c --- /dev/null +++ b/tests/htmls/no_fronts.ipynb @@ -0,0 +1,431 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "959e7cfa", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "data = \"tests/test_data/swarmIntelligence.csv\"\n", + "metrics = \"tests/test_data/multiobjectiveMetrics.csv\"\n", + "metric = \"HV\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8fd51bdd", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from SAES.latex_generation.stats_table import Friedman\n", + "from SAES.latex_generation.stats_table import Wilcoxon\n", + "from SAES.plots.boxplot import Boxplot\n", + "from SAES.plots.cdplot import CDplot\n", + "from SAES.plots.violin import Violin\n", + "from SAES.plots.pplot import Pplot\n", + "from SAES.plots.histoplot import HistoPlot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff774d79", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "In order to generate the HTML code from the notebook, write this command in the terminal:\n", + "jupyter nbconvert --to html --no-input multiobjective_optimization.ipynb\n", + "\n", + "If you do not have jupyter installed, you can use the python terminal:\n", + "python -m nbconvert --to html --no-input multiobjective_optimization.ipynb\n", + "\"\"\";" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54237675", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# Load the experiment data\n", + "experimentData = data\n", + "metrics = metrics" + ] + }, + { + "cell_type": "markdown", + "id": "ed79c690", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Comparative study of multi-objective metaheuristics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04c90694", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "print(\"Metric: \", metric)" + ] + }, + { + "cell_type": "markdown", + "id": "59ff8f3f", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Medians table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "75d4c58f", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "friedman = Friedman(experimentData, metrics, metric)\n", + "friedman.show()" + ] + }, + { + "cell_type": "markdown", + "id": "1746b54e", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Wilcoxon rank-sum test table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "69af9c60", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "wilcoxon = Wilcoxon(experimentData, metrics, metric)\n", + "wilcoxon.show()" + ] + }, + { + "cell_type": "markdown", + "id": "b1f0020e", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Critical distance ranking" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2807f83", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "cdplot = CDplot(experimentData, metrics, metric)\n", + "cdplot.show()" + ] + }, + { + "cell_type": "markdown", + "id": "b2abe8a4", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Boxplots" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3dff728", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "boxplot = Boxplot(experimentData, metrics, metric)\n", + "boxplot.show_all_instances()" + ] + }, + { + "cell_type": "markdown", + "id": "9cf9188e", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Violin Plots" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a30b377", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "violin = Violin(experimentData, metrics, metric)\n", + "violin.show_all_instances()" + ] + }, + { + "cell_type": "markdown", + "id": "51cbee68", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## HistoPlots" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7dc700e3", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "histoplot = HistoPlot(experimentData, metrics, metric)\n", + "histoplot.show_all_instances()" + ] + }, + { + "cell_type": "markdown", + "id": "ca60e3e6", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Bayesian Posterior Plots" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cf19114d", + "metadata": { + "papermill": { + "duration": null, + "end_time": null, + "exception": null, + "start_time": null, + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "pplot = Pplot(experimentData, metrics, metric)\n", + "pplot.show_pivot(\"AutoMOPSOD\", heigth=17)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + }, + "papermill": { + "default_parameters": {}, + "duration": 0.001856, + "end_time": "2025-11-08T18:27:09.931057", + "environment_variables": {}, + "exception": null, + "input_path": "/Users/rorro6787/Desktop/SAES/SAES/html/notebooks/multiobjective_optimization.ipynb", + "output_path": "tests/htmls/no_fronts.ipynb", + "parameters": { + "data": "tests/test_data/swarmIntelligence.csv", + "metric": "HV", + "metrics": "tests/test_data/multiobjectiveMetrics.csv" + }, + "start_time": "2025-11-08T18:27:09.929201", + "version": "2.6.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/tests/plots/test_CDplot.py b/tests/plots/test_CDplot.py index 967e36b..e485c8c 100644 --- a/tests/plots/test_CDplot.py +++ b/tests/plots/test_CDplot.py @@ -1,4 +1,4 @@ -from SAES.plots.CDplot import CDplot +from SAES.plots.cdplot import CDplot from PIL import Image import unittest, os diff --git a/tests/plots/test_Histoplot.py b/tests/plots/test_Histoplot.py index 55eda4b..ceed946 100644 --- a/tests/plots/test_Histoplot.py +++ b/tests/plots/test_Histoplot.py @@ -1,4 +1,4 @@ -from SAES.plots.HistoPlot import HistoPlot +from SAES.plots.histoplot import HistoPlot from PIL import Image import unittest, os diff --git a/tests/plots/test_Pplot.py b/tests/plots/test_Pplot.py index 319d485..f231618 100644 --- a/tests/plots/test_Pplot.py +++ b/tests/plots/test_Pplot.py @@ -1,4 +1,4 @@ -from SAES.plots.Pplot import Pplot +from SAES.plots.pplot import Pplot from PIL import Image import unittest, os diff --git a/tests/statistical_tests/test_bayesian.py b/tests/statistical_tests/test_bayesian.py new file mode 100644 index 0000000..d050fb3 --- /dev/null +++ b/tests/statistical_tests/test_bayesian.py @@ -0,0 +1,226 @@ +from SAES.statistical_tests.bayesian import bayesian_sign_test, bayesian_signed_rank_test +import pandas as pd +import numpy as np +import unittest + + +class TestBayesianTests(unittest.TestCase): + """Test suite for Bayesian statistical tests.""" + + def setUp(self): + """Set up test fixtures with various data scenarios.""" + + # Data where algorithm A is clearly better + self.data_a_better = pd.DataFrame({ + "Algorithm_A": [0.9, 0.85, 0.95, 0.9, 0.92, 0.88, 0.91, 0.89], + "Algorithm_B": [0.5, 0.6, 0.55, 0.58, 0.52, 0.54, 0.56, 0.57] + }) + + # Data where algorithms are equivalent + self.data_equivalent = pd.DataFrame({ + "Algorithm_A": [0.5, 0.6, 0.7, 0.55, 0.65], + "Algorithm_B": [0.5, 0.6, 0.7, 0.55, 0.65] + }) + + # Data with small differences (within ROPE) + self.data_rope = pd.DataFrame({ + "Algorithm_A": [0.500, 0.505, 0.498, 0.502, 0.501], + "Algorithm_B": [0.501, 0.504, 0.499, 0.503, 0.500] + }) + + # Data where algorithm B is clearly better + self.data_b_better = pd.DataFrame({ + "Algorithm_A": [0.5, 0.6, 0.55, 0.58, 0.52], + "Algorithm_B": [0.9, 0.85, 0.95, 0.9, 0.92] + }) + + def test_bayesian_sign_test_basic(self): + """Test basic functionality of bayesian_sign_test.""" + result, samples = bayesian_sign_test(self.data_a_better) + + # Check return types + self.assertIsInstance(result, np.ndarray) + self.assertIsInstance(samples, np.ndarray) + + # Check result shape (should be 3 probabilities) + self.assertEqual(result.shape, (3,)) + + # Check probabilities sum to 1 + self.assertAlmostEqual(np.sum(result), 1.0, places=5) + + # Check all probabilities are between 0 and 1 + self.assertTrue(np.all(result >= 0) and np.all(result <= 1)) + + def test_bayesian_sign_test_a_better(self): + """Test bayesian_sign_test when algorithm A is clearly better.""" + result, _ = bayesian_sign_test(self.data_a_better, sample_size=2500) + + # Algorithm A should have high probability of being better (result[2] should be high) + self.assertGreater(result[2], 0.5, + "Algorithm A should have >50% probability of being better") + + def test_bayesian_sign_test_equivalent(self): + """Test bayesian_sign_test when algorithms are equivalent.""" + result, _ = bayesian_sign_test(self.data_equivalent, sample_size=2500) + + # Rope probability should be highest for equivalent algorithms + self.assertGreater(result[1], result[0], + "Rope probability should be higher than left") + self.assertGreater(result[1], result[2], + "Rope probability should be higher than right") + + def test_bayesian_sign_test_custom_rope(self): + """Test bayesian_sign_test with custom ROPE limits.""" + result, _ = bayesian_sign_test( + self.data_rope, + rope_limits=[-0.1, 0.1], + sample_size=2500 + ) + + # With wide ROPE, should classify as equivalent + self.assertGreater(result[1], 0.3, + "With wide ROPE, rope probability should be significant") + + def test_bayesian_sign_test_prior_strength(self): + """Test bayesian_sign_test with different prior strengths.""" + result_weak, _ = bayesian_sign_test( + self.data_a_better, + prior_strength=0.1, + sample_size=2500 + ) + result_strong, _ = bayesian_sign_test( + self.data_a_better, + prior_strength=5.0, + sample_size=2500 + ) + + # Both should still identify A as better, but with different confidence + self.assertGreater(result_weak[2], result_strong[2], + "Weaker prior should allow stronger data influence") + + def test_bayesian_sign_test_prior_place(self): + """Test bayesian_sign_test with different prior placements.""" + for prior_place in ["left", "rope", "right"]: + result, _ = bayesian_sign_test( + self.data_a_better, + prior_place=prior_place, + sample_size=1000 + ) + # Should still work and return valid probabilities + self.assertAlmostEqual(np.sum(result), 1.0, places=5) + + def test_bayesian_sign_test_invalid_prior_strength(self): + """Test bayesian_sign_test with invalid prior strength.""" + with self.assertRaises(ValueError): + bayesian_sign_test(self.data_a_better, prior_strength=0) + + with self.assertRaises(ValueError): + bayesian_sign_test(self.data_a_better, prior_strength=-1) + + def test_bayesian_sign_test_invalid_prior_place(self): + """Test bayesian_sign_test with invalid prior place.""" + with self.assertRaises(ValueError): + bayesian_sign_test(self.data_a_better, prior_place="invalid") + + def test_bayesian_sign_test_invalid_data_shape(self): + """Test bayesian_sign_test with invalid data shape.""" + invalid_data = pd.DataFrame({ + "A": [1, 2, 3], + "B": [4, 5, 6], + "C": [7, 8, 9] + }) + + with self.assertRaises(ValueError): + bayesian_sign_test(invalid_data) + + def test_bayesian_signed_rank_test_basic(self): + """Test basic functionality of bayesian_signed_rank_test.""" + result, samples = bayesian_signed_rank_test(self.data_a_better, sample_size=500) + + # Check return types + self.assertIsInstance(result, np.ndarray) + self.assertIsInstance(samples, np.ndarray) + + # Check result shape + self.assertEqual(result.shape, (3,)) + + # Check probabilities sum to 1 + self.assertAlmostEqual(np.sum(result), 1.0, places=5) + + # Check all probabilities are between 0 and 1 + self.assertTrue(np.all(result >= 0) and np.all(result <= 1)) + + def test_bayesian_signed_rank_test_a_better(self): + """Test bayesian_signed_rank_test when algorithm A is clearly better.""" + result, _ = bayesian_signed_rank_test(self.data_a_better, sample_size=500) + + # Algorithm A should have high probability of being better + self.assertGreater(result[2], 0.5, + "Algorithm A should have >50% probability of being better") + + def test_bayesian_signed_rank_test_equivalent(self): + """Test bayesian_signed_rank_test when algorithms are equivalent.""" + result, _ = bayesian_signed_rank_test(self.data_equivalent, sample_size=500) + + # Rope probability should be highest + self.assertGreater(result[1], 0.3, + "Rope probability should be significant for equivalent algorithms") + + def test_bayesian_signed_rank_test_invalid_prior_strength(self): + """Test bayesian_signed_rank_test with invalid prior strength.""" + with self.assertRaises(ValueError): + bayesian_signed_rank_test(self.data_a_better, prior_strength=0) + + def test_bayesian_signed_rank_test_invalid_prior_place(self): + """Test bayesian_signed_rank_test with invalid prior place.""" + with self.assertRaises(ValueError): + bayesian_signed_rank_test(self.data_a_better, prior_place="invalid") + + def test_bayesian_signed_rank_test_invalid_data_shape(self): + """Test bayesian_signed_rank_test with invalid data shape.""" + invalid_data = pd.DataFrame({ + "A": [1, 2], + "B": [4, 5], + "C": [7, 8] + }) + + with self.assertRaises(ValueError): + bayesian_signed_rank_test(invalid_data) + + def test_bayesian_tests_with_numpy_array(self): + """Test that both functions work with numpy arrays.""" + data_array = self.data_a_better.values + + result_sign, _ = bayesian_sign_test(data_array, sample_size=1000) + result_rank, _ = bayesian_signed_rank_test(data_array, sample_size=500) + + # Both should return valid probabilities + self.assertAlmostEqual(np.sum(result_sign), 1.0, places=5) + self.assertAlmostEqual(np.sum(result_rank), 1.0, places=5) + + def test_bayesian_tests_reproducibility(self): + """Test that setting random seed produces consistent results.""" + np.random.seed(42) + result1, _ = bayesian_sign_test(self.data_a_better, sample_size=1000) + + np.random.seed(42) + result2, _ = bayesian_sign_test(self.data_a_better, sample_size=1000) + + # Results should be identical with same seed + np.testing.assert_array_almost_equal(result1, result2, decimal=10) + + def test_bayesian_sample_size_effect(self): + """Test that larger sample sizes give more stable results.""" + result_small, _ = bayesian_sign_test(self.data_a_better, sample_size=100) + result_large, _ = bayesian_sign_test(self.data_a_better, sample_size=5000) + + # Both should identify the same winner, but may have different confidence + winner_small = np.argmax(result_small) + winner_large = np.argmax(result_large) + self.assertEqual(winner_small, winner_large, + "Both sample sizes should identify the same winner") + + +if __name__ == '__main__': + unittest.main() +