diff --git a/_static/fonts/NunitoSans-Italic-VariableFont.woff2 b/_static/fonts/NunitoSans-Italic-VariableFont.woff2 new file mode 100644 index 000000000..425c62139 Binary files /dev/null and b/_static/fonts/NunitoSans-Italic-VariableFont.woff2 differ diff --git a/_static/fonts/NunitoSans-VariableFont.woff2 b/_static/fonts/NunitoSans-VariableFont.woff2 new file mode 100644 index 000000000..e7abd6577 Binary files /dev/null and b/_static/fonts/NunitoSans-VariableFont.woff2 differ diff --git a/_static/pyos.css b/_static/pyos.css index 229d7cf27..7a9334b1b 100644 --- a/_static/pyos.css +++ b/_static/pyos.css @@ -4,7 +4,7 @@ --pyos-color-secondary: #8045e5; --pyos-color-secondary-highlight: #591bc2; --pyos-color-tertiary: #a66c98; - --pyos-color-dark: #542568; + --pyos-color-dark: #33205c; /* Brand dark purple - #33205c */ --pyos-color-light: #daabcf; /* Darkmode Adjustments*/ @@ -12,8 +12,9 @@ /* Fonts (overrides base theme) */ --pst-font-family-heading: "Poppins", sans-serif; - --pst-font-family-base: "Poppins", sans-serif; - --pyos-font-family-h1: "Itim", serif; + --pst-font-family-base: "NunitoSans", sans-serif; + --pyos-font-family-h1: "Poppins", sans-serif; + --pst-font-family-heading-weight: 400; } /* anything related to the dark theme */ @@ -32,6 +33,17 @@ body { font-size: 1rem; } +/* nav bar poppins font */ +.bd-header, +.bd-header .navbar-nav, +.bd-header .navbar-nav .nav-link, +.bd-header .navbar-header-items, +.bd-header .navbar-item, +.navbar-item .nav-item { + font-family: "Poppins", sans-serif !important; + font-weight: 400 !important; +} + /* Allow the center content to expand to wide on wide screens */ @media (min-width: 960px) { .bd-page-width { @@ -65,6 +77,10 @@ body p { margin-bottom: 70px !important; } +.admonition-title { + font-family: "Poppins", sans-serif !important; +} + h1 { margin-top: 10px; margin-bottom: 40px; @@ -73,6 +89,7 @@ h1 { } h2 { margin-top: 1em; + color: var(--pyos-h1-color); } h3 { @@ -94,7 +111,9 @@ figcaption { } .admonition p { - font-size: 0.9em; + font-size: 0.95em; + font-family: Poppins, sans-serif; + font-weight: 200; } /* Navbar */ @@ -198,12 +217,14 @@ html[data-theme="light"] { --pst-color-success-text: #fff; --pst-color-success-highlight: #00381a; --sd-color-success: var(--pst-color-success); + --pst-color-link: #735fab; + --pst-color-link-hover: #591bc2; --sd-color-success-text: var(--pst-color-success-text); --sd-color-success-highlight: var(--pst-color-success-highlight); --sd-color-success-bg: #d6ece1; --sd-color-success-bg-text: #14181e; - --pst-color-info: #a66c98; /* general admonition */ - --pst-color-info-bg: #eac8e2; + --pst-color-info: #33205c; /* general admonition */ + --pst-color-info-bg: #bab3d4; --pst-heading-color: var(--pyos-color-dark); --pyos-h1-color: var(--pyos-color-dark); } @@ -318,6 +339,25 @@ See https://github.com/pydata/pydata-sphinx-theme/pull/1784 font-weight: 600; src: url("./fonts/poppins-v20-latin-600.woff2") format("woff2"); /* Chrome 36+, Opera 23+, Firefox 39+, Safari 12+, iOS 10+ */ } + +/* nunitosans-regular - latin */ +@font-face { + font-display: swap; + font-family: "NunitoSans"; + font-style: normal; + font-weight: 400; + src: url("./fonts/NunitoSans-VariableFont.woff2") format("woff2"); +} + +/* nunitosans-italic - latin */ +@font-face { + font-display: swap; + font-family: "NunitoSans"; + font-style: italic; + font-weight: 400; + src: url("./fonts/NunitoSans-Italic-VariableFont.woff2") format("woff2"); +} + /* Cards */ /* todo light and dark adaptations needed */ @@ -329,6 +369,7 @@ See https://github.com/pydata/pydata-sphinx-theme/pull/1784 border-bottom: 2px solid #999; font-size: 1.2rem; font-weight: 600 !important; + font-family: "Poppins", sans-serif !important; } .sd-card-header { diff --git a/package-structure-code/code-style-linting-format.md b/package-structure-code/code-style-linting-format.md index e41056658..a14b2b298 100644 --- a/package-structure-code/code-style-linting-format.md +++ b/package-structure-code/code-style-linting-format.md @@ -1,6 +1,6 @@ # Python Package Code Style, Format and Linters -```{admonition} Take Aways +:::{admonition} Take Aways * pyOpenSci requires authors to follow PEP 8 code format guidelines * Setting up a code formatters like Black and isort will help you enforce PEP 8 style guidelines and also consistent, readable code format @@ -9,7 +9,7 @@ each time you make a commit. * [precommit.ci](https://pre-commit.ci/) is a bot that you can add to your GitHub repository. It will automagically apply code format to every PR using the tools specified in your pre-commit-config.yaml file. It can save significant time and make contributions easier for new contributors. * Automation is good! By making code quality tools care of your code, you can focus on structural and high values tasks. -``` +::: Consistent code format and style is useful to both your package and across the scientific Python ecosystem because using similar @@ -35,6 +35,7 @@ a discussion of: 1. Setting up pre-commit hooks and the pre-commit.ci bot to make using code format tools in daily workflows and in pull requests on GitHub easier. +(code-style-tools)= ## Use a code format tool (or tools) to make your life easier We suggest that you use a code format tool, or a set of format tools, because @@ -57,18 +58,18 @@ the work out of manually implementing code format requirements. Consistent code format across packages within the (scientific) Python ecosystem, will also broadly make code easier to scan, understand and contribute to. -## Linting vs format and style +## Linting vs. format and style Before we dive in let's get a few definitions out of the way. -### Code Linting +### Code linting A code linter is a tool that will review your code and identify errors or issues. A linter typically does not modify your code. It will tell you what the error is and on what line it was discovered. Flake8, discussed below, is an example of a commonly-used code linter. -### Code Formatters (and stylers) +### Code formatters (and stylers) Code formatters will reformat your code for you. Python focused code formatters often follow PEP 8 standards. However, they also make stylistic decisions about @@ -82,6 +83,7 @@ You will learn more about Black below. ## Code linting, formatting and styling tools +(about-black)= ### Black [Black](https://black.readthedocs.io/en/stable/) is a code @@ -94,14 +96,15 @@ some exceptions. A few examples of those exceptions are below: - Black will not adjust line length in your comments or docstrings. - This tool will not review and fix import order (you need `isort` or `ruff` to do that - see below). -```{tip} +:::{tip} If you are interested in seeing how Black will format your code, you can use the [Black playground](https://black.vercel.app/) -``` +::: Using a code formatter like Black will leave you more time to work on code function rather than worry about format. +(about-flake8)= ### Flake8 To adhere to Python `pep8` format standards, you might want to add @@ -186,6 +189,7 @@ Python file `temporal.py` imports after `isort` has been run :end-before: def calc_annual_mean ::: +(about-ruff)= ### Ruff [Ruff](https://docs.astral.sh/ruff/) is a new addition to the code quality @@ -272,13 +276,13 @@ on your the code files in your commit. It will update any files to match black format standards. You can then retype the commit and push files to GitHub that have been formatted by black. --> -```{important} +:::{important} If have a Python code-base and multiple maintainers actively working on the code, and you intend to run a tool like Black, be sure to coordinate across your team. An initial commit that applies Black to your entire package will likely change a significant amount of your code. This could lead to merge conflicts on open and new PR's before the new changes are merged. -``` +::: ## General pre commit checks diff --git a/package-structure-code/complex-python-package-builds.md b/package-structure-code/complex-python-package-builds.md index 0fb8c9101..fb0619bdd 100644 --- a/package-structure-code/complex-python-package-builds.md +++ b/package-structure-code/complex-python-package-builds.md @@ -3,9 +3,11 @@ This guide is focused on packages that are either pure-python or that have a few simple extensions in another language such as C or C++. -In the future, we want to provide resources for packaging workflows that require more complex builds. If you have questions about these types of package, please [add a question to our discourse](https://pyopensci.discourse.group/) or open an [issue about this guide specifically in the GitHub repo for this guide](https://github.com/pyOpenSci/python-package-guide/issues). There are many nuances to building and distributing Python packages that have compiled extensions requiring non-Python dependencies at build time. For an overview and thorough discussion of these nuances, please see [this site.](https://pypackaging-native.github.io/) +For comprehensive guidance on packaging compiled projects with C/C++/Fortran/Rust extensions, see the [Scientific Python Development Guide on compiled packaging](https://learn.scientific-python.org/development/guides/packaging-compiled/). This is the best reference for complex builds and covers scikit-build-core, meson-python, maturin, and other modern build backends. -## Pure Python Packages vs. packages with extensions in other languages +If you have questions about these types of package, please open an [issue about this guide specifically in the GitHub repo for this guide](https://github.com/pyOpenSci/python-package-guide/issues). There are many nuances to building and distributing Python packages that have compiled extensions requiring non-Python dependencies at build time. For an overview and thorough discussion of these nuances, please see [this site.](https://pypackaging-native.github.io/) + +## Pure Python packages vs. packages with extensions in other languages You can classify Python package complexity into three general categories. These categories can in turn help you select the correct package frontend and diff --git a/package-structure-code/declare-dependencies.md b/package-structure-code/declare-dependencies.md index 039cbe7df..56519a44c 100644 --- a/package-structure-code/declare-dependencies.md +++ b/package-structure-code/declare-dependencies.md @@ -10,6 +10,7 @@ you learned how to set up a **pyproject.toml** file with basic metadata for your package. On this page, you will learn how to specify different types of dependencies in your `pyproject.toml`. +(package-dependencies)= ## What is a package dependency? A Python package dependency refers to an external package or @@ -19,6 +20,7 @@ A tool that is needed when using or working on your Python project. Declare your :class: tip While `pyproject.toml` is now the standard, you may sometimes encounter older approaches to storing dependencies "in the wild": + - **requirements.txt**: Previously common for dependencies, still used by some projects for local development - **setup.py or setup.cfg**: May be needed for packages with extensions in other languages @@ -63,6 +65,7 @@ A dependency is not part of your project's codebase. It is a package or software within the code of your project or used during the development of your package. ::: +(required-dependencies)= ## 1. Required dependencies Required dependencies are imported and called directly within your package's code. @@ -91,7 +94,6 @@ fewer dependencies reduce the possibility of version conflicts in user environments. ::: - ::::{dropdown} How to Add Required Dependencies with UV :icon: eye :color: primary @@ -101,7 +103,7 @@ You can use uv to add dependencies to your pyproject.toml file: **Add a required dependency:** ```bash -$ uv add numpy +uv add numpy ``` Will add numpy as a dependency to your `project.dependency` array: @@ -127,12 +129,14 @@ dependencies = [ "my_dependency >= 1.0.1 @ git+https://git.server.example.com/mydependency.git@commitHashHere" ] ``` + IMPORTANT: If your library depends on a GitHub-hosted project, you should point to a specific commit/tag/hash of that repository before you upload your project to PyPI. You never know how the project might change over time. Commit hashes are more reliable as they can't be changed ::: +(optional-dependencies)= ## 2. Optional dependencies Optional (also referred to as feature) dependencies can be installed by users as needed. Optional dependencies add specific features to your package that not all users need. For example, if your package has an optional interactive plotting feature that uses Bokeh, you would list Bokeh as an `[optional.dependency]`. Users who want interactive plotting will install it. Users who don't need plotting don't have to install it. @@ -163,23 +167,26 @@ uv add --optional feature pandas ``` Will add this to your pyproject.toml file: + ```toml [optional.dependencies] feature = [ "pandas>=2.3.3", ] ``` + ::: +(dependency-groups)= ## 3. Dependency groups Development dependencies include packages needed to work on your package locally. They are used to perform tasks such as: -* running your test suite (pytest, pytest-cov) -* building your documentation (sphinx, sphinx-theme packages) -* linting and formatting code (ruff, black) -* building package distribution files (build, twine) +- running your test suite (pytest, pytest-cov) +- building your documentation (sphinx, sphinx-theme packages) +- linting and formatting code (ruff, black) +- building package distribution files (build, twine) Dependency groups are optional because they are not required for users to install and use your package. However, they will make it easier for @@ -215,10 +222,12 @@ lint = ["ruff", "black"] You can use uv to add dependencies to your pyproject.toml file: **Add a development group dependency:** + ```bash uv add --group tests pytest uv add --group docs sphinx ``` + Will add the following to your pyproject.toml file: ```toml @@ -230,6 +239,7 @@ docs = [ "sphinx>=8.1.3", ] ``` + ::: :::{todo} @@ -252,13 +262,10 @@ dependencies that users need to run your package, and **optional** dependencies for development work or additional features. ::: - - - :::{admonition} Additional dependency resources -* [Learn more: View PyPA's overview of declaring optional dependencies](https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#dependencies-optional-dependencies) -* [Dependency specifiers](https://packaging.python.org/en/latest/specifications/dependency-specifiers/) +- [Learn more: View PyPA's overview of declaring optional dependencies](https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#dependencies-optional-dependencies) +- [Dependency specifiers](https://packaging.python.org/en/latest/specifications/dependency-specifiers/) ::: @@ -289,7 +296,6 @@ You can also use pip and install dependencies into the environment of your choic We shouldn't show UV pip install, so how do you add optional feature deps with UV?? ::: - **Install development groups:** :::::{tab-set} @@ -298,9 +304,9 @@ We shouldn't show UV pip install, so how do you add optional feature deps with U You can use uv sync to sync dependency groups in your uv-managed venv ```console -$ uv sync --group docs # Single group -$ uv sync --group docs --group test # Multiple groups -$ uv sync --all-groups # All development groups +uv sync --group docs # Single group +uv sync --group docs --group test # Multiple groups +uv sync --all-groups # All development groups ``` **Install optional dependencies:** @@ -314,7 +320,7 @@ $ uv pip install -e ".[docs,tests,lint]" # Multiple groups **Install everything (package + all dependencies):** ```console -$ uv sync --all-extras --all-groups +uv sync --all-extras --all-groups ``` `uv sync` is the recommended command for development workflows. It @@ -330,6 +336,7 @@ Use `uv pip install` when you need pip-compatible behavior. python -m pip install -e ".[docs]" # Single group python -m pip install -e ".[docs,tests,lint]" # Multiple groups ``` + **Install dependency groups:** ```console @@ -350,8 +357,6 @@ installation conflicts. ::::: - - ### Combining dependency groups You can also create combined groups that reference other groups: @@ -364,6 +369,7 @@ dev = ["your-package[test,docs]", "build", "twine"] ``` Then install everything with pip install or uv sync as needed: + ```bash uv pip install -e ".[dev]" # or @@ -375,7 +381,6 @@ When you install optional dependencies, pip and uv install your package and its core dependencies automatically. ::: - ## Version specifiers for dependencies Version specifiers control which versions of a dependency work with your @@ -395,6 +400,7 @@ set version ranges. avoid upper bounds unless you know at what version that dependency is no longer compatible. UV will do this by default when it adds a dependency to your pyproject.toml file. This keeps your package flexible and reduces dependency conflicts. + ```toml dependencies = [ "numpy>=1.20", # Good - flexible @@ -402,8 +408,8 @@ dependencies = [ "requests==2.28.0", # Avoid - too restrictive ] ``` -::: +::: :::{todo} ### Using conda and Pixi @@ -469,6 +475,7 @@ your dependencies during the build process, configure them in a **readthedocs.yaml** file. Here's an example that installs your `docs` optional dependencies: + ```yaml python: install: @@ -481,18 +488,17 @@ python: :::{admonition} Learn more about Read the Docs :class: note -* [Creating a readthedocs.yaml file](https://docs.readthedocs.io/en/stable/config-file/index.html) -* [Using uv with Read the Docs](https://docs.readthedocs.io/en/stable/build-customization.html) -* [Using Poetry with Read the Docs](https://docs.readthedocs.io/en/stable/build-customization.html#install-dependencies-with-poetry) +- [Creating a readthedocs.yaml file](https://docs.readthedocs.io/en/stable/config-file/index.html) +- [Using uv with Read the Docs](https://docs.readthedocs.io/en/stable/build-customization.html) +- [Using Poetry with Read the Docs](https://docs.readthedocs.io/en/stable/build-customization.html#install-dependencies-with-poetry) ::: - :::{todo} -Keep this comment - https://github.com/pyOpenSci/python-package-guide/pull/106#issuecomment-1844278487 in this file for now - Jeremiah did a nice inventory of common shells and whether they need quotes or not. It's really comprehensive. But do we want it in the guide?? It's really useful for more advanced users. +Keep this comment - in this file for now - Jeremiah did a nice inventory of common shells and whether they need quotes or not. It's really comprehensive. But do we want it in the guide?? It's really useful for more advanced users. Following this comment: -https://github.com/pyOpenSci/python-package-guide/pull/106#pullrequestreview-1766663571 + Jonny will add a section that talks about: diff --git a/package-structure-code/intro.md b/package-structure-code/intro.md index bc6c9b03d..ce0be5ab9 100644 --- a/package-structure-code/intro.md +++ b/package-structure-code/intro.md @@ -1,176 +1,183 @@ -# Python Package Structure +# Python Package Structure & Code -This section provides guidance on your Python package's structure, code format, -and style. It also reviews the various [packaging tools](python-package-build-tools) you can use to -[build](python-package-distribution-files-sdist-wheel) and [publish](publish-python-package-pypi-conda) your Python package. +This section covers everything you need to structure your Python package, configure metadata, choose build tools, and publish your package to PyPI and conda-forge. -If you want end-to-end tutorials, check out our tutorial series that starts by introducing [what a Python package is](what-is-a-package). +:::::{grid} 1 2 +:gutter: 3 + +::::{grid-item} +:::{card} New to Python packaging? +:class-card: sd-shadow-sm + +**Start with our step-by-step tutorials:** -If you are confused by Python packaging, you are not alone! The good news is -that some great modern packaging tools ensure you follow -best practices. Here, we review tool features and suggest tools you can use -for your Python packaging workflow. +- Follow along as we create a package from scratch +- Learn by doing with guided examples +- Perfect for your first package -:::{button-link} /tutorials/intro +```{button-link} /tutorials/intro :color: success -:class: sd-rounded-pill float-left +:class: sd-rounded-pill -Checkout our beginning-to-end create a Python package tutorials +Start the tutorial series +``` ::: +:::: + +::::{grid-item} +:::{card} Already have code to package? +:class-card: sd-shadow-sm + +**Jump into the reference guides:** + +- Learn about package structure and metadata +- Compare build tools and choose what's right for you +- Understand the publishing process + +Start with the cards below ↓ +::: +:::: + +::::: :::{admonition} How this content is developed +:class: note + All of the content in this guide has been vetted by community members, including maintainers and developers of the core packaging tools. ::: +## What you'll learn + +In this section, you'll learn how to: + +- **Structure your package** - Choose between src and flat layouts, organize tests and documentation +- **Configure metadata** - Set up `pyproject.toml` with project information, dependencies, and versioning +- **Choose build tools** - Compare Hatch, PDM, Poetry, and setuptools to find the right fit +- **Build distributions** - Create sdist and wheel files ready for publication +- **Publish your package** - Make your package available on PyPI and optionally conda-forge +- **Maintain code quality** - Set up linters and formatters to keep your code consistent + +Our recommendations align with current [Python packaging standards](https://packaging.python.org/en/latest/) and [Scientific Python community specs](https://scientific-python.org/specs/), while prioritizing tools that are beginner-friendly and well-maintained. + +## Package setup + :::::{grid} 1 1 2 2 :class-container: text-center :gutter: 3 ::::{grid-item} -:::{card} ✨ 1. Package file structure ✨ -:link: python-package-structure -:link-type: doc +:::{card} ✨ Package file structure ✨ -src layout, flat layout and where should tests folders live? No matter what your level of packaging knowledge is, this page will help you decide upon a package structure that follows modern python best practices. +Learn how to organize your package files using [src or flat layouts](package-source-layout). This page helps you decide on a package structure that follows modern Python best practices, including where to place [tests](src-layout-test) and [documentation](package-source-layout). ::: :::: ::::{grid-item} -:::{card} ✨ 2. Learn about building your package ✨ -:link: python-package-distribution-files-sdist-wheel -:link-type: doc - -To publish your Python package on PyPI, you will need to first build it. The act -of "building" refers to the process of placing your package code and -metadata into a format that can be published on PyPI. Learn more about building -your Python package. +:::{card} ✨ Add metadata ✨ + +Learn how to add [project metadata](pyproject-toml-python-package-metadata) to your Python package to support both +filtering on PyPI and also the metadata that a package installer needs to +build and install your package. ::: :::: ::::{grid-item} -:::{card} ✨ 3. Add metadata ✨ -:link: pyproject-toml-python-package-metadata -:link-type: doc +:::{card} ✨ Declare dependencies ✨ -Learn how to add project metadata to your Python package to support both -filtering on PyPI and also the metadata that a package installer needs to -build and install your package. +Learn how to specify [required dependencies](required-dependencies), [optional feature dependencies](optional-dependencies), and [development dependencies](dependency-groups) in your [pyproject.toml file](pyproject-toml-overview). +::: +:::: + +::::{grid-item} +:::{card} ✨ Setup package versioning ✨ + +Learn how to manage package versions using [semantic versioning (SemVer)](package-versioning) or [calendar versioning (CalVer)](package-versioning). This page helps you choose the right versioning strategy and set up [automated version management](tools-version-management) using tools like hatch_vcs or setuptools-scm. ::: :::: +::::: + +## Development practices + +:::::{grid} 1 1 2 2 +:class-container: text-center +:gutter: 3 + ::::{grid-item} -:::{card} ✨ 4. What Python package tool should you use? ✨ -:link: python-package-build-tools -:link-type: doc +:::{card} ✨ Code style & linters ✨ -Learn more about the suite of packaging tools out there. -And learn which tool might be best for you. +Learn how to set up [code formatters and linters](code-style-tools) ([Black](about-black), [Ruff](about-ruff), [flake8](about-flake8)) to ensure your package follows [PEP 8 standards](code-style-tools) and maintains consistent code style throughout your project. ::: :::: +::::: + +## Build & publish + +:::::{grid} 1 1 2 2 +:class-container: text-center +:gutter: 3 + ::::{grid-item} -:::{card} ✨ 5. Publish to PyPI and Conda ✨ -:link: publish-python-package-pypi-conda -:link-type: doc +:::{card} ✨ Choose your build tool ✨ -If you have a pure Python package, it's a straight forward -process to publish to both PyPI and then a Conda channel such as -conda-forge. Learn more here. +Learn how to choose the right packaging tool for your project. Compare [Hatch](about-hatch), [PDM](about-pdm), [Poetry](about-poetry), and [setuptools](about-setuptools) to find the best fit for your workflow. See the [summary comparison](summary-build-tools) to help decide. ::: :::: ::::{grid-item} -:::{card} ✨ 6. Setup package versioning ✨ -:link: python-package-versions -:link-type: doc +:::{card} ✨ Build your package ✨ -Semver (numeric versioning) and Calver (versioning using the date) are 2 -common ways to version a package. Which one should you pick? Learn more here. +Learn how to build your Python package into [distribution files](build-package) ([sdist](python-source-distribution) and [wheel](python-wheel)) that can be published on [PyPI](publish-pypi-conda). ::: :::: ::::{grid-item} -:::{card} ✨ 7. Code style & linters ✨ -:link: code-style-linting-format -:link-type: doc +:::{card} ✨ Publish to PyPI and Conda ✨ -Black, blue, flake8, Ruff - which tools can help you ensure your -package follows best practices for code format? Learn more about the options -and why this is important here. +Learn how to publish your package to [PyPI](publish-pypi-conda) and optionally to [conda-forge](how-to-submit-to-conda-forge). This page covers the complete process for making your package available to users, including the [conda-forge submission process](how-to-submit-to-conda-forge) after publishing to PyPI. ::: :::: ::::: +## Choosing the right tools + +Not sure which build tool to use? This decision tree can help you choose based on your package's needs: + :::{figure-md} packaging-tools-decision-tree Figure showing a decision tree with the various packaging tool front-end and back-end options. -Diagram showing the various front-end build tools that you can select from. -See the packaging tools page to learn more about each tool. +Use this decision tree to help select a packaging tool. See the [packaging tools page](python-package-build-tools) for detailed comparisons and recommendations. ::: -:::{note} -If you are considering submitting a package for peer review, have a look -at the bare-minimum [editor checks](https://www.pyopensci.org/software-peer-review/how-to/editor-in-chief-guide.html#editor-checklist-template) -that pyOpenSci performs before a review begins. These checks are useful -to explore for both authors planning to submit a package to us for review -and for anyone who is just getting started with creating a Python package. -::: +## Our recommendations + +We suggest tools and approaches based on three principles: + +1. **Beginner-friendly** - Tools that are easy to learn and use for those new to packaging +2. **Well-maintained** - Tools with active development and good documentation +3. **Standards-aligned** - Tools that follow current [Python packaging standards](https://packaging.python.org/en/latest/) and [Scientific Python community specs](https://scientific-python.org/specs/) + +### Pure Python vs. complex builds + +- **Pure Python packages** can use any modern tool (Hatch, PDM, Poetry, Flit) - choose based on the features you want +- **Packages with C/C++ extensions** may need additional build steps. See our [complex builds page](complex-python-package-builds) for guidance. For comprehensive information on packaging compiled projects, see the [Scientific Python Development Guide on compiled packaging](https://learn.scientific-python.org/development/guides/packaging-compiled/). -## What you will learn here - -In this section of our Python packaging guide, we: - -- Provide an overview of the options available to you when packaging your - code. -- Suggest tools and approaches that both meet your needs and also support - existing standards. -- Suggest tools and approaches that will allow you to expand upon a workflow - that may begin as a pure Python code and evolve into code that requires - addition layers of complexity in the packaging build. -- Align our suggestions with the most current, accepted - [PEPs (Python Enhancement Protocols)](https://peps.python.org/pep-0000/) - and the [Scientific Python community SPECs](https://scientific-python.org/specs/). -- In an effort to maintain consistency within our community, we also align - with existing best practices being implemented by developers of core - Scientific Python packages such as Numpy, SciPy and others. - -## Guidelines for pyOpenSci's packaging recommendations - -The flexibility of the Python programming language lends itself to a diverse -range of tool options for creating a Python package. Python is so flexible that -it is one of the few languages that can be used to wrap around other languages. -The ability of Python to wrap other languages is one the reasons you will often -hear Python described as a ["glue" language](https://numpy.org/doc/stable/user/c-info.python-as-glue.html)" - -If you are building a pure Python package, then your packaging setup can be -simple. However, some scientific packages have complex requirements as they may -need to support extensions or tools written in other languages such as C or C++. - -To support the many different uses of Python, there are many ways to create a -Python package. In this guide, we suggest packaging approaches and tools based on: - -1. What we think will be best and easiest to adopt for those who are newer to - packaging. -2. Tools that we think are well maintained and documented. -3. A shared goal of standardizing packaging approaches across this (scientific) - Python ecosystem. - -Here, we also try to align our suggestions with the most current, accepted -[Python community](https://packaging.python.org/en/latest/) and [scientific community](https://scientific-python.org/specs/). - -:::{admonition} Suggestions in this guide are not pyOpenSci review requirements -:class: important - -The suggestions for package layout in this section are made with the -intent of being helpful; they are not specific requirements for your -package to be reviewed and accepted into our pyOpenSci open source ecosystem. - -Please check out our [package scope page](https://www.pyopensci.org/software-peer-review/about/package-scope.html) -and [review requirements in our author guide](https://www.pyopensci.org/software-peer-review/how-to/author-guide.html#) -if you are looking for pyOpenSci's Python package review requirements! +Most scientific Python packages start simple and can evolve to handle more complex requirements as needed. + +## Submitting your package for peer review? + +If you're planning to submit your package to pyOpenSci for [peer review](https://www.pyopensci.org/about-peer-review/index.html), check out our [editor checklist](https://www.pyopensci.org/software-peer-review/how-to/editor-in-chief-guide.html#editor-checklist-template) for the minimum requirements. These checks are useful for anyone creating a Python package, not just those submitting for review. + +:::{admonition} These are recommendations, not requirements +:class: tip + +The suggestions in this guide are designed to help you create a well-structured package. They are **not** specific requirements for pyOpenSci peer review. + +If you're submitting to pyOpenSci, see our [package scope](https://www.pyopensci.org/software-peer-review/about/package-scope.html) and [author guide](https://www.pyopensci.org/software-peer-review/how-to/author-guide.html#) for actual review requirements. ::: :::{toctree} @@ -181,9 +188,9 @@ Intro Python package structure pyproject.toml Package Metadata -Build Your Package Declare dependencies Package Build Tools +Build Your Package Complex Builds ::: diff --git a/package-structure-code/publish-python-package-pypi-conda.md b/package-structure-code/publish-python-package-pypi-conda.md index 9cd5208bd..881d321e7 100644 --- a/package-structure-code/publish-python-package-pypi-conda.md +++ b/package-structure-code/publish-python-package-pypi-conda.md @@ -18,7 +18,6 @@ lead to package conflicts. Below you will learn more specifics about the differences between PyPI and conda publishing of your Python package. ::: - :::{figure-md} upload-conda-forge Image showing the progression of creating a Python package, building it and then publishing to PyPI and conda-forge. You take your code and turn it into distribution files (sdist and wheel) that PyPI accepts. Then there is an arrow towards the PyPI repository where ou publish both distributions. From PyPI if you create a conda-forge recipe you can then publish to conda-forge. @@ -26,6 +25,7 @@ Below you will learn more specifics about the differences between PyPI and conda Once you have published both package distributions (the source distribution and the wheel) to PyPI, you can then publish to conda-forge. The conda-forge requires a source distribution on PyPI in order to build your package on conda-forge. You do not need to rebuild your package to publish to conda-forge. ::: +(publish-pypi-conda)= ## What is PyPI [PyPI](https://pypi.org/) is an online Python package repository that @@ -47,7 +47,6 @@ only install Python packages. Click here for a tutorial on publishing your package to PyPI. ::: - :::{tip} On the package build page, we discussed the [two package distribution types that you will create when making a Python package](python-package-distribution-files-sdist-wheel): SDist (packaged as a .tar.gz or .zip) and @@ -95,9 +94,9 @@ channels. The conda package manager can install packages from different channels There are several core public channels that most people use to install packages using conda, including: -- **defaults:** this is a channel managed by Anaconda. It is the version of the Python packages that you will install if you install the Anaconda Distribution. Anaconda (the company) decides what packages live on the `defaults` channel. -- [**conda-forge:**](https://conda-forge.org/) this is a community-driven channel that focuses on scientific packages. This channel is ideal for tools that support geospatial data. Anyone can publish a package to this channel. -- [**bioconda**](https://bioconda.github.io/): this channel focuses on biomedical tools. +* **defaults:** this is a channel managed by Anaconda. It is the version of the Python packages that you will install if you install the Anaconda Distribution. Anaconda (the company) decides what packages live on the `defaults` channel. +* [**conda-forge:**](https://conda-forge.org/) this is a community-driven channel that focuses on scientific packages. This channel is ideal for tools that support geospatial data. Anyone can publish a package to this channel. +* [**bioconda**](https://bioconda.github.io/): this channel focuses on biomedical tools. **conda-forge** emerged as many of the scientific packages did not exist in the `defaults` Anaconda channel. @@ -109,7 +108,6 @@ exist in the `defaults` Anaconda channel. Conda channels represent various repositories that you can install packages from. Because conda-forge is community maintained, anyone can submit a recipe there. PyPI is also a community maintained repository. Anyone can submit a package to PyPI and test PyPI. Unlike conda-forge there are no manual checks of packages submitted to PyPI. ::: - ## conda channels, PyPI, conda, pip - Where to publish your package You might be wondering why there are different package repositories @@ -155,6 +153,7 @@ consider publishing to both PyPI and the conda-forge channel (_more on that below_). :::{admonition} Additional resources + * [learn more about why conda-forge was created, here](https://conda-forge.org/docs/user/introduction.html) * [To learn more about conda terminology, check out their glossary.](https://docs.conda.io/projects/conda/en/latest/glossary.html ) @@ -162,6 +161,7 @@ on that below_). +(how-to-submit-to-conda-forge)= ## How to submit to conda-forge While pyOpenSci doesn't require you to add your package to conda-forge, @@ -171,7 +171,6 @@ Once your package is on PyPI, the process to add your package to conda-forge is straight forward to do. [You can follow the detailed steps provided by the conda-forge maintainer team.](https://conda-forge.org/docs/maintainer/adding_pkgs.html). - :::{button-link} ../tutorials/publish-conda-forge.html :color: primary :class: sd-rounded-pill float-left @@ -181,7 +180,6 @@ Click here for a tutorial on adding your package to conda-forge. If you want a step by step tutorial, click here. - Once your package is added, you will have a feedstock repository on GitHub with your packages name :::{tip} diff --git a/package-structure-code/pyproject-toml-python-package-metadata.md b/package-structure-code/pyproject-toml-python-package-metadata.md index 9eb2317e4..a35a0faaf 100644 --- a/package-structure-code/pyproject-toml-python-package-metadata.md +++ b/package-structure-code/pyproject-toml-python-package-metadata.md @@ -46,6 +46,7 @@ Click here if need help migrating from setup.py/setup.cfg to pyproject.toml :::: ::::: +(pyproject-toml-overview)= ## About the pyproject.toml file Every modern Python package should include a `pyproject.toml` file. For pure Python packages, this file replaces the `setup.py` and/or `setup.cfg` file to describe project metadata. @@ -107,7 +108,7 @@ resources working with complex builds in the future. ::: -## Optional vs. Required pyproject.toml file fields +## Optional vs. required pyproject.toml file fields When you create your `pyproject.toml` file, there are numerous metadata fields that you can use. Below we suggest specific fields to get you started that support publication on PyPI and users finding your package. @@ -176,7 +177,7 @@ Following the above example, you install dependencies like this: - `python -m pip install -e .[tests]` -- pip install --group test *# requires pip 25.1 or greater* +- pip install --group test _# requires pip 25.1 or greater_ The above will install both your package in editable mode and all of the dependencies declared in the tests section of your `[project.optional-dependencies]` table. @@ -224,6 +225,6 @@ of values. It has two keys that specify the build backend API and containing pac :language: toml ::: -```{note} +:::{note} [Click here to read about our packaging build tools including PDM, setuptools, Poetry and Hatch.](/package-structure-code/python-package-build-tools) -``` +::: diff --git a/package-structure-code/python-package-build-tools.md b/package-structure-code/python-package-build-tools.md index 81958e1f6..cd13fbec4 100644 --- a/package-structure-code/python-package-build-tools.md +++ b/package-structure-code/python-package-build-tools.md @@ -1,5 +1,6 @@ # Python Packaging Tools +(build-tools-overview)= ## Tools for building your package There are a several different build tools that you can use to [create your Python package's _sdist_ and _wheel_ distributions](python-package-distribution-files-sdist-wheel). Below, we discuss the features, @@ -30,6 +31,7 @@ You will learn more about the following tools on this page: - [PDM](https://pdm-project.org/latest/) - [Poetry](https://python-poetry.org/docs/) +(summary-build-tools)= ## Summary of tools Hatch vs. PDM vs. Poetry (and setuptools) If you are looking for a quick summary, read below. @@ -96,9 +98,9 @@ to both build and publish your package to PyPI. However while **Hatch** and locking, you can use **PDM** or **Poetry** but not **Hatch**. If you only need to build your package's sdist and wheel distribution files, then you can stick with PyPA's Build. You'd then use Twine to publish to PyPI. -```{note} +:::{note} If you are using **Setuptools**, there is no default user-friendly build front-end that performs multiple tasks. You will need to use **build** to build your package and **twine** to publish to PyPI. -``` +::: ### Example build steps that can be simplified using a front-end tool @@ -151,7 +153,7 @@ Below we introduce several of the most commonly used Python packaging build front-end tools. We highlight the features that each tool offers as a way to help you decide what tool might be best for your workflow. -```{admonition} We do not suggest using setuptools +:::{admonition} We do not suggest using setuptools :class: note We suggest that you pick one of the modern tools listed above rather than @@ -160,7 +162,7 @@ to set up correctly. We review setuptools as a back-end because it is still popular. However it is not the most user friendly option. -``` +::: The most commonly used tools in the ecosystem are setuptools back-end (with build) and Poetry (a front end tool with numerous @@ -227,21 +229,22 @@ Notes: - _Hatch plans to support dependency management in the future_ - Poetry supports semantic versioning. Thus, it will support version bumping following commit messages if you use a tool such as Python Semantic Release +(about-pdm)= ## PDM [PDM is a Python packaging and dependency management tool](https://pdm-project.org/latest/). PDM supports builds for pure Python projects. It also provides multiple layers of support for projects that have C and C++ extensions. -```{admonition} PDM support for C and C++ extensions +:::{admonition} PDM support for C and C++ extensions PDM supports using the PDM-back-end and setuptools at the same time. This means that you can run setuptools to compile and build C extensions. PDM's build back-end receives the compiled extension files (.so, .pyd) and packages them with the pure Python files. -``` +::: -### PDM Features +### PDM features ```{csv-table} :header: Feature|PDM|Notes @@ -261,18 +264,18 @@ Install your package in editable mode|✅|PDM supports installing your package i Build your sdist and wheel distributions|✅| Similar to all of the other tools PDM builds your packages sdist and wheel files for you. ``` -```{admonition} PDM vs. Poetry +:::{admonition} PDM vs. Poetry The functionality of PDM is similar to Poetry. However, PDM also offers additional, documented support for C extensions and version control based versioning. As such, PDM is preferred for those working on non pure-Python packages. If you are deciding between the Poetry and PDM, a smaller difference is the default way that dependencies are added to your pyproject.toml file. -* Poetry by default follows strict semantic versioning adding dependencies to your pyproject.toml file [using an upper bounds constraint (`^`)](https://python-poetry.org/docs/dependency-specification/#version-constraints). Upper bounds lock means that Poetry will never bump a dependency to the next major version (i.e. from 1.2 to 2.0). However, you can tell Poetry to use an open bound approach by explicitly adding the package like this: `poetry add requests >= 1.2` rather than just using `poetry add requests` which will result in a upper bound locked (ie Upper bound locks means that requests 2.0 could never be installed even if it came out and your package could benefit from it). -* PDM defaults to open-bounds (`>=`) dependency additions which is the preferred approach in the scientific python ecosystem. However, PDM also allows you to specify the way dependencies are added by default. As such, you can also specify upper-bounds (`^`) using PDM if require that approach. +- Poetry by default follows strict semantic versioning adding dependencies to your pyproject.toml file [using an upper bounds constraint (`^`)](https://python-poetry.org/docs/dependency-specification/#version-constraints). Upper bounds lock means that Poetry will never bump a dependency to the next major version (i.e. from 1.2 to 2.0). However, you can tell Poetry to use an open bound approach by explicitly adding the package like this: `poetry add requests >= 1.2` rather than just using `poetry add requests` which will result in a upper bound locked (ie Upper bound locks means that requests 2.0 could never be installed even if it came out and your package could benefit from it). +- PDM defaults to open-bounds (`>=`) dependency additions which is the preferred approach in the scientific python ecosystem. However, PDM also allows you to specify the way dependencies are added by default. As such, you can also specify upper-bounds (`^`) using PDM if require that approach. Finally there are some nuanced differences in how both tools create lock files which we will not go into detail about here. -``` +::: ### Challenges with PDM @@ -294,7 +297,7 @@ an overview of what the PDM command line interface looks like when you use it. Flit is a great choice if you are building a basic package to use in a local workflow that doesn't require any advanced features. And if your package structure is already created. More on that below. -### Flit Features +### Flit features ```{csv-table} :header: Feature|Flit|Notes @@ -310,9 +313,10 @@ Build your sdist and wheel distributions|✅| Flit can be used to build your pac NOTE: _If you are using the most current version of pip, it supports both a symlink approach `flit install -s` and `python -m pip install -e .`_ -```{admonition} Learn more about flit -* [Why use flit?](https://flit.pypa.io/en/stable/rationale.html) -``` +:::{admonition} Learn more about flit + +- [Why use flit?](https://flit.pypa.io/en/stable/rationale.html) +::: ### Why you might not want to use Flit @@ -328,6 +332,7 @@ You may NOT want to use flit if: - You want environment management (use PDM, Hatch or Poetry) (hatch)= +(about-hatch)= ## Hatch [**Hatch**](https://hatch.pypa.io/latest/), similar to Poetry and PDM, provides a @@ -371,16 +376,17 @@ These include: - Similar to PDM, Hatch's documentation can difficult to work through, particularly if you are just getting started with creating a package. - Hatch, similar to PDM and Flit currently only has one maintainer. +(about-poetry)= ## Poetry [Poetry is a full-featured build tool.](https://python-poetry.org/) It is also the second most popular front-end packaging tool (based upon the PyPA survey). Poetry is user-friendly and has clean and easy-to-read documentation. -```{note} +:::{note} While some have used Poetry for Python builds with C/C++ extensions, this support is currently undocumented. Thus, we don't recommend using Poetry for more complex builds. -``` +::: ### Poetry features @@ -417,7 +423,7 @@ Poetry is a popular packaging tool and introduced many very useful features. How -```{admonition} Challenges with Poetry dependency pinning +:::{admonition} Challenges with Poetry dependency pinning :class: important By default, Poetry pins dependencies using `^` by default. This `^` symbol means that there is @@ -433,9 +439,10 @@ changes in the tool. However, not all tools follow strict semantic versioning. This approach also won't support others ways of versioning tools, for instance, some tools use [calver](https://calver.org/) which creates new versions based on the date. -``` +::: -## Using Setuptools Back-end for Python Packaging with Build Front-end +(about-setuptools)= +## Using Setuptools back-end for Python packaging with Build front-end [Setuptools](https://setuptools.pypa.io/en/latest/) is the most mature Python packaging build tool with [development dating back to 2009 and earlier](https://setuptools.pypa.io/en/latest/history.html#). @@ -451,7 +458,7 @@ to consider using a more modern tool for packaging such as Poetry, Hatch or PDM. We discuss setuptools here because it's commonly found in the ecosystem and contributors may benefit from understanding it. -### Setuptools Features +### Setuptools features Some of features of setuptools include: diff --git a/package-structure-code/python-package-distribution-files-sdist-wheel.md b/package-structure-code/python-package-distribution-files-sdist-wheel.md index 79cf26d2e..323c6a086 100644 --- a/package-structure-code/python-package-distribution-files-sdist-wheel.md +++ b/package-structure-code/python-package-distribution-files-sdist-wheel.md @@ -69,11 +69,11 @@ classifiers = [ ] ``` -```{admonition} What happened to setup.py and setup.cfg for metadata? +:::{admonition} What happened to setup.py and setup.cfg for metadata? :class: note Project metadata used to be stored in either a setup.py file or a setup.cfg file. The current recommended practice for storing package metadata is to use a pyproject.toml file. [Learn more about the pyproject.toml file here.](pyproject-toml-python-package-metadata) -``` +::: ### An example - xclim @@ -115,7 +115,7 @@ Maintainer names and GitHub usernames for the xclim package as they are displaye You could in theory create your own scripts to organize your code the way PyPI wants it to be. However, just like there are packages that handle known structures such as Pandas for data frames and Numpy for arrays, there are packages and tools that help you create package build distribution files. -```{note} +:::{note} There are a suite of packaging tools that can either help you with the entire packaging process or just one step of the process. For instance @@ -127,7 +127,7 @@ While this can cause some confusion and complexity in the packaging ecosystem - for the most part, each tool provides the same distribution output (with minor differences that most users may not care about). Learn more about those tools on this page. -``` +::: Below, you will learn about the two distribution files that PyPI expects you to publish: sdist and wheel. You will learn about their structure and what files belong in each. @@ -140,7 +140,7 @@ that can be directly installed onto anyones' computer. Learn more about both distributions below. -```{note} +:::{note} If your package is a pure python package with no additional build / compilation steps then the sdist and wheel distributions will have similar content. However if your package has extensions in other languages @@ -148,7 +148,7 @@ or is more complex in its build, the two distributions will be very different. Also note that we are not discussing conda build workflows in this section. [You can learn more about conda builds here.](https://docs.conda.io/projects/conda-build/en/latest/user-guide/tutorials/index.html) -``` +::: (python-source-distribution)= ## What is a source distribution (sdist) @@ -215,14 +215,14 @@ stravalib-1.1.0.post2-SDist.tar.gz file contents ``` -```{admonition} GitHub archive vs sdist +:::{admonition} GitHub archive vs sdist :class: tip When you make a release on GitHub, it creates a `git archive` that contains all of the files in your GitHub repository. While these files are similar to an sdist, these two archives are not the same. The sdist contains a few other items including a metadata directory and if you use `setuptools_scm` or `hatch_vcs` the sdist may also contain a file that stores the version. -``` +::: (python-wheel)= ## What is a Python wheel (whl): @@ -247,13 +247,13 @@ projects and can lead to consistent installs across machines. real security issues with this IF the whl is already built and that file isn't included what is the issue? i need more input here--> -```{tip} +:::{tip} Wheels are also useful in the case that a package needs a **setup.py** file to support a more complex build. In this case, because the files in the wheel bundle are pre built, the user installing doesn't have to worry about malicious code injections when it is installed. -``` +::: The filename of a wheel contains important metadata about your package. @@ -299,6 +299,6 @@ stravalib-1.1.0.post2-py3-none.whl file contents: ``` -```{tip} +:::{tip} [Read more about the wheel format here](https://pythonwheels.com/) -``` +::: diff --git a/package-structure-code/python-package-structure.md b/package-structure-code/python-package-structure.md index d4dc92dd4..f2fa2eda9 100644 --- a/package-structure-code/python-package-structure.md +++ b/package-structure-code/python-package-structure.md @@ -1,4 +1,4 @@ -# Python Package Structure for Scientific Python Projects +# Python Package Structure & Layout There are two different layouts that you will commonly see within the Python packaging ecosystem: @@ -9,7 +9,7 @@ We strongly suggest, but do not require, that you use the **src/** layout (discu for creating your Python package. This layout is also recommended in the [PyPA packaging guide tutorial](https://packaging.python.org/en/latest/tutorials/packaging-projects/). -```{admonition} pyOpenSci will never require a specific package structure for peer review +:::{admonition} pyOpenSci will never require a specific package structure for peer review :class: important We understand that it would take significant effort for existing @@ -20,13 +20,13 @@ someone getting started with Python packaging or someone who's package has a simple build and might be open to moving to a more fail-proof approach. Other resources you can check out: + * [PyPA's overview of src vs flat layouts](https://packaging.python.org/en/latest/discussions/src-layout-vs-flat-layout/) -``` +::: You can use tools like Hatch to quickly create a modern Python package structure. Check out our quickstart tutorial: - -:::{button-link} https://www.pyopensci.org/python-package-guide/tutorials/create-python-package.html#step-1-set-up-the-package-directory-structure +:::{button-link} :color: success :class: sd-rounded-pill float-left @@ -34,11 +34,12 @@ Want to learn how to create the structure to build your package? Click here. ::: +(package-source-layout)= ## What is the Python package source layout? An example of the **src/package** layout structure is below. -``` +```{console} myPackageRepoName ├── CHANGELOG.md ┐ ├── CODE_OF_CONDUCT.md │ @@ -60,19 +61,19 @@ myPackageRepoName Note the location of the following directories in the example above: -- **docs/:** Discussed in our docs chapter, this directory contains your user-facing documentation website. In a **src/** layout docs/ are normally included at the same directory level as the **src/** folder. -- **tests/** This directory contains the tests for your project code. In a **src/** layout, tests are normally included at the same directory level as the **src/** folder. -- **src/package/**: this is the directory that contains the code for your Python project. "Package" is normally your project's name. +* **docs/:** Discussed in our docs chapter, this directory contains your user-facing documentation website. In a **src/** layout docs/ are normally included at the same directory level as the **src/** folder. +* **tests/** This directory contains the tests for your project code. In a **src/** layout, tests are normally included at the same directory level as the **src/** folder. +* **src/package/**: this is the directory that contains the code for your Python project. "Package" is normally your project's name. Also in the above example, notice that all of the core documentation files that pyOpenSci requires live in the root of your project directory. These files include: -- CHANGELOG.md -- CODE_OF_CONDUCT.md -- CONTRIBUTING.md -- LICENSE.txt -- README.md +* CHANGELOG.md +* CODE_OF_CONDUCT.md +* CONTRIBUTING.md +* LICENSE.txt +* README.md @@ -81,9 +82,9 @@ include: :class: sd-rounded-pill Click here to read about our packaging documentation requirements. -``` +::: -```{admonition} Example scientific packages that use **src/package** layout +:::{admonition} Example scientific packages that use **src/package** layout * [Sourmash](https://github.com/sourmash-bio/sourmash) * [bokeh](https://github.com/bokeh/bokeh) @@ -91,8 +92,9 @@ Click here to read about our packaging documentation requirements. * [awkward](https://github.com/scikit-hep/awkward) * [poliastro](https://github.com/poliastro/poliastro/) -``` +::: +(src-layout-test)= ## The src/ layout and testing The benefit of using the **src/package** layout is that it ensures tests are run against the @@ -105,7 +107,7 @@ If `tests/` are outside the **src/package** directory, they aren't included in t - [Read more about reasons to use the **src/package** layout](https://hynek.me/articles/testing-packaging/) -```{admonition} How Python discovers and prioritizes importing modules +:::{admonition} How Python discovers and prioritizes importing modules By default, Python adds a module in your current working directory to the front of the Python module search path. @@ -114,7 +116,7 @@ This means that if you run your tests in your package's working directory, using However, if your package lives in a src/ directory structure **src/package**, then it won't be added to the Python path by default. This means that when you import your package, Python will be forced to search the active environment (which has your package installed). Note: Python versions 3.11 and above have a path setting that can be adjusted to ensure the priority is to use installed packages first (e.g., `PYTHONSAFEPATH`). -``` +::: ### Don't include tests in your package wheel @@ -131,16 +133,16 @@ Including the **tests/** directory in your **src/package** directory ensures tha Be sure to read the [pytest documentation for more about including tests in your package distribution](https://docs.pytest.org/en/7.2.x/explanation/goodpractices.html#choosing-a-test-layout-import-rules). -```{admonition} Challenges with including tests and data in a package wheel +:::{admonition} Challenges with including tests and data in a package wheel :class: tip Tests, especially when accompanied by test data, can create a few small challenges, including: -- Take up space in your distribution, which will build up over time as storage space on PyPI -- Large file sizes can also slow down package installation. +* Take up space in your distribution, which will build up over time as storage space on PyPI +* Large file sizes can also slow down package installation. However, in some cases, particularly in the scientific Python ecosystem, you may need to include tests. -``` +::: ### **Don't include test suite datasets in your package** @@ -152,22 +154,20 @@ the data when you (or a user) runs tests. For more information about Python package tests, see the [tests section of our guide](tests-intro). -- The **src/package** layout is semantically more clear. Code is always found in the +* The **src/package** layout is semantically more clear. Code is always found in the **src/package** directory, `tests/` and `docs/`are in the root directory. -```{important} +:::{important} If your package tests require data, do NOT include that data within your package structure. Including data in your package structure increases the size of your distribution files. This places a maintenance toll on repositories like PyPI and Anaconda.org that have to deal with thousands of package uploads. -``` - +::: :::{button-link} /tutorials/create-python-package.html#step-1-set-up-the-package-directory-structure :color: success :class: sd-rounded-pill float-left - Click here for a quickstart tutorial on creating your Python package. ::: @@ -177,35 +177,33 @@ Click here for a quickstart tutorial on creating your Python package. Many scientific packages use the **flat-layout** given: -- This layout is used by many core scientific Python packages such as NumPy, SciPy, and Matplotlib. -- Many Python tools depend upon tools in other languages and/or complex builds +* This layout is used by many core scientific Python packages such as NumPy, SciPy, and Matplotlib. +* Many Python tools depend upon tools in other languages and/or complex builds with compilation steps. Many maintainers prefer features of the flat layout for more complex builds. While we suggest that you use the **src/package** layout discussed above, it's important to also understand the flat layout, especially if you plan to contribute to a package that uses this layout. - -```{admonition} Why most scientific Python packages do not use source +:::{admonition} Why most scientific Python packages do not use src/ layout :class: tip -In most cases, moving to the **src/package** layout for -larger scientific packages that already use a flat layout would consume significant time. +Migrating larger scientific packages that already use a flat layout would consume significant time and resources. However, the advantages of using the **src/package** layout for a beginner are significant. As such, we recommend that you use the **src/package** layout if you are creating a new package. Numerous packages in the ecosystem [have had to move to a **src/package** layout](https://github.com/scikit-build/cmake-python-distributions/pull/145). -``` +::: ## What does the flat layout structure look like? The flat layout's primary characteristics are: -- The source code for your package lives in a directory with your package's +* The source code for your package lives in a directory with your package's name in the root of your directory -- Often the `tests/` directory also lives within that same `package` directory. +* Often the `tests/` directory also lives within that same `package` directory. Below you can see the recommended structure of a scientific Python package using the flat layout. @@ -233,9 +231,9 @@ myPackage/ There are some benefits to the scientific community in using the flat layout. -- This structure has historically been used across the ecosystem and packages +* This structure has historically been used across the ecosystem and packages using it are unlikely to change. -- You can import the package directly from the root directory. For some this +* You can import the package directly from the root directory. For some this is engrained in their respective workflows. However, for a beginner the danger of doing this is that you are not developing and testing against the installed version of your package. Rather, you are working directly with the @@ -257,18 +255,8 @@ packages to a different layout. The potential benefits of the source layout for these tools are not worth the maintenance investment. ``` - - - +``` diff --git a/package-structure-code/python-package-versions.md b/package-structure-code/python-package-versions.md index 95afca937..c835046cd 100644 --- a/package-structure-code/python-package-versions.md +++ b/package-structure-code/python-package-versions.md @@ -3,13 +3,13 @@ -```{admonition} Key Takeways +:::{admonition} Key Takeways * Follow [semantic versioning guidelines (SemVer) rules](https://semver.org/) when bumping (increasing) your Python's package version; for example a major version bump (version 1.0 --> 2.0) equates to breaking changes in your package's code for a user. * You may want to consider using a plugin like hatch_vsc for managing versions of your package - if you want to have a GitHub only release workflow. * Otherwise most major package build tools such as Hatch, Flit and PDM have a version feature that will help you update your package's version * Avoid updating your packages version number manually by hand in your code! -``` +::: pyOpenSci recommends that you follow the [Python PEP 440](https://peps.python.org/pep-0440) which recommends using [semantic versioning guidelines](https://www.python.org/dev/peps/pep-0440/#semantic-versioning) @@ -25,40 +25,41 @@ with how and when you update your package versions is important as: bump a package version based on standard rules. 3. Consistent version increases following semver rules mean that values of your package version explain the extent of the changes made in the code base from version to version. Thus your package version numbers become "expressive" in the same way that naming code variables well can [make code expressive](https://medium.com/@daniel.oliver.king/writing-expressive-code-b69ef7a5a2fa). -```{admonition} A note about versioning +:::{admonition} A note about versioning In some cases even small version changes can turn a package update into a breaking change for some users. What is also important is that you document how you version your code and if you can, also document your deprecation policy for code. -``` +::: +(package-versioning)= ## SemVer rules Following SemVer, your bump your package version to a: -- patch (1.1.1 --> 1.1.**2**) -- minor (1.1.1 --> 1.**2**.1) -- major (1.1.1 --> **2**.1.1) +* patch (1.1.1 --> 1.1.**2**) +* minor (1.1.1 --> 1.**2**.1) +* major (1.1.1 --> **2**.1.1) version number change based on the following rules: > Given a version number MAJOR.MINOR.PATCH, increment the: > -> - **MAJOR version** when you make incompatible API changes -> - **MINOR version** when you add functionality in a backwards compatible manner -> - **PATCH version** when you make backwards compatible bug fixes +> * **MAJOR version** when you make incompatible API changes +> * **MINOR version** when you add functionality in a backwards compatible manner +> * **PATCH version** when you make backwards compatible bug fixes > Additional labels for pre-release and build metadata are > available as extensions to the MAJOR.MINOR.PATCH format. -```{note} +:::{note} Some people prefer to use [calver](https://calver.org/index.html) for versioning. It may be a simpler-to-use system given it relies upon date values associated with released versions. However, calver does not provide a user with a sense of when a new version might break an existing build. As such we still suggest semver. pyOpenSci will never require semver in a peer review as long as a package has a reasonable approach to versioning! -``` +::: ## Avoid manually updating Python package version numbers if you can @@ -80,6 +81,7 @@ Python package versions. +(tools-version-management)= ## Tools to manage versions for your Python package There are a handful of tools that are widely used in the scientific ecosystem that you can use to manage your package @@ -114,7 +116,7 @@ build that: 1. Tests the build and publishes to test PyPI 1. Publishes the package to PyPI -```{note} +:::{note} Bumping a package version refers to the step of increasing the package version after a set number of changes have been made to it. For example, you might bump from version 0.8 to 0.9 of a package or from 0.9 to 1.0. @@ -123,17 +125,17 @@ Using semantic versioning, there are three main "levels" of versions that you might consider: Major, minor and patch. These are described in more detail below. -``` +::: ## Tools for bumping Python package versions In this section we discuss the following tools for managing your Python package's version: -- hatch & -- hatch_vcs plugin for hatchling -- setuptools-scm -- python-semantic-version +* hatch & +* hatch_vcs plugin for hatchling +* setuptools-scm +* python-semantic-version ### Tool 1: Hatch and other build tools that offer incremental versioning @@ -143,19 +145,19 @@ from Python Semantic Version in that they do not require specific commit messages to implement version. Rather, they allow you to update the version at the command line using commands such as: -- `tool-name version update major` -- `tool-name version update minor` +* `tool-name version update major` +* `tool-name version update minor` [Hatch](https://hatch.pypa.io/latest/version/), for instance offers `hatch version minor` which will modify the version of your package incrementally. With **Hatch** the version value will be found in your `pyproject.toml` file. -#### Hatch (or other tools like PDM) Pros +#### Hatch (or other tools like PDM) pros -- Easy to use version updates locally using a single tool! +* Easy to use version updates locally using a single tool! -#### Hatch (or other tools like PDM) Cons +#### Hatch (or other tools like PDM) cons -- There will be some setup involved to ensure package version is updated throughout your package +* There will be some setup involved to ensure package version is updated throughout your package ### Tool 2: Hatch_vcs & hatchling build back-end @@ -178,11 +180,11 @@ your package version. To use **hatch_vcs** you will need to use the **hatchling** build back end. -```{tip} +:::{tip} Hatchling can also be used with any of the modern build tools including **Flit** and **PDM** if you prefer those for your day to day workflow. -``` +::: #### Hatch example setup in your pyproject.toml @@ -202,23 +204,23 @@ push to PyPI workflow on GitHub. version-file = "_version.py" ``` -```{tip} +:::{tip} If you use **setuptools_scm**, then you might find **hatch_vcs** and **hatchling** to be the modern equivalent to your current setuptools / build workflow. -``` +::: -#### hatch_vcs Pros +#### hatch_vcs pros -- Hatch supports modern Python packaging standards -- It creates a single-source file that contains your package version. -- You never manually update the package version -- You can automate writing the version anywhere in your package including your documentation! -- It supports a purely GitHub based release workflow. This simplifies maintenance workflows. -- Version number is updated in your package via a hidden `_version.py` file. There is no manual configuration updates required. -- While we like detailed commit messages (See Python Semantic Version below), we know that sometimes when maintaining a package specific guidelines around commit messages can be hard to apply and manage. +* Hatch supports modern Python packaging standards +* It creates a single-source file that contains your package version. +* You never manually update the package version +* You can automate writing the version anywhere in your package including your documentation! +* It supports a purely GitHub based release workflow. This simplifies maintenance workflows. +* Version number is updated in your package via a hidden `_version.py` file. There is no manual configuration updates required. +* While we like detailed commit messages (See Python Semantic Version below), we know that sometimes when maintaining a package specific guidelines around commit messages can be hard to apply and manage. -#### hatch_vcs Cons +#### hatch_vcs cons -- In a CI workflow you will end up manually entering or creating the version number via a tag on GitHub. But you could locally develop a build to "bump" tag +* In a CI workflow you will end up manually entering or creating the version number via a tag on GitHub. But you could locally develop a build to "bump" tag versions ### Tool 3: setuptools-scm versioning using git tags @@ -235,31 +237,31 @@ If you are using **setuptools** as your primary build tool, then `*setuptools-sc setuptools_scm Pros -- It creates a single-source file that contains your package version. -- You never manually update the package version -- You can automate writing the version anywhere in your package including your documentation! -- It supports a purely GitHub based release workflow. This simplifies maintenance workflows. -- Version number is updated in your package via a hidden `_version.py` file. There is no manual configuration updates required. -- While we like detailed commit messages (See Python Semantic Version below), we know that sometimes when maintaining a package specific guidelines around commit messages can be hard to apply and manage. -- **setuptools** is still the most commonly used Python packaging build tool +* It creates a single-source file that contains your package version. +* You never manually update the package version +* You can automate writing the version anywhere in your package including your documentation! +* It supports a purely GitHub based release workflow. This simplifies maintenance workflows. +* Version number is updated in your package via a hidden `_version.py` file. There is no manual configuration updates required. +* While we like detailed commit messages (See Python Semantic Version below), we know that sometimes when maintaining a package specific guidelines around commit messages can be hard to apply and manage. +* **setuptools** is still the most commonly used Python packaging build tool -#### setuptools_scm Cons +#### setuptools_scm cons -- In a CI workflow you will end up manually entering or creating the version number via a tag on GitHub. -- Not well documented -- Because setuptools will always have to support backwards compatibility it will always be slower in adopting modern Python packaging conventions. +* In a CI workflow you will end up manually entering or creating the version number via a tag on GitHub. +* Not well documented +* Because setuptools will always have to support backwards compatibility it will always be slower in adopting modern Python packaging conventions. As such you might consider using a more modern tool such as **hatch_vcs** and **hatchling** to build your package and manage package versions. +::: --> ### Tool 4: [Python semantic release](https://python-semantic-release.readthedocs.io/en/latest/) @@ -278,7 +280,7 @@ made the commit below with the words fix(text-here), Python Semantic Release would bump your package to version 1.1.1. ```bash -$ git commit -m "fix(mod_plotting): fix warnings returned athlete attributes" +git commit -m "fix(mod_plotting): fix warnings returned athlete attributes" ``` Similarly a feature (`feat()`) triggers a minor version bump. @@ -288,20 +290,20 @@ For example from version 1.1 to version 1.2 git commit -m "feature(add_conversions): add value conversions to activity date" ``` -```{tip} +:::{tip} You can find a thoughtful discussion of python semantic version [in this Python package guide](https://py-pkgs.org/07-releasing-versioning#automatic-version-bumping). Note that the guide hasn't been updated since 2020 and will potentially be updated in the future! But for now, some of the commands are dated but the content is still excellent. -``` +::: -#### Python Semantic Release Pros +#### Python Semantic Release pros -- Follows semver versioning closely -- Enforces maintainers using descriptive commit messages which can simplify troubleshooting and ensure a cleaner and more self-describing git history. +* Follows semver versioning closely +* Enforces maintainers using descriptive commit messages which can simplify troubleshooting and ensure a cleaner and more self-describing git history. -#### Python Semantic Release Cons +#### Python Semantic Release cons -- Requires very specific commit language to work. In practice some maintainers and contributors may not be able to maintain that level of specificity in commit messages (NOTE: there are bots that will check git commit messages in a repo) -- Release happens at the command line. This makes is harder to implement a GitHub based release workflow as the wrong commit message could trigger a release. -- The version number is manually updated in a configuration file such as `pyproject.toml` vs. in a package **\_version.py** file. +* Requires very specific commit language to work. In practice some maintainers and contributors may not be able to maintain that level of specificity in commit messages (NOTE: there are bots that will check git commit messages in a repo) +* Release happens at the command line. This makes is harder to implement a GitHub based release workflow as the wrong commit message could trigger a release. +* The version number is manually updated in a configuration file such as `pyproject.toml` vs. in a package **\_version.py** file.