TST: Add pytest-xdist to run tests in parallel (IMAP-Science-Operations-Center#1600)

* TST: Add pytest-xdist to run tests in parallel

Make it opt-in but run the tests in parallel on the CI systems
at least and allow users to locally use more processors.

Author: greglucas

.github/workflows/test.yml

@@ -46,9 +46,9 @@ jobs:
         run: |
           # Ignore the network marks from the remote test environment
           if [ "$USE_EXTERNAL_DATA" = "true" ]; then
-            poetry run pytest --color=yes --cov --cov-report=xml
+            poetry run pytest -n auto --color=yes --cov --cov-report=xml
           else
-            poetry run pytest --color=yes --cov --cov-report=xml -m "not external_kernel and not external_test_data"
+            poetry run pytest -n auto --color=yes --cov --cov-report=xml -m "not external_kernel and not external_test_data"
           fi
 
       - name: Upload coverage reports to Codecov

imap_processing/tests/codice/test_codice_l1a.py

@@ -9,6 +9,7 @@
 from imap_processing.cdf.utils import load_cdf, write_cdf
 from imap_processing.codice import constants
 from imap_processing.codice.codice_l1a import process_codice_l1a
+from imap_processing.tests.conftest import _download_external_data, _test_data_paths
 
 from .conftest import TEST_L0_FILE, VALIDATION_DATA
 
@@ -112,7 +113,10 @@ def test_l1a_data() -> xr.Dataset:
     processed_datasets : list[xarray.Dataset]
         A list of ``xarray`` datasets containing the test data
     """
-
+    # Make sure we have the data available here. This test collection gets
+    # skipped at the module level if the mark isn't present. We can't decorate
+    # a fixture, so add the needed call directly here instead.
+    _download_external_data(_test_data_paths())
     processed_datasets = process_codice_l1a(file_path=TEST_L0_FILE)
 
     return processed_datasets

imap_processing/tests/conftest.py

@@ -94,7 +94,7 @@ def _download_external_kernels(spice_test_data_path):
 
 @pytest.fixture(scope="session")
 def _download_test_data():
-    _download_external_data(test_data_paths())
+    _download_external_data(_test_data_paths())
 
 
 def _download_external_data(test_data_path_list):
@@ -125,7 +125,7 @@ def _download_external_data(test_data_path_list):
             logger.info(f"File already exists: {destination}")
 
 
-def test_data_paths():
+def _test_data_paths():
     """Defines a list of test data files to download from the AWS S3 bucket
     and the corresponding location in which to store the downloaded file"""
     test_data_path_list = [

poetry.lock

@@ -49,6 +49,7 @@ pre-commit = {version="^3.3.3", optional=true}
 pydata-sphinx-theme = {version="*", optional=true}
 pytest = {version=">=6.2.5", optional=true}
 pytest-cov = {version="^4.0.0", optional=true}
+pytest-xdist = {version="^3.2", optional=true}
 ruff = {version="==0.2.1", optional=true}
 sphinx = {version="*", optional=true}
 sphinxcontrib-openapi = {version="^0.8.3", optional=true}
@@ -60,7 +61,7 @@ netcdf4 = {version ="^1.7.2",  optional = true}
 [tool.poetry.extras]
 dev = ["pre-commit", "ruff", "mypy"]
 doc = ["numpydoc", "pydata-sphinx-theme", "sphinx", "sphinxcontrib-openapi"]
-test = ["openpyxl", "pytest", "pytest-cov", "requests", "netcdf4"]
+test = ["openpyxl", "pytest", "pytest-cov", "pytest-xdist", "requests", "netcdf4"]
 tools = ["openpyxl", "pandas"]
 map_visualization = ["healpy"]