From c5c7db79ef4c374e7cbadbe7342e04d9df47ff00 Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Fri, 17 Feb 2023 20:36:34 +0000 Subject: [PATCH 01/10] Add support for Windows This requires changing a few implementation details: - Now fully resolve the command before launching it within the venv, otherwise, in some obscure cases, Windows would not find the executable - Add more environment variables as passthrough, they are hard requirement in order to be able to run on windows - Also add some missing PKG_CONFIG variables as we are reworking this area --- .github/workflows/ci.yml | 6 +++- src/dwas/_config.py | 67 ++++++++++++++++++++++++++++++---------- src/dwas/_frontend.py | 12 +++++-- src/dwas/_runners.py | 12 ++++--- 4 files changed, 73 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8821bf3..93c48e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,18 +79,22 @@ jobs: retention-days: 7 test: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} needs: package strategy: fail-fast: false matrix: + os: [ubuntu-latest] python: - "3.8" - "3.9" - "3.10" - "3.11" - "pypy3.8" + include: + - os: windows-latest + python: "3.8" steps: - name: Check out repository diff --git a/src/dwas/_config.py b/src/dwas/_config.py index 6d5e2c5..8530e43 100644 --- a/src/dwas/_config.py +++ b/src/dwas/_config.py @@ -158,25 +158,58 @@ def __init__( sys.__stdout__.isatty() and sys.__stderr__.isatty() ) + # XXX: keep this list in sync with the above documentation + passthrough_environment_variables = [ + # System + "PATH", + "LANG", + "LANGUAGE", + "PYTHONHASHSEED", + "TMPDIR", + # Pip + "PIP_INDEX_URL", + "PIP_EXTRA_INDEX_URL", + # Proxies + "http_proxy", + "https_proxy", + "no_proxy", + # Certificates + "URL_CA_BUNDLE", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_FILE", + # Compilation and such + "LD_LIBRARY_PATH", + "PKG_CONFIG", + "PKG_CONFIG_PATH", + "PKG_CONFIG_SYSROOT_DIR", + ] + + # Windows needs a few more to work + if sys.platform == "win32": # pragma: win32 cover + passthrough_environment_variables += [ + # Needed to find VisualStudio for building wheels + "PROGRAMDATA", + "PROGRAMFILES(x86)", + "PROGRAMFILES", + # Required to be able to import c extensions (why?) + "SYSTEMDRIVE", + "SYSTEMROOT", + # Needed for the platform module to work + "PROCESSOR_ARCHITECTURE", + # Required by multiprocessing.cpu_count on windows + "NUMBER_OF_PROCESSORS", + # Temporary files directory + "TEMP", + "TMP", + # Needed for expanduser() + "USERPROFILE", + # Needed to discover executables + "PATHEXT", + ] + self.environ = { - # XXX: keep this list in sync with the above documentation key: os.environ[key] - for key in [ - "URL_CA_BUNDLE", - "PATH", - "LANG", - "LANGUAGE", - "LD_LIBRARY_PATH", - "PIP_INDEX_URL", - "PIP_EXTRA_INDEX_URL", - "PYTHONHASHSEED", - "REQUESTS_CA_BUNDLE", - "SSL_CERT_FILE", - "http_proxy", - "https_proxy", - "no_proxy", - "TMPDIR", - ] + for key in passthrough_environment_variables if key in os.environ } diff --git a/src/dwas/_frontend.py b/src/dwas/_frontend.py index d5bff82..63a6c39 100644 --- a/src/dwas/_frontend.py +++ b/src/dwas/_frontend.py @@ -13,8 +13,16 @@ from ._scheduler import Scheduler from ._timing import format_timedelta -ANSI_SHOW_CURSOR = f"{ansi.CSI}?25h" -ANSI_HIDE_CURSOR = f"{ansi.CSI}?25l" +# XXX: If colorama adds support, we could probably enable this +# https://github.com/tartley/colorama/issues/272 +if sys.platform == "win32": + # Don't support ANSI cursor hiding on windows for now, it can't be done + # with ansi codes + ANSI_HIDE_CURSOR = "" + ANSI_SHOW_CURSOR = "" +else: + ANSI_HIDE_CURSOR = f"{ansi.CSI}?25l" + ANSI_SHOW_CURSOR = f"{ansi.CSI}?25h" class StepSummary: diff --git a/src/dwas/_runners.py b/src/dwas/_runners.py index ac9cdfa..52ef3a0 100644 --- a/src/dwas/_runners.py +++ b/src/dwas/_runners.py @@ -123,7 +123,7 @@ def run( silent_on_success: bool = False, ) -> subprocess.CompletedProcess[None]: env = self._merge_env(self._config, env) - self._validate_command(command[0], external_command, env) + command[0] = self._resolve_command(command[0], external_command, env) return self._proc_manager.run( command, @@ -139,16 +139,16 @@ def _merge_env( env.update( { "VIRTUAL_ENV": str(self._path), - "PATH": f"{':'.join(self._installer.paths)}:{env['PATH']}", + "PATH": f"{os.pathsep.join(self._installer.paths)}{os.pathsep}{env['PATH']}", } ) if additional_env is not None: env.update(additional_env) return env - def _validate_command( + def _resolve_command( self, command: str, external_command: bool, env: Dict[str, str] - ) -> None: + ) -> str: cmd = shutil.which(command, path=env["PATH"]) if cmd is None: @@ -166,3 +166,7 @@ def _validate_command( ) if not external_command and not command_in_venv: raise CommandNotInEnvironment(command) + + # Return the resolved command, this is easier for Windows, as it will + # not always be able to find the command if not fully referenced. + return cmd From 7d1cd4475919ed0e3df9f6cbc720b306f3b7d76b Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Fri, 17 Feb 2023 22:15:11 +0000 Subject: [PATCH 02/10] fixup! Add support for Windows --- tests/predefined/test_package.py | 21 +++++++++++++++++++++ tests/predefined/test_pylint.py | 24 ++++++++++++++++++++++++ tests/predefined/test_twine.py | 22 ++++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/tests/predefined/test_package.py b/tests/predefined/test_package.py index b82e7fa..f63dea9 100644 --- a/tests/predefined/test_package.py +++ b/tests/predefined/test_package.py @@ -1,4 +1,5 @@ import json +import sys from pathlib import Path import pytest @@ -22,3 +23,23 @@ def test_exposes_sdists_and_wheels(self, cache_path): Path(artifacts["wheels"][0]).name == "test_package-0.0.0-py3-none-any.whl" ) + + @pytest.mark.parametrize( + "enable_colors", + [ + pytest.param( + True, + marks=[ + pytest.mark.xfail( + sys.platform == "win32", + reason="colors are not supported on windows", + strict=True, + ) + ], + ), + False, + ], + ids=["colors", "no-colors"], + ) + def test_respects_color_settings(self, cache_path, enable_colors): + super().test_respects_color_settings(cache_path, enable_colors) diff --git a/tests/predefined/test_pylint.py b/tests/predefined/test_pylint.py index 8a197ea..f8a89f5 100644 --- a/tests/predefined/test_pylint.py +++ b/tests/predefined/test_pylint.py @@ -1,3 +1,7 @@ +import sys + +import pytest + from .mixins import BaseLinterTest @@ -13,3 +17,23 @@ class TestPylint(BaseLinterTest): import os """ valid_file = '"""This is a token file"""\n' + + @pytest.mark.parametrize( + "enable_colors", + [ + pytest.param( + True, + marks=[ + pytest.mark.xfail( + sys.platform == "win32", + reason="colors are not supported on windows", + strict=True, + ) + ], + ), + False, + ], + ids=["colors", "no-colors"], + ) + def test_respects_color_settings(self, cache_path, enable_colors): + super().test_respects_color_settings(cache_path, enable_colors) diff --git a/tests/predefined/test_twine.py b/tests/predefined/test_twine.py index cc7862a..76b35e9 100644 --- a/tests/predefined/test_twine.py +++ b/tests/predefined/test_twine.py @@ -1,3 +1,5 @@ +import sys + import pytest from .._utils import using_project @@ -9,3 +11,23 @@ class TestTwine(BaseStepTest): @pytest.fixture def expected_output(self): return "Checking" + + @pytest.mark.parametrize( + "enable_colors", + [ + pytest.param( + True, + marks=[ + pytest.mark.xfail( + sys.platform == "win32", + reason="colors are not supported on windows", + strict=True, + ) + ], + ), + False, + ], + ids=["colors", "no-colors"], + ) + def test_respects_color_settings(self, cache_path, enable_colors): + super().test_respects_color_settings(cache_path, enable_colors) From af4fc0c74609034ce37a4867a56d41b988dbde67 Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Fri, 17 Feb 2023 22:18:50 +0000 Subject: [PATCH 03/10] fixup! fixup! Add support for Windows --- tests/predefined/test_pylint.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/predefined/test_pylint.py b/tests/predefined/test_pylint.py index f8a89f5..1c07bae 100644 --- a/tests/predefined/test_pylint.py +++ b/tests/predefined/test_pylint.py @@ -35,5 +35,9 @@ class TestPylint(BaseLinterTest): ], ids=["colors", "no-colors"], ) - def test_respects_color_settings(self, cache_path, enable_colors): - super().test_respects_color_settings(cache_path, enable_colors) + def test_respects_color_settings( + self, cache_path, tmp_path, enable_colors + ): + super().test_respects_color_settings( + cache_path, tmp_path, enable_colors + ) From 9eb8c19d30c1c37425ff4fb4eb619a55173fa445 Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Sat, 18 Feb 2023 11:06:09 +0000 Subject: [PATCH 04/10] fixup! fixup! fixup! Add support for Windows --- .github/workflows/ci.yml | 2 +- src/dwas/_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93c48e0..be80213 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,7 +94,7 @@ jobs: - "pypy3.8" include: - os: windows-latest - python: "3.8" + python: "3.11" steps: - name: Check out repository diff --git a/src/dwas/_config.py b/src/dwas/_config.py index 8530e43..c851357 100644 --- a/src/dwas/_config.py +++ b/src/dwas/_config.py @@ -211,7 +211,7 @@ def __init__( key: os.environ[key] for key in passthrough_environment_variables if key in os.environ - } + }< Date: Sat, 18 Feb 2023 12:33:57 +0000 Subject: [PATCH 05/10] fixup! fixup! fixup! fixup! Add support for Windows --- src/dwas/_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dwas/_config.py b/src/dwas/_config.py index c851357..8530e43 100644 --- a/src/dwas/_config.py +++ b/src/dwas/_config.py @@ -211,7 +211,7 @@ def __init__( key: os.environ[key] for key in passthrough_environment_variables if key in os.environ - }< Date: Sat, 18 Feb 2023 12:45:43 +0000 Subject: [PATCH 06/10] fixup! fixup! fixup! fixup! fixup! Add support for Windows --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be80213..e4ef357 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,7 +94,7 @@ jobs: - "pypy3.8" include: - os: windows-latest - python: "3.11" + python: "3.10" steps: - name: Check out repository From 4a6ba6ee962c500d5441e019fb141331da1c4825 Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Sat, 18 Feb 2023 12:58:24 +0000 Subject: [PATCH 07/10] fixup! fixup! fixup! fixup! fixup! fixup! Add support for Windows --- .github/workflows/ci.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4ef357..3de056c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,7 +94,7 @@ jobs: - "pypy3.8" include: - os: windows-latest - python: "3.10" + python: "3.8" steps: - name: Check out repository @@ -126,6 +126,26 @@ jobs: path: ${{ env.COVERAGE_FILES_PATH }} retention-days: 7 + windows: + runs-on: windows-latest + + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + cache: pip + + # FIXME: we should take dwas from pypi once it's published + - name: Install dwas + run: python -m pip install .[test] + + - name: pytest + run: pytest + coverage: runs-on: ubuntu-latest needs: test From 032ea7b659c0dd6742e094e0818b4542386df7f2 Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Sat, 18 Feb 2023 13:06:08 +0000 Subject: [PATCH 08/10] fixup! fixup! fixup! fixup! fixup! fixup! fixup! Add support for Windows --- .github/workflows/ci.yml | 354 +++++++++++++++++++-------------------- 1 file changed, 177 insertions(+), 177 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3de056c..02374d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,110 +21,110 @@ concurrency: cancel-in-progress: true jobs: - lint: - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - session: - - black - - docformatter - - isort - - mypy - - pylint - - unimport - - steps: - - name: Check out repository - uses: actions/checkout@v3 - - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.DEFAULT_PYTHON }} - cache: pip - - # FIXME: we should take dwas from pypi once it's published - - name: Install dwas - run: python -m pip install . - - - name: ${{ matrix.session }} - run: dwas --verbose --only ${{ matrix.session }} - - package: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v3 - - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.DEFAULT_PYTHON }} - cache: pip - - # FIXME: we should take dwas from pypi once it's published - - name: Install dwas - run: python -m pip install . - - - name: package - run: dwas --verbose --only package - - - name: Save packaged dist - uses: actions/upload-artifact@v3 - with: - name: dist - path: ${{ env.PACKAGES_PATH }}/* - retention-days: 7 - - test: - runs-on: ${{ matrix.os }} - needs: package - - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - python: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - - "pypy3.8" - include: - - os: windows-latest - python: "3.8" - - steps: - - name: Check out repository - uses: actions/checkout@v3 - - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - cache: pip - - # FIXME: we should take dwas from pypi once it's published - - name: Install dwas - run: python -m pip install . - - - name: Download packaged dist - uses: actions/download-artifact@v3 - with: - name: dist - path: ${{ env.PACKAGES_PATH }} - - - name: pytest[${{ matrix.python }}] - run: dwas --verbose --only pytest[${{ matrix.python }}] -- --numprocesses auto - - - name: Save coverage files - uses: actions/upload-artifact@v3 - with: - name: coverage-files-pytest-${{ matrix.python }} - path: ${{ env.COVERAGE_FILES_PATH }} - retention-days: 7 + # lint: + # runs-on: ubuntu-latest + + # strategy: + # fail-fast: false + # matrix: + # session: + # - black + # - docformatter + # - isort + # - mypy + # - pylint + # - unimport + + # steps: + # - name: Check out repository + # uses: actions/checkout@v3 + + # - name: Setup python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ env.DEFAULT_PYTHON }} + # cache: pip + + # # FIXME: we should take dwas from pypi once it's published + # - name: Install dwas + # run: python -m pip install . + + # - name: ${{ matrix.session }} + # run: dwas --verbose --only ${{ matrix.session }} + + # package: + # runs-on: ubuntu-latest + # steps: + # - name: Check out repository + # uses: actions/checkout@v3 + + # - name: Setup python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ env.DEFAULT_PYTHON }} + # cache: pip + + # # FIXME: we should take dwas from pypi once it's published + # - name: Install dwas + # run: python -m pip install . + + # - name: package + # run: dwas --verbose --only package + + # - name: Save packaged dist + # uses: actions/upload-artifact@v3 + # with: + # name: dist + # path: ${{ env.PACKAGES_PATH }}/* + # retention-days: 7 + + # test: + # runs-on: ${{ matrix.os }} + # needs: package + + # strategy: + # fail-fast: false + # matrix: + # os: [ubuntu-latest] + # python: + # - "3.8" + # - "3.9" + # - "3.10" + # - "3.11" + # - "pypy3.8" + # include: + # - os: windows-latest + # python: "3.8" + + # steps: + # - name: Check out repository + # uses: actions/checkout@v3 + + # - name: Setup python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ matrix.python }} + # cache: pip + + # # FIXME: we should take dwas from pypi once it's published + # - name: Install dwas + # run: python -m pip install . + + # - name: Download packaged dist + # uses: actions/download-artifact@v3 + # with: + # name: dist + # path: ${{ env.PACKAGES_PATH }} + + # - name: pytest[${{ matrix.python }}] + # run: dwas --verbose --only pytest[${{ matrix.python }}] -- --numprocesses auto + + # - name: Save coverage files + # uses: actions/upload-artifact@v3 + # with: + # name: coverage-files-pytest-${{ matrix.python }} + # path: ${{ env.COVERAGE_FILES_PATH }} + # retention-days: 7 windows: runs-on: windows-latest @@ -144,76 +144,76 @@ jobs: run: python -m pip install .[test] - name: pytest - run: pytest - - coverage: - runs-on: ubuntu-latest - needs: test - - steps: - - name: Check out repository - uses: actions/checkout@v3 - - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.DEFAULT_PYTHON }} - cache: pip - - # FIXME: we should take dwas from pypi once it's published - - name: Install dwas - run: python -m pip install . - - # This will download add artifacts that were generated before, which is not ideal - # as we only need some of them, but it's simpler than having to repeat them one by one - - name: Download generated coverage files - uses: actions/download-artifact@v3 - with: - path: _cache - - - name: Move artifacts to the right place - run: mkdir -p .dwas/cache && mv _cache/*/pytest-* .dwas/cache && rm -rf _cache && ls -lR .dwas/cache - - - name: Generate coverage report - run: dwas --verbose --only coverage - - - name: Upload coverage to codecov - uses: codecov/codecov-action@v3 - with: - files: ./_artifacts/coverage/coverage.xml - fail_ci_if_error: true - verbose: true - - - name: Save coverage report - uses: actions/upload-artifact@v3 - with: - name: coverage [html] - path: _artifacts/coverage/html - retention-days: 7 - - twine: - runs-on: ubuntu-latest - needs: package - - steps: - - name: Check out repository - uses: actions/checkout@v3 - - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.DEFAULT_PYTHON }} - cache: pip - - - name: Download packaged dist - uses: actions/download-artifact@v3 - with: - name: dist - path: ${{ env.PACKAGES_PATH }} - - # FIXME: we should take dwas from pypi once it's published - - name: Install dwas - run: python -m pip install . - - - name: Validate packages with twine - run: dwas --verbose --only twine:check + run: pytest --numprocesses auto --color yes + + # coverage: + # runs-on: ubuntu-latest + # needs: test + + # steps: + # - name: Check out repository + # uses: actions/checkout@v3 + + # - name: Setup python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ env.DEFAULT_PYTHON }} + # cache: pip + + # # FIXME: we should take dwas from pypi once it's published + # - name: Install dwas + # run: python -m pip install . + + # # This will download add artifacts that were generated before, which is not ideal + # # as we only need some of them, but it's simpler than having to repeat them one by one + # - name: Download generated coverage files + # uses: actions/download-artifact@v3 + # with: + # path: _cache + + # - name: Move artifacts to the right place + # run: mkdir -p .dwas/cache && mv _cache/*/pytest-* .dwas/cache && rm -rf _cache && ls -lR .dwas/cache + + # - name: Generate coverage report + # run: dwas --verbose --only coverage + + # - name: Upload coverage to codecov + # uses: codecov/codecov-action@v3 + # with: + # files: ./_artifacts/coverage/coverage.xml + # fail_ci_if_error: true + # verbose: true + + # - name: Save coverage report + # uses: actions/upload-artifact@v3 + # with: + # name: coverage [html] + # path: _artifacts/coverage/html + # retention-days: 7 + + # twine: + # runs-on: ubuntu-latest + # needs: package + + # steps: + # - name: Check out repository + # uses: actions/checkout@v3 + + # - name: Setup python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ env.DEFAULT_PYTHON }} + # cache: pip + + # - name: Download packaged dist + # uses: actions/download-artifact@v3 + # with: + # name: dist + # path: ${{ env.PACKAGES_PATH }} + + # # FIXME: we should take dwas from pypi once it's published + # - name: Install dwas + # run: python -m pip install . + + # - name: Validate packages with twine + # run: dwas --verbose --only twine:check From 7a55e53d6ad2c8ae34dcea25f9a30dafd1cf505a Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Sat, 18 Feb 2023 13:09:26 +0000 Subject: [PATCH 09/10] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Add support for Windows --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02374d4..0f466e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,7 +144,7 @@ jobs: run: python -m pip install .[test] - name: pytest - run: pytest --numprocesses auto --color yes + run: dwas pylint # coverage: # runs-on: ubuntu-latest From 50e4ab16c72235e944ad93c8fc2a2660ca7b85ad Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Sat, 18 Feb 2023 13:16:41 +0000 Subject: [PATCH 10/10] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Add support for Windows --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f466e6..c9413c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,7 +136,7 @@ jobs: - name: Setup python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python }} + python-version: "3.11" cache: pip # FIXME: we should take dwas from pypi once it's published