From d97d49eca1416933ff553fee482ce01eacd74330 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Thu, 20 Mar 2025 17:32:07 +0100 Subject: [PATCH 01/24] work in progress --- rsconnect/actions.py | 4 +- rsconnect/bundle.py | 55 +++++- rsconnect/environment.py | 254 +++----------------------- rsconnect/main.py | 22 +-- rsconnect/pyproject.py | 19 +- rsconnect/subprocesses/__init__.py | 0 rsconnect/subprocesses/environment.py | 235 ++++++++++++++++++++++++ 7 files changed, 333 insertions(+), 256 deletions(-) create mode 100644 rsconnect/subprocesses/__init__.py create mode 100644 rsconnect/subprocesses/environment.py diff --git a/rsconnect/actions.py b/rsconnect/actions.py index c2589d63..5df9aeb9 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -37,7 +37,7 @@ make_quarto_source_bundle, read_manifest_file, ) -from .environment import Environment, EnvironmentException +from .environment import Environment from .exception import RSConnectException from .log import VERBOSE, logger from .models import AppMode, AppModes @@ -78,8 +78,6 @@ def failed(err: str): passed() except RSConnectException as exc: failed("Error: " + exc.message) - except EnvironmentException as exc: - failed("Error: " + str(exc)) except Exception as exc: traceback.print_exc() failed("Internal error: " + str(exc)) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index df9ba659..1a55b6c2 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -53,7 +53,8 @@ import click -from .environment import Environment, MakeEnvironment +from . import pyproject +from .environment import Environment from .exception import RSConnectException from .log import VERBOSE, logger from .models import AppMode, AppModes, GlobSet @@ -1669,6 +1670,22 @@ def _warn_on_ignored_requirements(directory: str, requirements_file_name: str) - ) +def _warn_on_missing_python_version(version_constraint: Optional[str]) -> None: + """ + Check that the project has a Python version constraint requested. + If it doesn't warn the user that it should be specified. + + :param version_constraint: the version constraint in the project. + """ + if version_constraint is None: + click.secho( + " Warning: Python version constraint missing from pyproject.toml or .python-version\n" + " Connect will guess the version to use based on local environment.\n" + " Consider specifying a Python version constraint.", + fg="yellow", + ) + + def fake_module_file_from_directory(directory: str) -> str: """ Takes a directory and invents a properly named file that though possibly fake, @@ -1717,7 +1734,7 @@ def inspect_environment( if force_generate: flags.append("f") - args = [python, "-m", "rsconnect.environment"] + args = [python, "-m", "rsconnect.subprocesses.environment"] if flags: args.append("-" + "".join(flags)) args.append(directory) @@ -1732,20 +1749,23 @@ def inspect_environment( except json.JSONDecodeError as e: raise RSConnectException("Error parsing environment JSON") from e - try: - return MakeEnvironment(**environment_data) - except TypeError as e: + if "error" in environment_data: system_error_message = environment_data.get("error") if system_error_message: raise RSConnectException(f"Error creating environment: {system_error_message}") from e + + try: + return Environment.from_json(environment_data) + except TypeError as e: raise RSConnectException("Error constructing environment object") from e -def get_python_env_info( +def _get_python_env_info( file_name: str, python: str | None, force_generate: bool = False, override_python_version: str | None = None, + python_version_requirement: str | None = None, ) -> tuple[str, Environment]: """ Gathers the python and environment information relating to the specified file @@ -1766,8 +1786,11 @@ def get_python_env_info( logger.debug("Python: %s" % python) logger.debug("Environment: %s" % pformat(environment._asdict())) + if python_version_requirement: + environment.python_version_requirement = python_version_requirement + if override_python_version: - environment = environment._replace(python=override_python_version) + environment = environment.python = override_python_version return python, environment @@ -2261,17 +2284,29 @@ def create_python_environment( force_generate: bool = False, python: Optional[str] = None, override_python_version: Optional[str] = None, + app_file: Optional[str] = None, ) -> Environment: - module_file = fake_module_file_from_directory(directory) + if app_file is None: + module_file = fake_module_file_from_directory(directory) + else: + module_file = app_file # click.secho(' Deploying %s to server "%s"' % (directory, connect_server.url)) - _warn_on_ignored_manifest(directory) _warn_if_no_requirements_file(directory) _warn_if_environment_directory(directory) + python_version_requirement = pyproject.detect_python_version_requirement(directory) + _warn_on_missing_python_version(python_version_requirement) + + if override_python_version: + # TODO: --override-python-version should be deprecated in the future + # and instead we should suggest the user sets it in .python-version + # or pyproject.toml + python_version_requirement = override_python_version + # with cli_feedback("Inspecting Python environment"): - _, environment = get_python_env_info(module_file, python, force_generate, override_python_version) + _, environment = _get_python_env_info(module_file, python, force_generate, override_python_version) if force_generate: _warn_on_ignored_requirements(directory, environment.filename) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 9858d392..9aebda77 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -1,233 +1,35 @@ -#!/usr/bin/env python -""" -Environment data class abstraction that is usable as an executable module +import typing +import dataclasses -```bash -python -m rsconnect.environment -``` -""" -from __future__ import annotations +from .subprocesses.environment import EnvironmentData, MakeEnvironmentData as _MakeEnvironmentData -import datetime -import json -import locale -import os -import re -import subprocess -import sys -from dataclasses import asdict, dataclass, replace -from typing import Callable, Optional -version_re = re.compile(r"\d+\.\d+(\.\d+)?") -exec_dir = os.path.dirname(sys.executable) - - -@dataclass(frozen=True) class Environment: - contents: str - filename: str - locale: str - package_manager: str - pip: str - python: str - source: str - error: str | None - - def _asdict(self): - return asdict(self) - - def _replace(self, **kwargs: object): - return replace(self, **kwargs) - - -def MakeEnvironment( - contents: str, - filename: str, - locale: str, - package_manager: str, - pip: str, - python: str, - source: str, - error: Optional[str] = None, - **kwargs: object, # provides compatibility where we no longer support some older properties -) -> Environment: - return Environment(contents, filename, locale, package_manager, pip, python, source, error) - - -class EnvironmentException(Exception): - pass - - -def detect_environment(dirname: str, force_generate: bool = False) -> Environment: - """Determine the python dependencies in the environment. - - `pip freeze` will be used to introspect the environment. - - :param: dirname Directory name - :param: force_generate Force the generation of an environment - :return: a dictionary containing the package spec filename and contents if successful, - or a dictionary containing `error` on failure. - """ - - if force_generate: - result = pip_freeze() - else: - result = output_file(dirname, "requirements.txt", "pip") or pip_freeze() - - if result is not None: - result["python"] = get_python_version() - result["pip"] = get_version("pip") - result["locale"] = get_default_locale() - - return MakeEnvironment(**result) - - -def get_python_version() -> str: - v = sys.version_info - return "%d.%d.%d" % (v[0], v[1], v[2]) - - -def get_default_locale(locale_source: Callable[..., tuple[str | None, str | None]] = locale.getlocale): - result = ".".join([item or "" for item in locale_source()]) - return "" if result == "." else result - - -def get_version(module: str): - try: - args = [sys.executable, "-m", module, "--version"] - proc = subprocess.Popen( - args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - stdout, _stderr = proc.communicate() - match = version_re.search(stdout) - if match: - return match.group() - - msg = "Failed to get version of '%s' from the output of: %s" % ( - module, - " ".join(args), - ) - raise EnvironmentException(msg) - except Exception as exception: - raise EnvironmentException("Error getting '%s' version: %s" % (module, str(exception))) - - -def output_file(dirname: str, filename: str, package_manager: str): - """Read an existing package spec file. + """A project environment, - Returns a dictionary containing the filename and contents - if successful, None if the file does not exist, - or a dictionary containing 'error' on failure. + The data is loaded from a rsconnect.utils.environment json response """ - try: - path = os.path.join(dirname, filename) - if not os.path.exists(path): - return None - - with open(path, "r") as f: - data = f.read() - - data = "\n".join([line for line in data.split("\n") if "rsconnect" not in line]) - - return { - "filename": filename, - "contents": data, - "source": "file", - "package_manager": package_manager, - } - except Exception as exception: - raise EnvironmentException("Error reading %s: %s" % (filename, str(exception))) - - -def pip_freeze(): - """Inspect the environment using `pip freeze --disable-pip-version-check version`. - - Returns a dictionary containing the filename - (always 'requirements.txt') and contents if successful, - or a dictionary containing 'error' on failure. - """ - try: - proc = subprocess.Popen( - [sys.executable, "-m", "pip", "freeze", "--disable-pip-version-check"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - - pip_stdout, pip_stderr = proc.communicate() - pip_status = proc.returncode - except Exception as exception: - raise EnvironmentException("Error during pip freeze: %s" % str(exception)) - - if pip_status != 0: - msg = pip_stderr or ("exited with code %d" % pip_status) - raise EnvironmentException("Error during pip freeze: %s" % msg) - - pip_stdout = filter_pip_freeze_output(pip_stdout) - - pip_stdout = ( - "# requirements.txt generated by rsconnect-python on " - + str(datetime.datetime.now(datetime.timezone.utc)) - + "\n" - + pip_stdout - ) - - return { - "filename": "requirements.txt", - "contents": pip_stdout, - "source": "pip_freeze", - "package_manager": "pip", - } - - -def filter_pip_freeze_output(pip_stdout: str): - # Filter out dependency on `rsconnect` and ignore output lines from pip which start with `[notice]` - return "\n".join( - [line for line in pip_stdout.split("\n") if (("rsconnect" not in line) and (line.find("[notice]") != 0))] - ) - - -def strip_ref(line: str): - # remove erroneous conda build paths that will break pip install - return line.split(" @ file:", 1)[0].strip() - - -def exclude(line: str): - return line and line.startswith("setuptools") and "post" in line - - -def main(): - """ - Run `detect_environment` and dump the result as JSON. - """ - try: - if len(sys.argv) < 2: - raise EnvironmentException("Usage: %s [-fc] DIRECTORY" % sys.argv[0]) - # directory is always the last argument - directory = sys.argv[len(sys.argv) - 1] - flags = "" - force_generate = False - if len(sys.argv) > 2: - flags = sys.argv[1] - if "f" in flags: - force_generate = True - envinfo = detect_environment(directory, force_generate)._asdict() - if "contents" in envinfo: - keepers = list(map(strip_ref, envinfo["contents"].split("\n"))) - keepers = [line for line in keepers if not exclude(line)] - envinfo["contents"] = "\n".join(keepers) - - json.dump( - envinfo, - sys.stdout, - indent=4, - ) - except EnvironmentException as exception: - json.dump(dict(error=str(exception)), sys.stdout, indent=4) - -if __name__ == "__main__": - main() + def __init__(self, data: EnvironmentData, python_version_requirement: typing.Optional[str] = None): + self._data = data + self._data_fields = dataclasses.fields(self.data) + + # Fields that are not loaded from the environment subprocess + self.python_version_requirement + + def __getattr__(self, name: str) -> typing.Any: + # We directly proxy the attributes of the EnvironmentData object + # so that schema changes can be handled in EnvironmentData exclusively. + return self._data[name] + + def __setattr__(self, name, value): + if name in self._data_fields: + # proxy the attribute to the underlying EnvironmentData object + self._data._replace(name=value) + else: + super().__setattr__(name, value) + + @classmethod + def from_json(cls, json_data: dict) -> "Environment": + """Create an Environment instance from the JSON representation of EnvironmentData.""" + return cls(_MakeEnvironmentData(**json_data)) diff --git a/rsconnect/main.py b/rsconnect/main.py index 28470478..736ed2eb 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -74,7 +74,6 @@ write_tensorflow_manifest_json, write_voila_manifest_json, ) -from .environment import EnvironmentException from .exception import RSConnectException from .json_web_token import ( TokenGenerator, @@ -116,8 +115,6 @@ def failed(err: str) -> Never: result = func(*args, **kwargs) except RSConnectException as exc: failed("Error: " + exc.message) - except EnvironmentException as exc: - failed("Error: " + str(exc)) except Exception as exc: traceback.print_exc() failed("Internal error: " + str(exc)) @@ -948,10 +945,10 @@ def deploy_notebook( app_mode = AppModes.JUPYTER_NOTEBOOK if not static else AppModes.STATIC base_dir = dirname(file) - _warn_on_ignored_manifest(base_dir) - _warn_if_no_requirements_file(base_dir) - _warn_if_environment_directory(base_dir) - python, environment = get_python_env_info(file, python, force_generate, override_python_version) + environment = create_python_environment(base_dir, app_file=file, + force_generate=force_generate, + python=python, + override_python_version=override_python_version) if force_generate: _warn_on_ignored_requirements(base_dir, environment.filename) @@ -1290,7 +1287,6 @@ def deploy_quarto( base_dir = file_or_directory if not isdir(file_or_directory): base_dir = dirname(file_or_directory) - module_file = fake_module_file_from_directory(file_or_directory) extra_files = validate_extra_files(base_dir, extra_files) _warn_on_ignored_manifest(base_dir) @@ -1301,17 +1297,11 @@ def deploy_quarto( inspect = quarto_inspect(quarto, file_or_directory) engines = validate_quarto_engines(inspect) - python = None environment = None if "jupyter" in engines: - _warn_if_no_requirements_file(base_dir) - _warn_if_environment_directory(base_dir) - with cli_feedback("Inspecting Python environment"): - python, environment = get_python_env_info(module_file, python, force_generate, override_python_version) - - if force_generate: - _warn_on_ignored_requirements(base_dir, environment.filename) + environment = create_python_environment(base_dir, force_generate=force_generate, + override_python_version=override_python_version) ce = RSConnectExecutor( ctx=ctx, diff --git a/rsconnect/pyproject.py b/rsconnect/pyproject.py index 553ece9c..febd719d 100644 --- a/rsconnect/pyproject.py +++ b/rsconnect/pyproject.py @@ -16,6 +16,23 @@ import toml as tomllib # type: ignore[no-redef] +def detect_python_version_requirement(directory: typing.Union[str, pathlib.Path]) -> typing.Optional[str]: + """Detect the python version requirement for a project. + + The directory should contain a metadata file such as pyproject.toml, + setup.cfg, or .python-version. + + Returns the python version requirement as a string or None if not found. + """ + for _, metadata_file in lookup_metadata_file(directory): + parser = get_python_version_requirement_parser(metadata_file) + version_constraint = parser(metadata_file) + if version_constraint: + return version_constraint + + return None + + def lookup_metadata_file(directory: typing.Union[str, pathlib.Path]) -> typing.List[typing.Tuple[str, pathlib.Path]]: """Given the directory of a project return the path of a usable metadata file. @@ -50,7 +67,7 @@ def get_python_version_requirement_parser( elif metadata_file.name == ".python-version": return parse_pyversion_python_requires else: - return None + raise NotImplementedError(f"Unknown metadata file type: {metadata_file.name}") def parse_pyproject_python_requires(pyproject_file: pathlib.Path) -> typing.Optional[str]: diff --git a/rsconnect/subprocesses/__init__.py b/rsconnect/subprocesses/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rsconnect/subprocesses/environment.py b/rsconnect/subprocesses/environment.py new file mode 100644 index 00000000..d2ff5928 --- /dev/null +++ b/rsconnect/subprocesses/environment.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python +""" +Environment data class abstraction that is usable as an executable module + +```bash +python -m rsconnect.utils.environment +``` +""" +from __future__ import annotations + +import datetime +import json +import locale +import os +import re +import subprocess +import sys +from dataclasses import asdict, dataclass, replace +from typing import Callable, Optional + +version_re = re.compile(r"\d+\.\d+(\.\d+)?") +exec_dir = os.path.dirname(sys.executable) + + +@dataclass(frozen=True) +class EnvironmentData: + contents: str + filename: str + locale: str + package_manager: str + pip: str + python: str + source: str + python_requires: Optional[str] + error: Optional[str] + + def _asdict(self): + return asdict(self) + + def _replace(self, **kwargs: object): + return replace(self, **kwargs) + + +def MakeEnvironmentData( + contents: str, + filename: str, + locale: str, + package_manager: str, + pip: str, + python: str, + source: str, + python_requires: Optional[str] = None, + error: Optional[str] = None, + **kwargs: object, # provides compatibility where we no longer support some older properties +) -> Environment: + return Environment(contents, filename, locale, package_manager, pip, python, source, python_requires, error) + + +class EnvironmentException(Exception): + pass + + +def detect_environment(dirname: str, force_generate: bool = False) -> EnvironmentData: + """Determine the python dependencies in the environment. + + `pip freeze` will be used to introspect the environment. + + :param: dirname Directory name + :param: force_generate Force the generation of an environment + :return: a dictionary containing the package spec filename and contents if successful, + or a dictionary containing `error` on failure. + """ + + if force_generate: + result = pip_freeze() + else: + result = output_file(dirname, "requirements.txt", "pip") or pip_freeze() + + if result is not None: + result["python"] = get_python_version() + result["pip"] = get_version("pip") + result["locale"] = get_default_locale() + + return MakeEnvironmentData(**result) + + +def get_python_version() -> str: + v = sys.version_info + return "%d.%d.%d" % (v[0], v[1], v[2]) + + +def get_default_locale(locale_source: Callable[..., tuple[str | None, str | None]] = locale.getlocale): + result = ".".join([item or "" for item in locale_source()]) + return "" if result == "." else result + + +def get_version(module: str): + try: + args = [sys.executable, "-m", module, "--version"] + proc = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, _stderr = proc.communicate() + match = version_re.search(stdout) + if match: + return match.group() + + msg = "Failed to get version of '%s' from the output of: %s" % ( + module, + " ".join(args), + ) + raise EnvironmentException(msg) + except Exception as exception: + raise EnvironmentException("Error getting '%s' version: %s" % (module, str(exception))) + + +def output_file(dirname: str, filename: str, package_manager: str): + """Read an existing package spec file. + + Returns a dictionary containing the filename and contents + if successful, None if the file does not exist, + or a dictionary containing 'error' on failure. + """ + try: + path = os.path.join(dirname, filename) + if not os.path.exists(path): + return None + + with open(path, "r") as f: + data = f.read() + + data = "\n".join([line for line in data.split("\n") if "rsconnect" not in line]) + + return { + "filename": filename, + "contents": data, + "source": "file", + "package_manager": package_manager, + } + except Exception as exception: + raise EnvironmentException("Error reading %s: %s" % (filename, str(exception))) + + +def pip_freeze(): + """Inspect the environment using `pip freeze --disable-pip-version-check version`. + + Returns a dictionary containing the filename + (always 'requirements.txt') and contents if successful, + or a dictionary containing 'error' on failure. + """ + try: + proc = subprocess.Popen( + [sys.executable, "-m", "pip", "freeze", "--disable-pip-version-check"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + pip_stdout, pip_stderr = proc.communicate() + pip_status = proc.returncode + except Exception as exception: + raise EnvironmentException("Error during pip freeze: %s" % str(exception)) + + if pip_status != 0: + msg = pip_stderr or ("exited with code %d" % pip_status) + raise EnvironmentException("Error during pip freeze: %s" % msg) + + pip_stdout = filter_pip_freeze_output(pip_stdout) + + pip_stdout = ( + "# requirements.txt generated by rsconnect-python on " + + str(datetime.datetime.now(datetime.timezone.utc)) + + "\n" + + pip_stdout + ) + + return { + "filename": "requirements.txt", + "contents": pip_stdout, + "source": "pip_freeze", + "package_manager": "pip", + } + + +def filter_pip_freeze_output(pip_stdout: str): + # Filter out dependency on `rsconnect` and ignore output lines from pip which start with `[notice]` + return "\n".join( + [line for line in pip_stdout.split("\n") if (("rsconnect" not in line) and (line.find("[notice]") != 0))] + ) + + +def strip_ref(line: str): + # remove erroneous conda build paths that will break pip install + return line.split(" @ file:", 1)[0].strip() + + +def exclude(line: str): + return line and line.startswith("setuptools") and "post" in line + + +def main(): + """ + Run `detect_environment` and dump the result as JSON. + """ + try: + if len(sys.argv) < 2: + raise EnvironmentException("Usage: %s [-fc] DIRECTORY" % sys.argv[0]) + # directory is always the last argument + directory = sys.argv[len(sys.argv) - 1] + flags = "" + force_generate = False + if len(sys.argv) > 2: + flags = sys.argv[1] + if "f" in flags: + force_generate = True + envinfo = detect_environment(directory, force_generate)._asdict() + if "contents" in envinfo: + keepers = list(map(strip_ref, envinfo["contents"].split("\n"))) + keepers = [line for line in keepers if not exclude(line)] + envinfo["contents"] = "\n".join(keepers) + + json.dump( + envinfo, + sys.stdout, + indent=4, + ) + except EnvironmentException as exception: + json.dump(dict(error=str(exception)), sys.stdout, indent=4) + + +if __name__ == "__main__": + main() From 357ec679342a12a064a535c04482dc4f68d7fb98 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 21 Mar 2025 10:17:40 +0100 Subject: [PATCH 02/24] migreate all deploys to create_python_environment --- rsconnect/bundle.py | 3 ++- rsconnect/main.py | 27 ++++++++++++++++++++++----- rsconnect/subprocesses/environment.py | 2 +- tests/test_bundle.py | 6 +++--- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 1a55b6c2..f8b5e31a 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -2306,7 +2306,8 @@ def create_python_environment( python_version_requirement = override_python_version # with cli_feedback("Inspecting Python environment"): - _, environment = _get_python_env_info(module_file, python, force_generate, override_python_version) + detected_python, environment = _get_python_env_info(module_file, python, force_generate, override_python_version) + environment.python = detected_python # If python is provided, _get_python_env_info will return it as is. if force_generate: _warn_on_ignored_requirements(directory, environment.filename) diff --git a/rsconnect/main.py b/rsconnect/main.py index 736ed2eb..4a98447f 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -53,7 +53,6 @@ create_python_environment, default_title_from_manifest, fake_module_file_from_directory, - get_python_env_info, is_environment_dir, make_api_bundle, make_html_bundle, @@ -1813,7 +1812,9 @@ def write_manifest_notebook( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - python, environment = get_python_env_info(file, python, force_generate, override_python_version) + environment = create_python_environment(base_dir, force_generate=force_generate, python=python, + override_python_version=override_python_version, + app_file=file) with cli_feedback("Creating manifest.json"): environment_file_exists = write_notebook_manifest_json( @@ -1919,7 +1920,13 @@ def write_manifest_voila( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - python, environment = get_python_env_info(path, python, force_generate, override_python_version) + environment = create_python_environment( + base_dir, + force_generate=force_generate, + override_python_version=override_python_version, + python=python, + app_file=path, + ) environment_file_exists = exists(join(base_dir, environment.filename)) @@ -2042,7 +2049,12 @@ def write_manifest_quarto( environment = None if "jupyter" in engines: with cli_feedback("Inspecting Python environment"): - python, environment = get_python_env_info(base_dir, python, force_generate, override_python_version) + environment = create_python_environment( + base_dir, + force_generate=force_generate, + override_python_version=override_python_version, + python=python + ) environment_file_exists = exists(join(base_dir, environment.filename)) if environment_file_exists and not force_generate: @@ -2285,7 +2297,12 @@ def _write_framework_manifest( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - _, environment = get_python_env_info(directory, python, force_generate, override_python_version) + environment = create_python_environment( + directory, + force_generate=force_generate, + override_python_version=override_python_version, + python=python, + ) if app_mode == AppModes.PYTHON_SHINY: with cli_feedback("Inspecting Shiny for Python app"): diff --git a/rsconnect/subprocesses/environment.py b/rsconnect/subprocesses/environment.py index d2ff5928..ab749b7e 100644 --- a/rsconnect/subprocesses/environment.py +++ b/rsconnect/subprocesses/environment.py @@ -3,7 +3,7 @@ Environment data class abstraction that is usable as an executable module ```bash -python -m rsconnect.utils.environment +python -m rsconnect.subprocesses.environment ``` """ from __future__ import annotations diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 30e727c2..509e9525 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -19,7 +19,7 @@ create_html_manifest, create_python_environment, create_voila_manifest, - get_python_env_info, + _get_python_env_info, guess_deploy_dir, inspect_environment, keep_manifest_specified_file, @@ -1298,9 +1298,9 @@ def fake_inspect_environment( if expected_environment.error is not None: with pytest.raises(RSConnectException): - _, _ = get_python_env_info(file_name, python, force_generate=force_generate) + _, _ = _get_python_env_info(file_name, python, force_generate=force_generate) else: - python, environment = get_python_env_info(file_name, python, force_generate=force_generate) + python, environment = _get_python_env_info(file_name, python, force_generate=force_generate) assert python == expected_python assert environment == expected_environment From d67458cd3dc7a90041f0c0d548e85194e6a23f3b Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 21 Mar 2025 11:02:00 +0100 Subject: [PATCH 03/24] Consolidate Metadata creation --- rsconnect/bundle.py | 107 ++++++++++---------------- rsconnect/environment.py | 8 +- rsconnect/subprocesses/environment.py | 2 +- 3 files changed, 44 insertions(+), 73 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index f8b5e31a..3b2c0c28 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -104,9 +104,14 @@ class ManifestDataQuarto(TypedDict): engines: list[str] +class ManifestDataEnvironmentPython(TypedDict): + requires: NotRequired[str] + + class ManifestDataEnvironment(TypedDict): image: NotRequired[str] environment_management: NotRequired[dict[Literal["python", "r"], bool]] + python: NotRequired[ManifestDataEnvironmentPython] class ManifestDataPython(TypedDict): @@ -192,16 +197,24 @@ def __init__( }, } + if environment.python_version_requirement: + # If the environment has a python version requirement, + # add it to the manifest as environment.python.requires + manifest_environment = self.data.setdefault("environment", {}) + manifest_environment["python"] = { + "requires": environment.python_version_requirement + } + if image or env_management_py is not None or env_management_r is not None: - self.data["environment"] = {} + manifest_environment = self.data.setdefault("environment", {}) if image: - self.data["environment"]["image"] = image + manifest_environment["image"] = image if env_management_py is not None or env_management_r is not None: - self.data["environment"]["environment_management"] = {} + manifest_environment["environment_management"] = {} if env_management_py is not None: - self.data["environment"]["environment_management"]["python"] = env_management_py + manifest_environment["environment_management"]["python"] = env_management_py if env_management_r is not None: - self.data["environment"]["environment_management"]["r"] = env_management_r + manifest_environment["environment_management"]["r"] = env_management_r self.data["files"] = {} if files: @@ -379,56 +392,16 @@ def make_source_manifest( env_management_py: Optional[bool] = None, env_management_r: Optional[bool] = None, ) -> ManifestData: - manifest: ManifestData = cast(ManifestData, {"version": 1}) - - # When adding locale, add it early so it is ordered immediately after - # version. - if environment: - manifest["locale"] = environment.locale - - manifest["metadata"] = { - "appmode": app_mode.name(), - } - - if entrypoint: - manifest["metadata"]["entrypoint"] = entrypoint - - if quarto_inspection: - manifest["quarto"] = { - "version": quarto_inspection.get("quarto", {}).get("version", "99.9.9"), - "engines": quarto_inspection.get("engines", []), - } - - files_data = quarto_inspection.get("files", {}) - files_input_data = files_data.get("input", []) - if len(files_input_data) > 1: - manifest["metadata"]["content_category"] = "site" - - if environment: - package_manager = environment.package_manager - manifest["python"] = { - "version": environment.python, - "package_manager": { - "name": package_manager, - "version": getattr(environment, package_manager), - "package_file": environment.filename, - }, - } - - if image or env_management_py is not None or env_management_r is not None: - manifest["environment"] = {} - if image: - manifest["environment"]["image"] = image - if env_management_py is not None or env_management_r is not None: - manifest["environment"]["environment_management"] = {} - if env_management_py is not None: - manifest["environment"]["environment_management"]["python"] = env_management_py - if env_management_r is not None: - manifest["environment"]["environment_management"]["r"] = env_management_r - - manifest["files"] = {} - - return manifest + manifest: Manifest = Manifest( + app_mode=app_mode, + environment=environment, + entrypoint=entrypoint, + quarto_inspection=quarto_inspection, + image=image, + env_management_py=env_management_py, + env_management_r=env_management_r, + ) + return manifest.data def manifest_add_file(manifest: ManifestData, rel_path: str, base_dir: str) -> None: @@ -704,15 +677,13 @@ def make_html_manifest( filename: str, ) -> ManifestData: # noinspection SpellCheckingInspection - manifest: ManifestData = { - "version": 1, - "metadata": { - "appmode": "static", - "primary_html": filename, - }, - "files": {}, - } - return manifest + manifest: Manifest( + metadata=ManifestDataMetadata( + appmode="static", + primary_html=filename, + ) + ) + return manifest.data def make_notebook_html_bundle( @@ -1790,7 +1761,7 @@ def _get_python_env_info( environment.python_version_requirement = python_version_requirement if override_python_version: - environment = environment.python = override_python_version + environment.python = override_python_version return python, environment @@ -2303,11 +2274,11 @@ def create_python_environment( # TODO: --override-python-version should be deprecated in the future # and instead we should suggest the user sets it in .python-version # or pyproject.toml - python_version_requirement = override_python_version + python_version_requirement = f"=={override_python_version}" # with cli_feedback("Inspecting Python environment"): - detected_python, environment = _get_python_env_info(module_file, python, force_generate, override_python_version) - environment.python = detected_python # If python is provided, _get_python_env_info will return it as is. + detected_python, environment = _get_python_env_info(module_file, python, force_generate, override_python_version, + python_version_requirement) if force_generate: _warn_on_ignored_requirements(directory, environment.filename) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 9aebda77..4c226553 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -9,21 +9,21 @@ class Environment: The data is loaded from a rsconnect.utils.environment json response """ + DATA_FIELDS = dataclasses.fields(EnvironmentData) def __init__(self, data: EnvironmentData, python_version_requirement: typing.Optional[str] = None): self._data = data - self._data_fields = dataclasses.fields(self.data) # Fields that are not loaded from the environment subprocess - self.python_version_requirement + self.python_version_requirement = python_version_requirement def __getattr__(self, name: str) -> typing.Any: # We directly proxy the attributes of the EnvironmentData object # so that schema changes can be handled in EnvironmentData exclusively. - return self._data[name] + return getattr(self._data, name) def __setattr__(self, name, value): - if name in self._data_fields: + if name in self.DATA_FIELDS: # proxy the attribute to the underlying EnvironmentData object self._data._replace(name=value) else: diff --git a/rsconnect/subprocesses/environment.py b/rsconnect/subprocesses/environment.py index ab749b7e..61652709 100644 --- a/rsconnect/subprocesses/environment.py +++ b/rsconnect/subprocesses/environment.py @@ -53,7 +53,7 @@ def MakeEnvironmentData( error: Optional[str] = None, **kwargs: object, # provides compatibility where we no longer support some older properties ) -> Environment: - return Environment(contents, filename, locale, package_manager, pip, python, source, python_requires, error) + return EnvironmentData(contents, filename, locale, package_manager, pip, python, source, python_requires, error) class EnvironmentException(Exception): From fd0b137ca007a1f81cbbdeb384eed5be83603495 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 21 Mar 2025 11:08:39 +0100 Subject: [PATCH 04/24] Tweaks --- rsconnect/bundle.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 3b2c0c28..8091345f 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -1650,7 +1650,7 @@ def _warn_on_missing_python_version(version_constraint: Optional[str]) -> None: """ if version_constraint is None: click.secho( - " Warning: Python version constraint missing from pyproject.toml or .python-version\n" + " Warning: Python version constraint missing from pyproject.toml, setup.cfg or .python-version\n" " Connect will guess the version to use based on local environment.\n" " Consider specifying a Python version constraint.", fg="yellow", @@ -1734,9 +1734,7 @@ def inspect_environment( def _get_python_env_info( file_name: str, python: str | None, - force_generate: bool = False, - override_python_version: str | None = None, - python_version_requirement: str | None = None, + force_generate: bool = False ) -> tuple[str, Environment]: """ Gathers the python and environment information relating to the specified file @@ -1756,13 +1754,6 @@ def _get_python_env_info( raise RSConnectException(environment.error) logger.debug("Python: %s" % python) logger.debug("Environment: %s" % pformat(environment._asdict())) - - if python_version_requirement: - environment.python_version_requirement = python_version_requirement - - if override_python_version: - environment.python = override_python_version - return python, environment @@ -2277,8 +2268,13 @@ def create_python_environment( python_version_requirement = f"=={override_python_version}" # with cli_feedback("Inspecting Python environment"): - detected_python, environment = _get_python_env_info(module_file, python, force_generate, override_python_version, - python_version_requirement) + detected_python, environment = _get_python_env_info(module_file, python, force_generate) + environment.python_version_requirement = python_version_requirement + + if override_python_version: + # Retaing backward compatibility with old Connect versions + # that didn't support environment.python.requires + environment.python = override_python_version if force_generate: _warn_on_ignored_requirements(directory, environment.filename) From f71bf06603cc61194d1af1a2fed14a4694720d70 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 21 Mar 2025 16:56:50 +0100 Subject: [PATCH 05/24] Lint and format --- rsconnect/bundle.py | 15 +++---- rsconnect/environment.py | 3 +- rsconnect/main.py | 61 ++++++++------------------- rsconnect/pyproject.py | 2 +- rsconnect/subprocesses/environment.py | 2 +- 5 files changed, 27 insertions(+), 56 deletions(-) diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index 8091345f..db985ddb 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -201,9 +201,7 @@ def __init__( # If the environment has a python version requirement, # add it to the manifest as environment.python.requires manifest_environment = self.data.setdefault("environment", {}) - manifest_environment["python"] = { - "requires": environment.python_version_requirement - } + manifest_environment["python"] = {"requires": environment.python_version_requirement} if image or env_management_py is not None or env_management_r is not None: manifest_environment = self.data.setdefault("environment", {}) @@ -677,9 +675,10 @@ def make_html_manifest( filename: str, ) -> ManifestData: # noinspection SpellCheckingInspection + appmode = "static" manifest: Manifest( metadata=ManifestDataMetadata( - appmode="static", + appmode=appmode, primary_html=filename, ) ) @@ -1723,7 +1722,7 @@ def inspect_environment( if "error" in environment_data: system_error_message = environment_data.get("error") if system_error_message: - raise RSConnectException(f"Error creating environment: {system_error_message}") from e + raise RSConnectException(f"Error creating environment: {system_error_message}") try: return Environment.from_json(environment_data) @@ -1731,11 +1730,7 @@ def inspect_environment( raise RSConnectException("Error constructing environment object") from e -def _get_python_env_info( - file_name: str, - python: str | None, - force_generate: bool = False -) -> tuple[str, Environment]: +def _get_python_env_info(file_name: str, python: str | None, force_generate: bool = False) -> tuple[str, Environment]: """ Gathers the python and environment information relating to the specified file with an eye to deploy it. diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 4c226553..a10899ba 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -9,6 +9,7 @@ class Environment: The data is loaded from a rsconnect.utils.environment json response """ + DATA_FIELDS = dataclasses.fields(EnvironmentData) def __init__(self, data: EnvironmentData, python_version_requirement: typing.Optional[str] = None): @@ -22,7 +23,7 @@ def __getattr__(self, name: str) -> typing.Any: # so that schema changes can be handled in EnvironmentData exclusively. return getattr(self._data, name) - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: typing.Any) -> None: if name in self.DATA_FIELDS: # proxy the attribute to the underlying EnvironmentData object self._data._replace(name=value) diff --git a/rsconnect/main.py b/rsconnect/main.py index 4a98447f..2d2aeb43 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -53,7 +53,6 @@ create_python_environment, default_title_from_manifest, fake_module_file_from_directory, - is_environment_dir, make_api_bundle, make_html_bundle, make_manifest_bundle, @@ -809,35 +808,6 @@ def _warn_on_ignored_manifest(directory: str): ) -def _warn_if_no_requirements_file(directory: str): - """ - Checks for the existence of a file called requirements.txt in the given directory. - If it's not there, a warning will be printed. - - :param directory: the directory to check in. - """ - if not exists(join(directory, "requirements.txt")): - click.secho( - " Warning: Capturing the environment using 'pip freeze --disable-pip-version-check'.\n" - " Consider creating a requirements.txt file instead.", - fg="yellow", - ) - - -def _warn_if_environment_directory(directory: str): - """ - Issue a warning if the deployment directory is itself a virtualenv (yikes!). - - :param directory: the directory to check in. - """ - if is_environment_dir(directory): - click.secho( - " Warning: The deployment directory appears to be a python virtual environment.\n" - " Python libraries and binaries will be excluded from the deployment.", - fg="yellow", - ) - - def _warn_on_ignored_requirements(directory: str, requirements_file_name: str): """ Checks for the existence of a file called manifest.json in the given directory. @@ -944,10 +914,13 @@ def deploy_notebook( app_mode = AppModes.JUPYTER_NOTEBOOK if not static else AppModes.STATIC base_dir = dirname(file) - environment = create_python_environment(base_dir, app_file=file, - force_generate=force_generate, - python=python, - override_python_version=override_python_version) + environment = create_python_environment( + base_dir, + app_file=file, + force_generate=force_generate, + python=python, + override_python_version=override_python_version, + ) if force_generate: _warn_on_ignored_requirements(base_dir, environment.filename) @@ -1299,8 +1272,9 @@ def deploy_quarto( environment = None if "jupyter" in engines: with cli_feedback("Inspecting Python environment"): - environment = create_python_environment(base_dir, force_generate=force_generate, - override_python_version=override_python_version) + environment = create_python_environment( + base_dir, force_generate=force_generate, override_python_version=override_python_version + ) ce = RSConnectExecutor( ctx=ctx, @@ -1812,9 +1786,13 @@ def write_manifest_notebook( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - environment = create_python_environment(base_dir, force_generate=force_generate, python=python, - override_python_version=override_python_version, - app_file=file) + environment = create_python_environment( + base_dir, + force_generate=force_generate, + python=python, + override_python_version=override_python_version, + app_file=file, + ) with cli_feedback("Creating manifest.json"): environment_file_exists = write_notebook_manifest_json( @@ -2050,10 +2028,7 @@ def write_manifest_quarto( if "jupyter" in engines: with cli_feedback("Inspecting Python environment"): environment = create_python_environment( - base_dir, - force_generate=force_generate, - override_python_version=override_python_version, - python=python + base_dir, force_generate=force_generate, override_python_version=override_python_version, python=python ) environment_file_exists = exists(join(base_dir, environment.filename)) diff --git a/rsconnect/pyproject.py b/rsconnect/pyproject.py index febd719d..0e3c0adc 100644 --- a/rsconnect/pyproject.py +++ b/rsconnect/pyproject.py @@ -55,7 +55,7 @@ def _generate(): def get_python_version_requirement_parser( metadata_file: pathlib.Path, -) -> typing.Optional[typing.Callable[[pathlib.Path], typing.Optional[str]]]: +) -> typing.Callable[[pathlib.Path], typing.Optional[str]]: """Given the metadata file, return the appropriate parser function. The returned function takes a pathlib.Path and returns the parsed value. diff --git a/rsconnect/subprocesses/environment.py b/rsconnect/subprocesses/environment.py index 61652709..76d44d70 100644 --- a/rsconnect/subprocesses/environment.py +++ b/rsconnect/subprocesses/environment.py @@ -52,7 +52,7 @@ def MakeEnvironmentData( python_requires: Optional[str] = None, error: Optional[str] = None, **kwargs: object, # provides compatibility where we no longer support some older properties -) -> Environment: +) -> EnvironmentData: return EnvironmentData(contents, filename, locale, package_manager, pip, python, source, python_requires, error) From d135e3f55d6a2f847187d37a0e0cb5a2dadedc2e Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 21 Mar 2025 17:27:21 +0100 Subject: [PATCH 06/24] Further refactoring to consolidate Environment --- rsconnect/actions.py | 3 +- rsconnect/bundle.py | 241 +------------------------------------ rsconnect/environment.py | 253 ++++++++++++++++++++++++++++++++++++++- rsconnect/main.py | 19 ++- 4 files changed, 263 insertions(+), 253 deletions(-) diff --git a/rsconnect/actions.py b/rsconnect/actions.py index 5df9aeb9..1045dff8 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -31,7 +31,6 @@ from . import api from .bundle import ( - create_python_environment, get_default_entrypoint, make_api_bundle, make_quarto_source_bundle, @@ -373,7 +372,7 @@ def deploy_app( ) ) - environment = create_python_environment( + _, environment = Environment.create_python_environment( directory, # pyright: ignore force_generate, python, diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index db985ddb..274b7ee8 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -30,7 +30,6 @@ splitext, ) from pathlib import Path -from pprint import pformat from typing import ( IO, TYPE_CHECKING, @@ -53,8 +52,7 @@ import click -from . import pyproject -from .environment import Environment +from .environment import Environment, list_environment_dirs, is_environment_dir from .exception import RSConnectException from .log import VERBOSE, logger from .models import AppMode, AppModes, GlobSet @@ -675,10 +673,9 @@ def make_html_manifest( filename: str, ) -> ManifestData: # noinspection SpellCheckingInspection - appmode = "static" - manifest: Manifest( + manifest = Manifest( metadata=ManifestDataMetadata( - appmode=appmode, + appmode="static", primary_html=filename, ) ) @@ -845,27 +842,6 @@ def create_glob_set(directory: str | Path, excludes: Sequence[str]) -> GlobSet: return GlobSet(work) -def is_environment_dir(directory: str | Path): - """Detect whether `directory` is a virtualenv""" - - # A virtualenv will have Python at ./bin/python - python_path = join(directory, "bin", "python") - # But on Windows, it's at Scripts\Python.exe - win_path = join(directory, "Scripts", "Python.exe") - return exists(python_path) or exists(win_path) - - -def list_environment_dirs(directory: str | Path): - """Returns a list of subdirectories in `directory` that appear to contain virtual environments.""" - envs: list[str] = [] - - for name in os.listdir(directory): - path = join(directory, name) - if is_environment_dir(path): - envs.append(name) - return envs - - def make_api_manifest( directory: str, entry_point: str, @@ -1582,176 +1558,6 @@ def _warn_on_ignored_entrypoint(entrypoint: Optional[str]) -> None: ) -def _warn_on_ignored_manifest(directory: str) -> None: - """ - Checks for the existence of a file called manifest.json in the given directory. - If it's there, a warning noting that it will be ignored will be printed. - - :param directory: the directory to check in. - """ - if exists(join(directory, "manifest.json")): - click.secho( - " Warning: the existing manifest.json file will not be used or considered.", - fg="yellow", - ) - - -def _warn_if_no_requirements_file(directory: str) -> None: - """ - Checks for the existence of a file called requirements.txt in the given directory. - If it's not there, a warning will be printed. - - :param directory: the directory to check in. - """ - if not exists(join(directory, "requirements.txt")): - click.secho( - " Warning: Capturing the environment using 'pip freeze'.\n" - " Consider creating a requirements.txt file instead.", - fg="yellow", - ) - - -def _warn_if_environment_directory(directory: str | Path) -> None: - """ - Issue a warning if the deployment directory is itself a virtualenv (yikes!). - - :param directory: the directory to check in. - """ - if is_environment_dir(directory): - click.secho( - " Warning: The deployment directory appears to be a python virtual environment.\n" - " Python libraries and binaries will be excluded from the deployment.", - fg="yellow", - ) - - -def _warn_on_ignored_requirements(directory: str, requirements_file_name: str) -> None: - """ - Checks for the existence of a file called manifest.json in the given directory. - If it's there, a warning noting that it will be ignored will be printed. - - :param directory: the directory to check in. - :param requirements_file_name: the name of the requirements file. - """ - if exists(join(directory, requirements_file_name)): - click.secho( - " Warning: the existing %s file will not be used or considered." % requirements_file_name, - fg="yellow", - ) - - -def _warn_on_missing_python_version(version_constraint: Optional[str]) -> None: - """ - Check that the project has a Python version constraint requested. - If it doesn't warn the user that it should be specified. - - :param version_constraint: the version constraint in the project. - """ - if version_constraint is None: - click.secho( - " Warning: Python version constraint missing from pyproject.toml, setup.cfg or .python-version\n" - " Connect will guess the version to use based on local environment.\n" - " Consider specifying a Python version constraint.", - fg="yellow", - ) - - -def fake_module_file_from_directory(directory: str) -> str: - """ - Takes a directory and invents a properly named file that though possibly fake, - can be used for other name/title derivation. - - :param directory: the directory to start with. - :return: the directory plus the (potentially) fake module file. - """ - app_name = abspath(directory) - app_name = dirname(app_name) if app_name.endswith(os.path.sep) else basename(app_name) - return join(directory, app_name + ".py") - - -def which_python(python: Optional[str] = None) -> str: - """Determines which Python executable to use. - - If the :param python: is provided, then validation is performed to check if the path is an executable file. If - None, the invoking system Python executable location is returned. - - :param python: (Optional) path to a python executable. - :return: :param python: or `sys.executable`. - """ - if python is None: - return sys.executable - if not exists(python): - raise RSConnectException(f"The path '{python}' does not exist. Expected a Python executable.") - if isdir(python): - raise RSConnectException(f"The path '{python}' is a directory. Expected a Python executable.") - if not os.access(python, os.X_OK): - raise RSConnectException(f"The path '{python}' is not executable. Expected a Python executable") - return python - - -def inspect_environment( - python: str, - directory: str, - force_generate: bool = False, - check_output: Callable[..., bytes] = subprocess.check_output, -) -> Environment: - """Run the environment inspector using the specified python binary. - - Returns a dictionary of information about the environment, - or containing an "error" field if an error occurred. - """ - flags: list[str] = [] - if force_generate: - flags.append("f") - - args = [python, "-m", "rsconnect.subprocesses.environment"] - if flags: - args.append("-" + "".join(flags)) - args.append(directory) - - try: - environment_json = check_output(args, text=True) - except Exception as e: - raise RSConnectException("Error inspecting environment (subprocess failed)") from e - - try: - environment_data = json.loads(environment_json) - except json.JSONDecodeError as e: - raise RSConnectException("Error parsing environment JSON") from e - - if "error" in environment_data: - system_error_message = environment_data.get("error") - if system_error_message: - raise RSConnectException(f"Error creating environment: {system_error_message}") - - try: - return Environment.from_json(environment_data) - except TypeError as e: - raise RSConnectException("Error constructing environment object") from e - - -def _get_python_env_info(file_name: str, python: str | None, force_generate: bool = False) -> tuple[str, Environment]: - """ - Gathers the python and environment information relating to the specified file - with an eye to deploy it. - - :param file_name: the primary file being deployed. - :param python: the optional name of a Python executable. - :param force_generate: force generating "requirements.txt" or "environment.yml", - even if it already exists. - :return: information about the version of Python in use plus some environmental - stuff. - """ - python = which_python(python) - logger.debug("Python: %s" % python) - environment = inspect_environment(python, dirname(file_name), force_generate=force_generate) - if environment.error: - raise RSConnectException(environment.error) - logger.debug("Python: %s" % python) - logger.debug("Environment: %s" % pformat(environment._asdict())) - return python, environment - - def create_notebook_manifest_and_environment_file( entry_point_file: str, environment: Environment, @@ -2234,44 +2040,3 @@ def write_manifest_json(manifest_path: str | Path, manifest: ManifestData) -> No with open(manifest_path, "w") as f: json.dump(manifest, f, indent=2) f.write("\n") - - -def create_python_environment( - directory: str, - force_generate: bool = False, - python: Optional[str] = None, - override_python_version: Optional[str] = None, - app_file: Optional[str] = None, -) -> Environment: - if app_file is None: - module_file = fake_module_file_from_directory(directory) - else: - module_file = app_file - - # click.secho(' Deploying %s to server "%s"' % (directory, connect_server.url)) - _warn_on_ignored_manifest(directory) - _warn_if_no_requirements_file(directory) - _warn_if_environment_directory(directory) - - python_version_requirement = pyproject.detect_python_version_requirement(directory) - _warn_on_missing_python_version(python_version_requirement) - - if override_python_version: - # TODO: --override-python-version should be deprecated in the future - # and instead we should suggest the user sets it in .python-version - # or pyproject.toml - python_version_requirement = f"=={override_python_version}" - - # with cli_feedback("Inspecting Python environment"): - detected_python, environment = _get_python_env_info(module_file, python, force_generate) - environment.python_version_requirement = python_version_requirement - - if override_python_version: - # Retaing backward compatibility with old Connect versions - # that didn't support environment.python.requires - environment.python = override_python_version - - if force_generate: - _warn_on_ignored_requirements(directory, environment.filename) - - return environment diff --git a/rsconnect/environment.py b/rsconnect/environment.py index a10899ba..77314fd8 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -1,8 +1,19 @@ import typing +import sys import dataclasses +import pprint +import subprocess +import json +import pathlib +import os.path +from . import pyproject +from .log import logger +from .exception import RSConnectException from .subprocesses.environment import EnvironmentData, MakeEnvironmentData as _MakeEnvironmentData +import click + class Environment: """A project environment, @@ -10,7 +21,7 @@ class Environment: The data is loaded from a rsconnect.utils.environment json response """ - DATA_FIELDS = dataclasses.fields(EnvironmentData) + DATA_FIELDS = {f.name for f in dataclasses.fields(EnvironmentData)} def __init__(self, data: EnvironmentData, python_version_requirement: typing.Optional[str] = None): self._data = data @@ -31,6 +42,242 @@ def __setattr__(self, name: str, value: typing.Any) -> None: super().__setattr__(name, value) @classmethod - def from_json(cls, json_data: dict) -> "Environment": + def from_dict(cls, data: dict[str, typing.Any]) -> "Environment": """Create an Environment instance from the JSON representation of EnvironmentData.""" - return cls(_MakeEnvironmentData(**json_data)) + return cls(_MakeEnvironmentData(**data)) + + @classmethod + def create_python_environment( + cls, + directory: str, + force_generate: bool = False, + python: typing.Optional[str] = None, + override_python_version: typing.Optional[str] = None, + app_file: typing.Optional[str] = None, + ) -> typing.Tuple[str, "Environment"]: + if app_file is None: + module_file = fake_module_file_from_directory(directory) + else: + module_file = app_file + + # click.secho(' Deploying %s to server "%s"' % (directory, connect_server.url)) + _warn_on_ignored_manifest(directory) + _warn_if_no_requirements_file(directory) + _warn_if_environment_directory(directory) + + python_version_requirement = pyproject.detect_python_version_requirement(directory) + _warn_on_missing_python_version(python_version_requirement) + + if override_python_version: + # TODO: --override-python-version should be deprecated in the future + # and instead we should suggest the user sets it in .python-version + # or pyproject.toml + python_version_requirement = f"=={override_python_version}" + + # with cli_feedback("Inspecting Python environment"): + python_interpreter, environment = cls._get_python_env_info(module_file, python, force_generate) + environment.python_version_requirement = python_version_requirement + + if override_python_version: + # Retaing backward compatibility with old Connect versions + # that didn't support environment.python.requires + environment.python = override_python_version + + if force_generate: + _warn_on_ignored_requirements(directory, environment.filename) + + return python_interpreter, environment + + @classmethod + def _get_python_env_info( + cls, file_name: str, python: str | None, force_generate: bool = False + ) -> tuple[str, "Environment"]: + """ + Gathers the python and environment information relating to the specified file + with an eye to deploy it. + + :param file_name: the primary file being deployed. + :param python: the optional name of a Python executable. + :param force_generate: force generating "requirements.txt" or "environment.yml", + even if it already exists. + :return: information about the version of Python in use plus some environmental + stuff. + """ + python = which_python(python) + logger.debug("Python: %s" % python) + environment = cls._inspect_environment(python, os.path.dirname(file_name), force_generate=force_generate) + if environment.error: + raise RSConnectException(environment.error) + logger.debug("Python: %s" % python) + logger.debug("Environment: %s" % pprint.pformat(environment._asdict())) + return python, environment + + @classmethod + def _inspect_environment( + cls, + python: str, + directory: str, + force_generate: bool = False, + check_output: typing.Callable[..., bytes] = subprocess.check_output, + ) -> "Environment": + """Run the environment inspector using the specified python binary. + + Returns a dictionary of information about the environment, + or containing an "error" field if an error occurred. + """ + flags: list[str] = [] + if force_generate: + flags.append("f") + + args = [python, "-m", "rsconnect.subprocesses.environment"] + if flags: + args.append("-" + "".join(flags)) + args.append(directory) + + try: + environment_json = check_output(args, text=True) + except Exception as e: + raise RSConnectException("Error inspecting environment (subprocess failed)") from e + + try: + environment_data = json.loads(environment_json) + except json.JSONDecodeError as e: + raise RSConnectException("Error parsing environment JSON") from e + + if "error" in environment_data: + system_error_message = environment_data.get("error") + if system_error_message: + raise RSConnectException(f"Error creating environment: {system_error_message}") + + try: + return cls.from_dict(environment_data) + except TypeError as e: + raise RSConnectException("Error constructing environment object") from e + + +def which_python(python: typing.Optional[str] = None) -> str: + """Determines which Python executable to use. + + If the :param python: is provided, then validation is performed to check if the path is an executable file. If + None, the invoking system Python executable location is returned. + + :param python: (Optional) path to a python executable. + :return: :param python: or `sys.executable`. + """ + if python is None: + return sys.executable + if not os.path.exists(python): + raise RSConnectException(f"The path '{python}' does not exist. Expected a Python executable.") + if os.path.isdir(python): + raise RSConnectException(f"The path '{python}' is a directory. Expected a Python executable.") + if not os.access(python, os.X_OK): + raise RSConnectException(f"The path '{python}' is not executable. Expected a Python executable") + return python + + +def fake_module_file_from_directory(directory: str) -> str: + """ + Takes a directory and invents a properly named file that though possibly fake, + can be used for other name/title derivation. + + :param directory: the directory to start with. + :return: the directory plus the (potentially) fake module file. + """ + app_name = os.path.abspath(directory) + app_name = os.path.dirname(app_name) if app_name.endswith(os.path.sep) else os.path.basename(app_name) + return os.path.join(directory, app_name + ".py") + + +def is_environment_dir(directory: str | pathlib.Path) -> bool: + """Detect whether `directory` is a virtualenv""" + + # A virtualenv will have Python at ./bin/python + python_path = os.path.join(directory, "bin", "python") + # But on Windows, it's at Scripts\Python.exe + win_path = os.path.join(directory, "Scripts", "Python.exe") + return os.path.exists(python_path) or os.path.exists(win_path) + + +def list_environment_dirs(directory: str | pathlib.Path) -> list[str]: + """Returns a list of subdirectories in `directory` that appear to contain virtual environments.""" + envs: list[str] = [] + + for name in os.listdir(directory): + path = os.path.join(directory, name) + if is_environment_dir(path): + envs.append(name) + return envs + + +def _warn_on_ignored_manifest(directory: str) -> None: + """ + Checks for the existence of a file called manifest.json in the given directory. + If it's there, a warning noting that it will be ignored will be printed. + + :param directory: the directory to check in. + """ + if os.path.exists(os.path.join(directory, "manifest.json")): + click.secho( + " Warning: the existing manifest.json file will not be used or considered.", + fg="yellow", + ) + + +def _warn_if_no_requirements_file(directory: str) -> None: + """ + Checks for the existence of a file called requirements.txt in the given directory. + If it's not there, a warning will be printed. + + :param directory: the directory to check in. + """ + if not os.path.exists(os.path.join(directory, "requirements.txt")): + click.secho( + " Warning: Capturing the environment using 'pip freeze'.\n" + " Consider creating a requirements.txt file instead.", + fg="yellow", + ) + + +def _warn_if_environment_directory(directory: str | pathlib.Path) -> None: + """ + Issue a warning if the deployment directory is itself a virtualenv (yikes!). + + :param directory: the directory to check in. + """ + if is_environment_dir(directory): + click.secho( + " Warning: The deployment directory appears to be a python virtual environment.\n" + " Python libraries and binaries will be excluded from the deployment.", + fg="yellow", + ) + + +def _warn_on_ignored_requirements(directory: str, requirements_file_name: str) -> None: + """ + Checks for the existence of a file called manifest.json in the given directory. + If it's there, a warning noting that it will be ignored will be printed. + + :param directory: the directory to check in. + :param requirements_file_name: the name of the requirements file. + """ + if os.path.exists(os.path.join(directory, requirements_file_name)): + click.secho( + " Warning: the existing %s file will not be used or considered." % requirements_file_name, + fg="yellow", + ) + + +def _warn_on_missing_python_version(version_constraint: typing.Optional[str]) -> None: + """ + Check that the project has a Python version constraint requested. + If it doesn't warn the user that it should be specified. + + :param version_constraint: the version constraint in the project. + """ + if version_constraint is None: + click.secho( + " Warning: Python version constraint missing from pyproject.toml, setup.cfg or .python-version\n" + " Connect will guess the version to use based on local environment.\n" + " Consider specifying a Python version constraint.", + fg="yellow", + ) diff --git a/rsconnect/main.py b/rsconnect/main.py index 2d2aeb43..19bdf17f 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -48,11 +48,10 @@ get_content, search_content, ) +from .environment import Environment, fake_module_file_from_directory from .api import RSConnectClient, RSConnectExecutor, RSConnectServer from .bundle import ( - create_python_environment, default_title_from_manifest, - fake_module_file_from_directory, make_api_bundle, make_html_bundle, make_manifest_bundle, @@ -914,7 +913,7 @@ def deploy_notebook( app_mode = AppModes.JUPYTER_NOTEBOOK if not static else AppModes.STATIC base_dir = dirname(file) - environment = create_python_environment( + python, environment = Environment.create_python_environment( base_dir, app_file=file, force_generate=force_generate, @@ -1055,7 +1054,7 @@ def deploy_voila( set_verbosity(verbose) output_params(ctx, locals().items()) app_mode = AppModes.JUPYTER_VOILA - environment = create_python_environment( + _, environment = Environment.create_python_environment( path if isdir(path) else dirname(path), force_generate, python, override_python_version ) @@ -1272,7 +1271,7 @@ def deploy_quarto( environment = None if "jupyter" in engines: with cli_feedback("Inspecting Python environment"): - environment = create_python_environment( + _, environment = Environment.create_python_environment( base_dir, force_generate=force_generate, override_python_version=override_python_version ) @@ -1615,7 +1614,7 @@ def deploy_app( set_verbosity(verbose) entrypoint = validate_entry_point(entrypoint, directory) extra_files_list = validate_extra_files(directory, extra_files) - environment = create_python_environment( + _, environment = Environment.create_python_environment( directory, force_generate, python, override_python_version=override_python_version ) @@ -1786,7 +1785,7 @@ def write_manifest_notebook( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - environment = create_python_environment( + _, environment = Environment.create_python_environment( base_dir, force_generate=force_generate, python=python, @@ -1898,7 +1897,7 @@ def write_manifest_voila( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - environment = create_python_environment( + _, environment = Environment.create_python_environment( base_dir, force_generate=force_generate, override_python_version=override_python_version, @@ -2027,7 +2026,7 @@ def write_manifest_quarto( environment = None if "jupyter" in engines: with cli_feedback("Inspecting Python environment"): - environment = create_python_environment( + _, environment = Environment.create_python_environment( base_dir, force_generate=force_generate, override_python_version=override_python_version, python=python ) @@ -2272,7 +2271,7 @@ def _write_framework_manifest( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - environment = create_python_environment( + _, environment = Environment.create_python_environment( directory, force_generate=force_generate, override_python_version=override_python_version, From ffe8a64fffde8b5e6737244954325b00963fd649 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 21 Mar 2025 17:47:03 +0100 Subject: [PATCH 07/24] Move the interpreter into the Environment info --- rsconnect/actions.py | 2 +- rsconnect/environment.py | 59 ++++++++++++++++++++++++++++++++-------- rsconnect/main.py | 18 ++++++------ 3 files changed, 57 insertions(+), 22 deletions(-) diff --git a/rsconnect/actions.py b/rsconnect/actions.py index 1045dff8..2c57b664 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -372,7 +372,7 @@ def deploy_app( ) ) - _, environment = Environment.create_python_environment( + environment = Environment.create_python_environment( directory, # pyright: ignore force_generate, python, diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 77314fd8..fc9402d0 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -1,3 +1,12 @@ +"""Detects the configuration of a Python environment. + +Given a directory and a Python executable, this module inspects the environment +and returns information about the Python version and the environment itself. + +To inspect the environment it relies on a subprocess that runs the `rsconnect.subprocesses.environment` +module. This module is responsible for gathering the environment information and returning it in a JSON format. +""" + import typing import sys import dataclasses @@ -23,11 +32,17 @@ class Environment: DATA_FIELDS = {f.name for f in dataclasses.fields(EnvironmentData)} - def __init__(self, data: EnvironmentData, python_version_requirement: typing.Optional[str] = None): + def __init__( + self, + data: EnvironmentData, + python_interpreter: typing.Optional[str] = None, + python_version_requirement: typing.Optional[str] = None, + ): self._data = data # Fields that are not loaded from the environment subprocess self.python_version_requirement = python_version_requirement + self.python_interpreter = python_interpreter def __getattr__(self, name: str) -> typing.Any: # We directly proxy the attributes of the EnvironmentData object @@ -42,9 +57,18 @@ def __setattr__(self, name: str, value: typing.Any) -> None: super().__setattr__(name, value) @classmethod - def from_dict(cls, data: dict[str, typing.Any]) -> "Environment": - """Create an Environment instance from the JSON representation of EnvironmentData.""" - return cls(_MakeEnvironmentData(**data)) + def from_dict( + cls, + data: dict[str, typing.Any], + python_interpreter: typing.Optional[str] = None, + python_version_requirement: typing.Optional[str] = None, + ) -> "Environment": + """Create an Environment instance from the dictionary representation of EnvironmentData.""" + return cls( + _MakeEnvironmentData(**data), + python_interpreter=python_interpreter, + python_version_requirement=python_version_requirement, + ) @classmethod def create_python_environment( @@ -54,7 +78,20 @@ def create_python_environment( python: typing.Optional[str] = None, override_python_version: typing.Optional[str] = None, app_file: typing.Optional[str] = None, - ) -> typing.Tuple[str, "Environment"]: + ) -> "Environment": + """Given a project directory and a Python executable, return Environment information. + + If no Python executable is provided, the current system Python executable is used. + + :param directory: the project directory to inspect. + :param force_generate: force generating "requirements.txt" to snapshot the environment + packages even if it already exists. + :param python: the Python executable of the environment to use for inspection. + :param override_python_version: the Python version required by the project. + :param app_file: the main application file to use for inspection. + + :return: a tuple containing the Python executable of the environment and the Environment object. + """ if app_file is None: module_file = fake_module_file_from_directory(directory) else: @@ -75,7 +112,7 @@ def create_python_environment( python_version_requirement = f"=={override_python_version}" # with cli_feedback("Inspecting Python environment"): - python_interpreter, environment = cls._get_python_env_info(module_file, python, force_generate) + environment = cls._get_python_env_info(module_file, python, force_generate) environment.python_version_requirement = python_version_requirement if override_python_version: @@ -86,12 +123,10 @@ def create_python_environment( if force_generate: _warn_on_ignored_requirements(directory, environment.filename) - return python_interpreter, environment + return environment @classmethod - def _get_python_env_info( - cls, file_name: str, python: str | None, force_generate: bool = False - ) -> tuple[str, "Environment"]: + def _get_python_env_info(cls, file_name: str, python: str | None, force_generate: bool = False) -> "Environment": """ Gathers the python and environment information relating to the specified file with an eye to deploy it. @@ -110,7 +145,7 @@ def _get_python_env_info( raise RSConnectException(environment.error) logger.debug("Python: %s" % python) logger.debug("Environment: %s" % pprint.pformat(environment._asdict())) - return python, environment + return environment @classmethod def _inspect_environment( @@ -150,7 +185,7 @@ def _inspect_environment( raise RSConnectException(f"Error creating environment: {system_error_message}") try: - return cls.from_dict(environment_data) + return cls.from_dict(environment_data, python_interpreter=python) except TypeError as e: raise RSConnectException("Error constructing environment object") from e diff --git a/rsconnect/main.py b/rsconnect/main.py index 19bdf17f..a979c0e4 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -913,7 +913,7 @@ def deploy_notebook( app_mode = AppModes.JUPYTER_NOTEBOOK if not static else AppModes.STATIC base_dir = dirname(file) - python, environment = Environment.create_python_environment( + environment = Environment.create_python_environment( base_dir, app_file=file, force_generate=force_generate, @@ -944,7 +944,7 @@ def deploy_notebook( ce.make_bundle( make_notebook_html_bundle, file, - python, + environment.python, hide_all_input, hide_tagged_input, ) @@ -1054,7 +1054,7 @@ def deploy_voila( set_verbosity(verbose) output_params(ctx, locals().items()) app_mode = AppModes.JUPYTER_VOILA - _, environment = Environment.create_python_environment( + environment = Environment.create_python_environment( path if isdir(path) else dirname(path), force_generate, python, override_python_version ) @@ -1271,7 +1271,7 @@ def deploy_quarto( environment = None if "jupyter" in engines: with cli_feedback("Inspecting Python environment"): - _, environment = Environment.create_python_environment( + environment = Environment.create_python_environment( base_dir, force_generate=force_generate, override_python_version=override_python_version ) @@ -1614,7 +1614,7 @@ def deploy_app( set_verbosity(verbose) entrypoint = validate_entry_point(entrypoint, directory) extra_files_list = validate_extra_files(directory, extra_files) - _, environment = Environment.create_python_environment( + environment = Environment.create_python_environment( directory, force_generate, python, override_python_version=override_python_version ) @@ -1785,7 +1785,7 @@ def write_manifest_notebook( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - _, environment = Environment.create_python_environment( + environment = Environment.create_python_environment( base_dir, force_generate=force_generate, python=python, @@ -1897,7 +1897,7 @@ def write_manifest_voila( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - _, environment = Environment.create_python_environment( + environment = Environment.create_python_environment( base_dir, force_generate=force_generate, override_python_version=override_python_version, @@ -2026,7 +2026,7 @@ def write_manifest_quarto( environment = None if "jupyter" in engines: with cli_feedback("Inspecting Python environment"): - _, environment = Environment.create_python_environment( + environment = Environment.create_python_environment( base_dir, force_generate=force_generate, override_python_version=override_python_version, python=python ) @@ -2271,7 +2271,7 @@ def _write_framework_manifest( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - _, environment = Environment.create_python_environment( + environment = Environment.create_python_environment( directory, force_generate=force_generate, override_python_version=override_python_version, From 34059547b6e279a81d45f9b6a749c26c1e884702 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 21 Mar 2025 17:49:10 +0100 Subject: [PATCH 08/24] Improve docstring --- rsconnect/environment.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index fc9402d0..0ecc4f26 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -25,9 +25,13 @@ class Environment: - """A project environment, + """A Python project environment, - The data is loaded from a rsconnect.utils.environment json response + The data is loaded from a rsconnect.utils.environment json response, + the environment contains all the information provided by :class:`EnvironmentData` plus + the environment python interpreter and the python interpreter version requirement. + + The goal is to capture all the information needed to replicate such environment. """ DATA_FIELDS = {f.name for f in dataclasses.fields(EnvironmentData)} From 99d70f582ecc53c6273e1d1e24dcf7f82c6b1f52 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 21 Mar 2025 17:55:21 +0100 Subject: [PATCH 09/24] Fix typing and attribute proxying --- rsconnect/api.py | 3 ++- rsconnect/environment.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/rsconnect/api.py b/rsconnect/api.py index adf01b3e..4d455876 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -51,7 +51,8 @@ from typing_extensions import NotRequired, TypedDict from . import validation -from .bundle import _default_title, fake_module_file_from_directory +from .bundle import _default_title +from .environment import fake_module_file_from_directory from .certificates import read_certificate_file from .exception import DeploymentFailedException, RSConnectException from .http_support import CookieJar, HTTPResponse, HTTPServer, JsonData, append_to_path diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 0ecc4f26..180ba146 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -56,7 +56,7 @@ def __getattr__(self, name: str) -> typing.Any: def __setattr__(self, name: str, value: typing.Any) -> None: if name in self.DATA_FIELDS: # proxy the attribute to the underlying EnvironmentData object - self._data._replace(name=value) + self._data._replace(**{name: value}) else: super().__setattr__(name, value) @@ -130,7 +130,7 @@ def create_python_environment( return environment @classmethod - def _get_python_env_info(cls, file_name: str, python: str | None, force_generate: bool = False) -> "Environment": + def _get_python_env_info(cls, file_name: str, python: typing.Optional[str], force_generate: bool = False) -> "Environment": """ Gathers the python and environment information relating to the specified file with an eye to deploy it. @@ -227,7 +227,7 @@ def fake_module_file_from_directory(directory: str) -> str: return os.path.join(directory, app_name + ".py") -def is_environment_dir(directory: str | pathlib.Path) -> bool: +def is_environment_dir(directory: typing.Union[str, pathlib.Path]) -> bool: """Detect whether `directory` is a virtualenv""" # A virtualenv will have Python at ./bin/python @@ -237,7 +237,7 @@ def is_environment_dir(directory: str | pathlib.Path) -> bool: return os.path.exists(python_path) or os.path.exists(win_path) -def list_environment_dirs(directory: str | pathlib.Path) -> list[str]: +def list_environment_dirs(directory: typing.Union[str, pathlib.Path]) -> list[str]: """Returns a list of subdirectories in `directory` that appear to contain virtual environments.""" envs: list[str] = [] @@ -277,7 +277,7 @@ def _warn_if_no_requirements_file(directory: str) -> None: ) -def _warn_if_environment_directory(directory: str | pathlib.Path) -> None: +def _warn_if_environment_directory(directory: typing.Union[str, pathlib.Path]) -> None: """ Issue a warning if the deployment directory is itself a virtualenv (yikes!). From 9d73a301fd93d3e3926db602395e52f5a1870397 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Fri, 21 Mar 2025 17:56:59 +0100 Subject: [PATCH 10/24] format --- rsconnect/environment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 180ba146..0639aa9c 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -130,7 +130,9 @@ def create_python_environment( return environment @classmethod - def _get_python_env_info(cls, file_name: str, python: typing.Optional[str], force_generate: bool = False) -> "Environment": + def _get_python_env_info( + cls, file_name: str, python: typing.Optional[str], force_generate: bool = False + ) -> "Environment": """ Gathers the python and environment information relating to the specified file with an eye to deploy it. From 9c1d0f8123ead653ee836202ca1e015d91cd460b Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 24 Mar 2025 11:59:55 +0100 Subject: [PATCH 11/24] Updated test_bundle --- tests/test_bundle.py | 211 ++++++++------------------------------ tests/test_environment.py | 140 +++++++++++++++++++++++-- 2 files changed, 173 insertions(+), 178 deletions(-) diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 509e9525..b4629ef8 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -11,17 +11,13 @@ import pytest -import rsconnect.bundle from rsconnect.bundle import ( Manifest, _default_title, _default_title_from_manifest, create_html_manifest, - create_python_environment, create_voila_manifest, - _get_python_env_info, guess_deploy_dir, - inspect_environment, keep_manifest_specified_file, list_files, make_api_bundle, @@ -40,9 +36,8 @@ to_bytes, validate_entry_point, validate_extra_files, - which_python, ) -from rsconnect.environment import Environment, MakeEnvironment, detect_environment +from rsconnect.environment import Environment from rsconnect.exception import RSConnectException from rsconnect.models import AppModes @@ -85,7 +80,7 @@ def test_make_notebook_source_bundle1(self): # the test environment. Don't do this in the production code, which # runs in the notebook server. We need the introspection to run in # the kernel environment and not the notebook server environment. - environment = detect_environment(directory) + environment = Environment.create_python_environment(directory) with make_notebook_source_bundle( nb_path, environment, @@ -155,7 +150,7 @@ def test_make_notebook_source_bundle2(self): # the test environment. Don't do this in the production code, which # runs in the notebook server. We need the introspection to run in # the kernel environment and not the notebook server environment. - environment = detect_environment(directory) + environment = Environment.create_python_environment(directory) with make_notebook_source_bundle( nb_path, @@ -255,7 +250,7 @@ def test_make_quarto_source_bundle_from_simple_project(self): # input file. create_fake_quarto_rendered_output(temp_proj, "myquarto") - environment = detect_environment(temp_proj) + environment = Environment.create_python_environment(temp_proj) # mock the result of running of `quarto inspect ` inspect = { @@ -352,7 +347,7 @@ def test_make_quarto_source_bundle_from_complex_project(self): create_fake_quarto_rendered_output(site_dir, "index") create_fake_quarto_rendered_output(site_dir, "about") - environment = detect_environment(temp_proj) + environment = Environment.create_python_environment(temp_proj) # mock the result of running of `quarto inspect ` inspect = { @@ -454,7 +449,7 @@ def test_make_quarto_source_bundle_from_project_with_requirements(self): fp.write("pandas\n") fp.close() - environment = detect_environment(temp_proj) + environment = Environment.create_python_environment(temp_proj) # mock the result of running of `quarto inspect ` inspect = { @@ -743,7 +738,7 @@ def test_make_source_manifest(self): # include environment parameter manifest = make_source_manifest( AppModes.PYTHON_API, - Environment( + Environment.from_dict(dict( contents="", error=None, filename="requirements.txt", @@ -752,7 +747,7 @@ def test_make_source_manifest(self): pip="22.0.4", python="3.9.12", source="file", - ), + )), None, None, None, @@ -922,7 +917,7 @@ def test_make_quarto_manifest_project_with_env(self): "config": {"project": {"title": "quarto-proj-py"}, "editor": "visual", "language": {}}, }, AppModes.SHINY_QUARTO, - Environment( + Environment.from_dict(dict( contents="", error=None, filename="requirements.txt", @@ -931,7 +926,7 @@ def test_make_quarto_manifest_project_with_env(self): pip="22.0.4", python="3.9.12", source="file", - ), + )), [], [], None, @@ -1207,132 +1202,6 @@ def test_default_title_from_manifest(self): m = {"metadata": {"entrypoint": "module:object"}} self.assertEqual(_default_title_from_manifest(m, "dir/to/manifest.json"), "0to") - def test_inspect_environment(self): - environment = inspect_environment(sys.executable, get_dir("pip1")) - assert environment is not None - assert environment.python != "" - - def test_inspect_environment_catches_type_error(self): - with pytest.raises(RSConnectException) as exec_info: - inspect_environment(sys.executable, None) # type: ignore - - assert isinstance(exec_info.value, RSConnectException) - assert isinstance(exec_info.value.__cause__, TypeError) - - -@pytest.mark.parametrize( - ( - "file_name", - "python", - "force_generate", - "expected_python", - "expected_environment", - ), - [ - pytest.param( - "path/to/file.py", - sys.executable, - False, - sys.executable, - MakeEnvironment( - contents=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip=None, - python=None, - source="pip_freeze", - error=None, - ), - id="basic", - ), - pytest.param( - "another/file.py", - basename(sys.executable), - False, - sys.executable, - MakeEnvironment( - contents=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip=None, - python=None, - source="pip_freeze", - error=None, - ), - id="which_python", - ), - pytest.param( - "will/the/files/never/stop.py", - "argh.py", - False, - "unused", - MakeEnvironment(None, None, None, None, None, None, None, error="Could not even do things"), - id="exploding", - ), - ], -) -def test_get_python_env_info( - monkeypatch, - file_name, - python, - force_generate, - expected_python, - expected_environment, -): - def fake_which_python(python, env=os.environ): - return expected_python - - def fake_inspect_environment( - python, - directory, - force_generate=False, - check_output=subprocess.check_output, - ): - return expected_environment - - monkeypatch.setattr(rsconnect.bundle, "inspect_environment", fake_inspect_environment) - - monkeypatch.setattr(rsconnect.bundle, "which_python", fake_which_python) - - if expected_environment.error is not None: - with pytest.raises(RSConnectException): - _, _ = _get_python_env_info(file_name, python, force_generate=force_generate) - else: - python, environment = _get_python_env_info(file_name, python, force_generate=force_generate) - - assert python == expected_python - assert environment == expected_environment - - -class WhichPythonTestCase(TestCase): - def test_default(self): - self.assertEqual(which_python(), sys.executable) - - def test_none(self): - self.assertEqual(which_python(None), sys.executable) - - def test_sys(self): - self.assertEqual(which_python(sys.executable), sys.executable) - - def test_does_not_exist(self): - with tempfile.NamedTemporaryFile() as tmpfile: - name = tmpfile.name - with self.assertRaises(RSConnectException): - which_python(name) - - def test_is_directory(self): - with tempfile.TemporaryDirectory() as tmpdir: - with self.assertRaises(RSConnectException): - which_python(tmpdir) - - @pytest.mark.skipif(sys.platform.startswith("win"), reason="os.X_OK always returns True") - def test_is_not_executable(self): - with tempfile.NamedTemporaryFile() as tmpfile: - with self.assertRaises(RSConnectException): - which_python(tmpfile.name) - cur_dir = os.path.dirname(__file__) bqplot_dir = os.path.join(cur_dir, "testdata", "voila", "bqplot", "") @@ -1396,7 +1265,7 @@ def test_guess_deploy_dir(self): ], ) def test_create_voila_manifest_1(path, entrypoint): - environment = Environment( + environment = Environment.from_dict(dict( contents="bqplot\n", error=None, filename="requirements.txt", @@ -1405,7 +1274,7 @@ def test_create_voila_manifest_1(path, entrypoint): pip="23.0", python="3.8.12", source="file", - ) + )) if sys.platform == "win32": checksum_hash = "b7ba4ec7b6721c86ab883f5e6e2ea68f" @@ -1469,7 +1338,7 @@ def test_create_voila_manifest_1(path, entrypoint): ], ) def test_create_voila_manifest_2(path, entrypoint): - environment = Environment( + environment = Environment.from_dict(dict( contents="numpy\nipywidgets\nbqplot\n", error=None, filename="requirements.txt", @@ -1478,7 +1347,7 @@ def test_create_voila_manifest_2(path, entrypoint): pip="23.0", python="3.8.12", source="file", - ) + )) if sys.platform == "win32": bqplot_hash = "b7ba4ec7b6721c86ab883f5e6e2ea68f" @@ -1515,7 +1384,7 @@ def test_create_voila_manifest_2(path, entrypoint): def test_create_voila_manifest_extra(): - environment = Environment( + environment = Environment.from_dict(dict( contents="numpy\nipywidgets\nbqplot\n", error=None, filename="requirements.txt", @@ -1524,7 +1393,7 @@ def test_create_voila_manifest_extra(): pip="23.0.1", python="3.8.12", source="file", - ) + )) if sys.platform == "win32": requirements_checksum = "d51994456975ff487749acc247ae6d63" @@ -1599,7 +1468,7 @@ def test_create_voila_manifest_extra(): ], ) def test_create_voila_manifest_multi_notebook(path, entrypoint): - environment = Environment( + environment = Environment.from_dict(dict( contents="bqplot\n", error=None, filename="requirements.txt", @@ -1608,7 +1477,7 @@ def test_create_voila_manifest_multi_notebook(path, entrypoint): pip="23.0", python="3.8.12", source="file", - ) + )) if sys.platform == "win32": bqplot_hash = "ddb4070466d3c45b2f233dd39906ddf6" @@ -1704,7 +1573,7 @@ def test_make_voila_bundle( path, entrypoint, ): - environment = Environment( + environment = Environment.from_dict(dict( contents="bqplot", error=None, filename="requirements.txt", @@ -1713,7 +1582,7 @@ def test_make_voila_bundle( pip="23.0", python="3.8.12", source="file", - ) + )) if sys.platform == "win32": checksum_hash = "b7ba4ec7b6721c86ab883f5e6e2ea68f" @@ -1811,7 +1680,7 @@ def test_make_voila_bundle_multi_notebook( path, entrypoint, ): - environment = Environment( + environment = Environment.from_dict(dict( contents="bqplot", error=None, filename="requirements.txt", @@ -1820,7 +1689,7 @@ def test_make_voila_bundle_multi_notebook( pip="23.0", python="3.8.12", source="file", - ) + )) if sys.platform == "win32": bqplot_hash = "ddb4070466d3c45b2f233dd39906ddf6" @@ -1900,7 +1769,7 @@ def test_make_voila_bundle_2( path, entrypoint, ): - environment = Environment( + environment = Environment.from_dict(dict( contents="numpy\nipywidgets\nbqplot\n", error=None, filename="requirements.txt", @@ -1909,7 +1778,7 @@ def test_make_voila_bundle_2( pip="23.0", python="3.8.12", source="file", - ) + )) if sys.platform == "win32": bqplot_hash = "b7ba4ec7b6721c86ab883f5e6e2ea68f" @@ -1964,7 +1833,7 @@ def test_make_voila_bundle_extra(): requirements_hash = "d51994456975ff487749acc247ae6d63" - environment = Environment( + environment = Environment.from_dict(dict( contents="numpy\nipywidgets\nbqplot\n", error=None, filename="requirements.txt", @@ -1973,7 +1842,7 @@ def test_make_voila_bundle_extra(): pip="23.0.1", python="3.8.12", source="file", - ) + )) ans = { "version": 1, "locale": "en_US.UTF-8", @@ -2437,7 +2306,7 @@ def test_make_api_manifest_fastapi(): "prices.csv": {"checksum": "012afa636c426748177b38160135307a"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( fastapi_dir, ) manifest, _ = make_api_manifest( @@ -2469,7 +2338,7 @@ def test_make_api_bundle_fastapi(): "prices.csv": {"checksum": "012afa636c426748177b38160135307a"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( fastapi_dir, ) with make_api_bundle( @@ -2513,7 +2382,7 @@ def test_make_api_manifest_flask(): "prices.csv": {"checksum": "012afa636c426748177b38160135307a"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( flask_dir, ) manifest, _ = make_api_manifest( @@ -2545,7 +2414,7 @@ def test_make_api_bundle_flask(): "prices.csv": {"checksum": "012afa636c426748177b38160135307a"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( flask_dir, ) with make_api_bundle( @@ -2589,7 +2458,7 @@ def test_make_api_manifest_streamlit(): "data.csv": {"checksum": "aabd9d1210246c69403532a6a9d24286"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( streamlit_dir, ) manifest, _ = make_api_manifest( @@ -2620,7 +2489,7 @@ def test_make_api_bundle_streamlit(): "data.csv": {"checksum": "aabd9d1210246c69403532a6a9d24286"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( streamlit_dir, ) with make_api_bundle( @@ -2665,7 +2534,7 @@ def test_make_api_manifest_dash(): "prices.csv": {"checksum": "3efb0ed7ad93bede9dc88f7a81ad4153"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( dash_dir, ) manifest, _ = make_api_manifest( @@ -2697,7 +2566,7 @@ def test_make_api_bundle_dash(): "prices.csv": {"checksum": "3efb0ed7ad93bede9dc88f7a81ad4153"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( dash_dir, ) with make_api_bundle( @@ -2741,7 +2610,7 @@ def test_make_api_manifest_bokeh(): "data.csv": {"checksum": "aabd9d1210246c69403532a6a9d24286"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( bokeh_dir, ) manifest, _ = make_api_manifest( @@ -2774,7 +2643,7 @@ def test_make_api_bundle_bokeh(): }, } - environment = create_python_environment( + environment = Environment.create_python_environment( bokeh_dir, ) with make_api_bundle( @@ -2818,7 +2687,7 @@ def test_make_api_manifest_shiny(): "data.csv": {"checksum": "aabd9d1210246c69403532a6a9d24286"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( shiny_dir, ) manifest, _ = make_api_manifest( @@ -2850,7 +2719,7 @@ def test_make_api_bundle_shiny(): "data.csv": {"checksum": "aabd9d1210246c69403532a6a9d24286"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( shiny_dir, ) with make_api_bundle( @@ -2928,7 +2797,7 @@ def test_make_api_manifest_gradio(): "app.py": {"checksum": "22feec76e9c02ac6b5a34a083e2983b6"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( gradio_dir, ) manifest, _ = make_api_manifest( @@ -2958,7 +2827,7 @@ def test_make_api_bundle_gradio(): "app.py": {"checksum": "22feec76e9c02ac6b5a34a083e2983b6"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( gradio_dir, ) with make_api_bundle( diff --git a/tests/test_environment.py b/tests/test_environment.py index c168153e..f64a37f3 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,17 +1,15 @@ import re import sys +import os +import tempfile from unittest import TestCase -from rsconnect.environment import ( - MakeEnvironment, - detect_environment, - filter_pip_freeze_output, - get_default_locale, - get_python_version, -) +from rsconnect.environment import Environment from .utils import get_dir +import pytest + version_re = re.compile(r"\d+\.\d+(\.\d+)?") @@ -94,3 +92,131 @@ def test_filter_pip_freeze_output(self): expected = "numpy\npandas\nnot at beginning [notice]" self.assertEqual(filtered, expected) + + +class WhichPythonTestCase(TestCase): + def test_default(self): + self.assertEqual(which_python(), sys.executable) + + def test_none(self): + self.assertEqual(which_python(None), sys.executable) + + def test_sys(self): + self.assertEqual(which_python(sys.executable), sys.executable) + + def test_does_not_exist(self): + with tempfile.NamedTemporaryFile() as tmpfile: + name = tmpfile.name + with self.assertRaises(RSConnectException): + which_python(name) + + def test_is_directory(self): + with tempfile.TemporaryDirectory() as tmpdir: + with self.assertRaises(RSConnectException): + which_python(tmpdir) + + @pytest.mark.skipif(sys.platform.startswith("win"), reason="os.X_OK always returns True") + def test_is_not_executable(self): + with tempfile.NamedTemporaryFile() as tmpfile: + with self.assertRaises(RSConnectException): + which_python(tmpfile.name) + + +def test_inspect_environment(): + environment = inspect_environment(sys.executable, get_dir("pip1")) + assert environment is not None + assert environment.python != "" + + +def test_inspect_environment_catches_type_error(): + with pytest.raises(RSConnectException) as exec_info: + inspect_environment(sys.executable, None) # type: ignore + + assert isinstance(exec_info.value, RSConnectException) + assert isinstance(exec_info.value.__cause__, TypeError) + + +@pytest.mark.parametrize( + ( + "file_name", + "python", + "force_generate", + "expected_python", + "expected_environment", + ), + [ + pytest.param( + "path/to/file.py", + sys.executable, + False, + sys.executable, + Environment.from_dict(dict( + contents=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip=None, + python=None, + source="pip_freeze", + error=None, + )), + id="basic", + ), + pytest.param( + "another/file.py", + os.path.basename(sys.executable), + False, + sys.executable, + Environment.from_dict(dict( + contents=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip=None, + python=None, + source="pip_freeze", + error=None, + )), + id="which_python", + ), + pytest.param( + "will/the/files/never/stop.py", + "argh.py", + False, + "unused", + Environment.from_dict(dict(contents=None, filename=None, locale=None, package_manager=None, pip=None, python=None, source=None, error="Could not even do things")), + id="exploding", + ), + ], +) +def test_get_python_env_info( + monkeypatch, + file_name, + python, + force_generate, + expected_python, + expected_environment, +): + def fake_which_python(python, env=os.environ): + return expected_python + + def fake_inspect_environment( + python, + directory, + force_generate=False, + check_output=subprocess.check_output, + ): + return expected_environment + + monkeypatch.setattr(rsconnect.bundle, "inspect_environment", fake_inspect_environment) + + monkeypatch.setattr(rsconnect.bundle, "which_python", fake_which_python) + + if expected_environment.error is not None: + with pytest.raises(RSConnectException): + _, _ = _get_python_env_info(file_name, python, force_generate=force_generate) + else: + python, environment = _get_python_env_info(file_name, python, force_generate=force_generate) + + assert python == expected_python + assert environment == expected_environment From 30cfa2c86045eb8e9d8a2c3c249d963a20787b49 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 24 Mar 2025 12:12:57 +0100 Subject: [PATCH 12/24] update test_environment --- rsconnect/environment.py | 10 ++++++++++ tests/test_environment.py | 36 ++++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 0639aa9c..65c2f709 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -60,6 +60,16 @@ def __setattr__(self, name: str, value: typing.Any) -> None: else: super().__setattr__(name, value) + def __eq__(self, other: typing.Any) -> bool: + if not isinstance(other, Environment): + return False + + return ( + self._data == other._data + and self.python_interpreter == other.python_interpreter + and self.python_version_requirement == other.python_version_requirement + ) + @classmethod def from_dict( cls, diff --git a/tests/test_environment.py b/tests/test_environment.py index f64a37f3..6df56dfb 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -2,9 +2,13 @@ import sys import os import tempfile +import subprocess from unittest import TestCase -from rsconnect.environment import Environment +import rsconnect.environment +from rsconnect.exception import RSConnectException +from rsconnect.environment import Environment, which_python +from rsconnect.subprocesses.environment import get_python_version, get_default_locale, filter_pip_freeze_output from .utils import get_dir @@ -31,14 +35,14 @@ def test_get_default_locale(self): self.assertEqual(get_default_locale(lambda: (None, None)), "") def test_file(self): - result = detect_environment(get_dir("pip1")) + result = Environment.create_python_environment(get_dir("pip1")) self.assertTrue(version_re.match(result.pip)) self.assertIsInstance(result.locale, str) self.assertIn(".", result.locale) - expected = MakeEnvironment( + expected = Environment.from_dict(dict( contents="numpy\npandas\nmatplotlib\n", filename="requirements.txt", locale=result.locale, @@ -46,11 +50,11 @@ def test_file(self): pip=result.pip, python=self.python_version(), source="file", - ) + ), python_interpreter=sys.executable) self.assertEqual(expected, result) def test_pip_freeze(self): - result = detect_environment(get_dir("pip2")) + result = Environment.create_python_environment(get_dir("pip2")) # these are the dependencies declared in our pyproject.toml self.assertIn("six", result.contents) @@ -61,7 +65,7 @@ def test_pip_freeze(self): self.assertIsInstance(result.locale, str) self.assertIn(".", result.locale) - expected = MakeEnvironment( + expected = Environment.from_dict(dict( contents=result.contents, filename="requirements.txt", locale=result.locale, @@ -69,7 +73,7 @@ def test_pip_freeze(self): pip=result.pip, python=self.python_version(), source="pip_freeze", - ) + ), python_interpreter=sys.executable) self.assertEqual(expected, result) def test_filter_pip_freeze_output(self): @@ -123,14 +127,14 @@ def test_is_not_executable(self): def test_inspect_environment(): - environment = inspect_environment(sys.executable, get_dir("pip1")) + environment = Environment._inspect_environment(sys.executable, get_dir("pip1")) assert environment is not None assert environment.python != "" def test_inspect_environment_catches_type_error(): with pytest.raises(RSConnectException) as exec_info: - inspect_environment(sys.executable, None) # type: ignore + Environment._inspect_environment(sys.executable, None) # type: ignore assert isinstance(exec_info.value, RSConnectException) assert isinstance(exec_info.value.__cause__, TypeError) @@ -159,7 +163,7 @@ def test_inspect_environment_catches_type_error(): python=None, source="pip_freeze", error=None, - )), + ), python_interpreter=sys.executable), id="basic", ), pytest.param( @@ -176,7 +180,7 @@ def test_inspect_environment_catches_type_error(): python=None, source="pip_freeze", error=None, - )), + ), python_interpreter=sys.executable), id="which_python", ), pytest.param( @@ -208,15 +212,15 @@ def fake_inspect_environment( ): return expected_environment - monkeypatch.setattr(rsconnect.bundle, "inspect_environment", fake_inspect_environment) + monkeypatch.setattr(Environment, "_inspect_environment", fake_inspect_environment) - monkeypatch.setattr(rsconnect.bundle, "which_python", fake_which_python) + monkeypatch.setattr(rsconnect.environment, "which_python", fake_which_python) if expected_environment.error is not None: with pytest.raises(RSConnectException): - _, _ = _get_python_env_info(file_name, python, force_generate=force_generate) + _ = Environment._get_python_env_info(file_name, python, force_generate=force_generate) else: - python, environment = _get_python_env_info(file_name, python, force_generate=force_generate) + environment = Environment._get_python_env_info(file_name, python, force_generate=force_generate) - assert python == expected_python + assert environment.python_interpreter == expected_python assert environment == expected_environment From 29934572bdafe99ae93c16170f5cc090c2996f43 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 24 Mar 2025 12:52:38 +0100 Subject: [PATCH 13/24] lint and format --- tests/test_bundle.py | 221 +++++++++++++++++++++----------------- tests/test_environment.py | 101 ++++++++++------- 2 files changed, 182 insertions(+), 140 deletions(-) diff --git a/tests/test_bundle.py b/tests/test_bundle.py index b4629ef8..e300954e 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import json import os -import subprocess import sys import tarfile import tempfile @@ -738,16 +737,18 @@ def test_make_source_manifest(self): # include environment parameter manifest = make_source_manifest( AppModes.PYTHON_API, - Environment.from_dict(dict( - contents="", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="22.0.4", - python="3.9.12", - source="file", - )), + Environment.from_dict( + dict( + contents="", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="22.0.4", + python="3.9.12", + source="file", + ) + ), None, None, None, @@ -917,16 +918,18 @@ def test_make_quarto_manifest_project_with_env(self): "config": {"project": {"title": "quarto-proj-py"}, "editor": "visual", "language": {}}, }, AppModes.SHINY_QUARTO, - Environment.from_dict(dict( - contents="", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="22.0.4", - python="3.9.12", - source="file", - )), + Environment.from_dict( + dict( + contents="", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="22.0.4", + python="3.9.12", + source="file", + ) + ), [], [], None, @@ -1265,16 +1268,18 @@ def test_guess_deploy_dir(self): ], ) def test_create_voila_manifest_1(path, entrypoint): - environment = Environment.from_dict(dict( - contents="bqplot\n", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0", - python="3.8.12", - source="file", - )) + environment = Environment.from_dict( + dict( + contents="bqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) + ) if sys.platform == "win32": checksum_hash = "b7ba4ec7b6721c86ab883f5e6e2ea68f" @@ -1338,16 +1343,18 @@ def test_create_voila_manifest_1(path, entrypoint): ], ) def test_create_voila_manifest_2(path, entrypoint): - environment = Environment.from_dict(dict( - contents="numpy\nipywidgets\nbqplot\n", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0", - python="3.8.12", - source="file", - )) + environment = Environment.from_dict( + dict( + contents="numpy\nipywidgets\nbqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) + ) if sys.platform == "win32": bqplot_hash = "b7ba4ec7b6721c86ab883f5e6e2ea68f" @@ -1384,16 +1391,18 @@ def test_create_voila_manifest_2(path, entrypoint): def test_create_voila_manifest_extra(): - environment = Environment.from_dict(dict( - contents="numpy\nipywidgets\nbqplot\n", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0.1", - python="3.8.12", - source="file", - )) + environment = Environment.from_dict( + dict( + contents="numpy\nipywidgets\nbqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0.1", + python="3.8.12", + source="file", + ) + ) if sys.platform == "win32": requirements_checksum = "d51994456975ff487749acc247ae6d63" @@ -1468,16 +1477,18 @@ def test_create_voila_manifest_extra(): ], ) def test_create_voila_manifest_multi_notebook(path, entrypoint): - environment = Environment.from_dict(dict( - contents="bqplot\n", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0", - python="3.8.12", - source="file", - )) + environment = Environment.from_dict( + dict( + contents="bqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) + ) if sys.platform == "win32": bqplot_hash = "ddb4070466d3c45b2f233dd39906ddf6" @@ -1573,16 +1584,18 @@ def test_make_voila_bundle( path, entrypoint, ): - environment = Environment.from_dict(dict( - contents="bqplot", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0", - python="3.8.12", - source="file", - )) + environment = Environment.from_dict( + dict( + contents="bqplot", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) + ) if sys.platform == "win32": checksum_hash = "b7ba4ec7b6721c86ab883f5e6e2ea68f" @@ -1680,16 +1693,18 @@ def test_make_voila_bundle_multi_notebook( path, entrypoint, ): - environment = Environment.from_dict(dict( - contents="bqplot", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0", - python="3.8.12", - source="file", - )) + environment = Environment.from_dict( + dict( + contents="bqplot", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) + ) if sys.platform == "win32": bqplot_hash = "ddb4070466d3c45b2f233dd39906ddf6" @@ -1769,16 +1784,18 @@ def test_make_voila_bundle_2( path, entrypoint, ): - environment = Environment.from_dict(dict( - contents="numpy\nipywidgets\nbqplot\n", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0", - python="3.8.12", - source="file", - )) + environment = Environment.from_dict( + dict( + contents="numpy\nipywidgets\nbqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) + ) if sys.platform == "win32": bqplot_hash = "b7ba4ec7b6721c86ab883f5e6e2ea68f" @@ -1833,16 +1850,18 @@ def test_make_voila_bundle_extra(): requirements_hash = "d51994456975ff487749acc247ae6d63" - environment = Environment.from_dict(dict( - contents="numpy\nipywidgets\nbqplot\n", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0.1", - python="3.8.12", - source="file", - )) + environment = Environment.from_dict( + dict( + contents="numpy\nipywidgets\nbqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0.1", + python="3.8.12", + source="file", + ) + ) ans = { "version": 1, "locale": "en_US.UTF-8", diff --git a/tests/test_environment.py b/tests/test_environment.py index 6df56dfb..87935a71 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -42,15 +42,18 @@ def test_file(self): self.assertIsInstance(result.locale, str) self.assertIn(".", result.locale) - expected = Environment.from_dict(dict( - contents="numpy\npandas\nmatplotlib\n", - filename="requirements.txt", - locale=result.locale, - package_manager="pip", - pip=result.pip, - python=self.python_version(), - source="file", - ), python_interpreter=sys.executable) + expected = Environment.from_dict( + dict( + contents="numpy\npandas\nmatplotlib\n", + filename="requirements.txt", + locale=result.locale, + package_manager="pip", + pip=result.pip, + python=self.python_version(), + source="file", + ), + python_interpreter=sys.executable, + ) self.assertEqual(expected, result) def test_pip_freeze(self): @@ -65,15 +68,18 @@ def test_pip_freeze(self): self.assertIsInstance(result.locale, str) self.assertIn(".", result.locale) - expected = Environment.from_dict(dict( - contents=result.contents, - filename="requirements.txt", - locale=result.locale, - package_manager="pip", - pip=result.pip, - python=self.python_version(), - source="pip_freeze", - ), python_interpreter=sys.executable) + expected = Environment.from_dict( + dict( + contents=result.contents, + filename="requirements.txt", + locale=result.locale, + package_manager="pip", + pip=result.pip, + python=self.python_version(), + source="pip_freeze", + ), + python_interpreter=sys.executable, + ) self.assertEqual(expected, result) def test_filter_pip_freeze_output(self): @@ -154,16 +160,19 @@ def test_inspect_environment_catches_type_error(): sys.executable, False, sys.executable, - Environment.from_dict(dict( - contents=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip=None, - python=None, - source="pip_freeze", - error=None, - ), python_interpreter=sys.executable), + Environment.from_dict( + dict( + contents=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip=None, + python=None, + source="pip_freeze", + error=None, + ), + python_interpreter=sys.executable, + ), id="basic", ), pytest.param( @@ -171,16 +180,19 @@ def test_inspect_environment_catches_type_error(): os.path.basename(sys.executable), False, sys.executable, - Environment.from_dict(dict( - contents=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip=None, - python=None, - source="pip_freeze", - error=None, - ), python_interpreter=sys.executable), + Environment.from_dict( + dict( + contents=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip=None, + python=None, + source="pip_freeze", + error=None, + ), + python_interpreter=sys.executable, + ), id="which_python", ), pytest.param( @@ -188,7 +200,18 @@ def test_inspect_environment_catches_type_error(): "argh.py", False, "unused", - Environment.from_dict(dict(contents=None, filename=None, locale=None, package_manager=None, pip=None, python=None, source=None, error="Could not even do things")), + Environment.from_dict( + dict( + contents=None, + filename=None, + locale=None, + package_manager=None, + pip=None, + python=None, + source=None, + error="Could not even do things", + ) + ), id="exploding", ), ], From eb32b99ec3d6caa6b7274ba1ef4a78934e4081be Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 24 Mar 2025 13:07:26 +0100 Subject: [PATCH 14/24] Update README --- README.md | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 19dcdcc5..767310f1 100644 --- a/README.md +++ b/README.md @@ -219,24 +219,34 @@ ensuring that you use the same Python that you use to run your Jupyter Notebook: #### Python Version When deploying Python content to Posit Connect, -the server will require matching `` versions of Python. For example, -a server with only Python 3.9 installed will fail to match content deployed with -Python 3.8. Your administrator may also enable exact Python version matching which -will be stricter and require matching major, minor, and patch versions. For more -information see the [Posit Connect Admin Guide chapter titled Python Version +the server will require a version of Python that matches the content +requirements. + +For example, a server with only Python 3.9 installed will fail to match content +that requires Python 3.8. + +`rsconnect` will supports detecting Python version requirements in 4 ways: + 1. A `.python-version` file exists. In such case + `rsconnect` will use its content to determine the python version requirement. + 2. A `pyproject.toml` with a `project.requires-python` field exists. + In such case the requirement specified in the field will be used + if no `.python-version` file exists. + 3. A `setup.cfg` with a `options.python_requirs` field exists. + In such case the requirement specified in the field will be used + if no `.python-version` or `pyproject.toml` files exist. + 4. If no other source of version requirement was found, then + the interpreter in use is considered the one required to run the content. + +On newer Posit Connect versions the requirement detected by `rsconnect` is +always respected. Older Connect versions will instead rely only on the +python version used to deploy the content to determine the requirement. + +For more information see the [Posit Connect Admin Guide chapter titled Python Version Matching](https://docs.posit.co/connect/admin/python/#python-version-matching). -We recommend installing a version of Python on your client that is also available -in your Connect installation. If that's not possible, you can override -rsconnect-python's detected Python version and request a version of Python -that is installed in Connect, For example, this command: - -```bash -rsconnect deploy api --override-python-version 3.11.5 my-api/ -``` - -will deploy the content in `my-api` while requesting that Connect -use Python version 3.11.5. +We recommend providing a `pyproject.toml` with a `project.requires-python` field +if the deployed content is an installable package and a `.python-version` file +for plain directories. > **Note** > The packages and package versions listed in `requirements.txt` must be From 771d9eb4ba9194467cd79d2454f7ae02e79cedc6 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 24 Mar 2025 13:09:51 +0100 Subject: [PATCH 15/24] Fix type hinting --- rsconnect/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 65c2f709..03e3c6cc 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -73,7 +73,7 @@ def __eq__(self, other: typing.Any) -> bool: @classmethod def from_dict( cls, - data: dict[str, typing.Any], + data: typing.Dict[str, typing.Any], python_interpreter: typing.Optional[str] = None, python_version_requirement: typing.Optional[str] = None, ) -> "Environment": From e6d69395e3f833ddde565c3f6693ed2961cc13d7 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 24 Mar 2025 13:11:27 +0100 Subject: [PATCH 16/24] Update CHANGELOG --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 532abdd9..4830b427 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [??] - ?? + +### Added + +- `rsconnect` now detects Python interpreter version requirements from + `.python-version`, `pyproject.toml` and `setup.cfg` + ## [1.25.2] - 2025-02-26 ### Fixed From 7aebdcd6e6d2363667f1b044af165a085f4281b0 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 24 Mar 2025 13:13:28 +0100 Subject: [PATCH 17/24] More type hinting fixes --- rsconnect/environment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 03e3c6cc..8e842cd2 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -176,7 +176,7 @@ def _inspect_environment( Returns a dictionary of information about the environment, or containing an "error" field if an error occurred. """ - flags: list[str] = [] + flags: typing.List[str] = [] if force_generate: flags.append("f") @@ -249,9 +249,9 @@ def is_environment_dir(directory: typing.Union[str, pathlib.Path]) -> bool: return os.path.exists(python_path) or os.path.exists(win_path) -def list_environment_dirs(directory: typing.Union[str, pathlib.Path]) -> list[str]: +def list_environment_dirs(directory: typing.Union[str, pathlib.Path]) -> typing.List[str]: """Returns a list of subdirectories in `directory` that appear to contain virtual environments.""" - envs: list[str] = [] + envs: typing.List[str] = [] for name in os.listdir(directory): path = os.path.join(directory, name) From b28e0d3fa83d46096544d6c204965b681caa4edd Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 24 Mar 2025 13:28:41 +0100 Subject: [PATCH 18/24] Add integration test for Environment --- rsconnect/environment.py | 5 +++++ tests/test_environment.py | 23 +++++++++++++++++++++++ tests/test_pyproject.py | 11 ++++++++--- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 8e842cd2..aec5c119 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -70,6 +70,11 @@ def __eq__(self, other: typing.Any) -> bool: and self.python_version_requirement == other.python_version_requirement ) + def __repr__(self) -> str: + data = self._data._asdict() + data.pop("contents", None) # Remove contents as it's too long to display + return f"Environment({data}, python_interpreter={self.python_interpreter}, python_version_requirement={self.python_version_requirement})" + @classmethod def from_dict( cls, diff --git a/tests/test_environment.py b/tests/test_environment.py index 87935a71..b52f5716 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -15,6 +15,7 @@ import pytest version_re = re.compile(r"\d+\.\d+(\.\d+)?") +TESTDATA = os.path.join(os.path.dirname(__file__), "testdata") class TestEnvironment(TestCase): @@ -132,6 +133,28 @@ def test_is_not_executable(self): which_python(tmpfile.name) +class TestPythonVersionRequirements: + def test_pyproject_toml(self): + env = Environment.create_python_environment(os.path.join(TESTDATA, "python-project", "using_pyproject")) + assert env.python_interpreter == sys.executable + assert env.python_version_requirement == ">=3.8" + + def test_python_version(self): + env = Environment.create_python_environment(os.path.join(TESTDATA, "python-project", "using_pyversion")) + assert env.python_interpreter == sys.executable + assert env.python_version_requirement == ">=3.8, <3.12" + + def test_all_of_them(self): + env = Environment.create_python_environment(os.path.join(TESTDATA, "python-project", "allofthem")) + assert env.python_interpreter == sys.executable + assert env.python_version_requirement == ">=3.8, <3.12" + + def test_missing(self): + env = Environment.create_python_environment(os.path.join(TESTDATA, "python-project", "empty")) + assert env.python_interpreter == sys.executable + assert env.python_version_requirement is None + + def test_inspect_environment(): environment = Environment._inspect_environment(sys.executable, get_dir("pip1")) assert environment is not None diff --git a/tests/test_pyproject.py b/tests/test_pyproject.py index a245988e..8fd3fe57 100644 --- a/tests/test_pyproject.py +++ b/tests/test_pyproject.py @@ -50,15 +50,20 @@ def test_python_project_metadata_detect(project_dir, expected): ("pyproject.toml", parse_pyproject_python_requires), ("setup.cfg", parse_setupcfg_python_requires), (".python-version", parse_pyversion_python_requires), - ("invalid.txt", None), + ("invalid.txt", NotImplementedError("Unknown metadata file type: invalid.txt")), ], ids=["pyproject.toml", "setup.cfg", ".python-version", "invalid"], ) def test_get_python_version_requirement_parser(filename, expected_parser): """Test that given a metadata file name, the correct parser is returned.""" metadata_file = pathlib.Path(PROJECTS_DIRECTORY) / filename - parser = get_python_version_requirement_parser(metadata_file) - assert parser == expected_parser + if isinstance(expected_parser, Exception): + with pytest.raises(expected_parser.__class__) as excinfo: + parser = get_python_version_requirement_parser(metadata_file) + assert str(excinfo.value) == expected_parser.args[0] + else: + parser = get_python_version_requirement_parser(metadata_file) + assert parser == expected_parser @pytest.mark.parametrize( From 68ff27d0b5f0d7427653115d419705dd190e25a7 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 24 Mar 2025 13:32:29 +0100 Subject: [PATCH 19/24] Reformat __repr__ --- rsconnect/environment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index aec5c119..361404e4 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -73,7 +73,9 @@ def __eq__(self, other: typing.Any) -> bool: def __repr__(self) -> str: data = self._data._asdict() data.pop("contents", None) # Remove contents as it's too long to display - return f"Environment({data}, python_interpreter={self.python_interpreter}, python_version_requirement={self.python_version_requirement})" + return (f"Environment({data}, " + f"python_interpreter={self.python_interpreter}, " + f"python_version_requirement={self.python_version_requirement})") @classmethod def from_dict( From 0e7ebf2d415a13d0d7fae3d7014d8f8187fe1c25 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 24 Mar 2025 13:32:41 +0100 Subject: [PATCH 20/24] Reformat __repr__ --- rsconnect/environment.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 361404e4..6e105f1c 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -73,9 +73,11 @@ def __eq__(self, other: typing.Any) -> bool: def __repr__(self) -> str: data = self._data._asdict() data.pop("contents", None) # Remove contents as it's too long to display - return (f"Environment({data}, " - f"python_interpreter={self.python_interpreter}, " - f"python_version_requirement={self.python_version_requirement})") + return ( + f"Environment({data}, " + f"python_interpreter={self.python_interpreter}, " + f"python_version_requirement={self.python_version_requirement})" + ) @classmethod def from_dict( From 08ff61440f947c98cb0b032aa2b5090008f34b79 Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 24 Mar 2025 14:17:23 +0100 Subject: [PATCH 21/24] Rename subprocess.environment to subprocesses.inspect_environment --- rsconnect/environment.py | 6 +++--- .../subprocesses/{environment.py => inspect_environment.py} | 2 +- tests/test_environment.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename rsconnect/subprocesses/{environment.py => inspect_environment.py} (99%) diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 6e105f1c..ec0052bd 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -3,7 +3,7 @@ Given a directory and a Python executable, this module inspects the environment and returns information about the Python version and the environment itself. -To inspect the environment it relies on a subprocess that runs the `rsconnect.subprocesses.environment` +To inspect the environment it relies on a subprocess that runs the `rsconnect.subprocesses.inspect_environment` module. This module is responsible for gathering the environment information and returning it in a JSON format. """ @@ -19,7 +19,7 @@ from . import pyproject from .log import logger from .exception import RSConnectException -from .subprocesses.environment import EnvironmentData, MakeEnvironmentData as _MakeEnvironmentData +from .subprocesses.inspect_environment import EnvironmentData, MakeEnvironmentData as _MakeEnvironmentData import click @@ -189,7 +189,7 @@ def _inspect_environment( if force_generate: flags.append("f") - args = [python, "-m", "rsconnect.subprocesses.environment"] + args = [python, "-m", "rsconnect.subprocesses.inspect_environment"] if flags: args.append("-" + "".join(flags)) args.append(directory) diff --git a/rsconnect/subprocesses/environment.py b/rsconnect/subprocesses/inspect_environment.py similarity index 99% rename from rsconnect/subprocesses/environment.py rename to rsconnect/subprocesses/inspect_environment.py index 76d44d70..97b2c9a4 100644 --- a/rsconnect/subprocesses/environment.py +++ b/rsconnect/subprocesses/inspect_environment.py @@ -3,7 +3,7 @@ Environment data class abstraction that is usable as an executable module ```bash -python -m rsconnect.subprocesses.environment +python -m rsconnect.subprocesses.inspect_environment ``` """ from __future__ import annotations diff --git a/tests/test_environment.py b/tests/test_environment.py index b52f5716..52df3c35 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -8,7 +8,7 @@ import rsconnect.environment from rsconnect.exception import RSConnectException from rsconnect.environment import Environment, which_python -from rsconnect.subprocesses.environment import get_python_version, get_default_locale, filter_pip_freeze_output +from rsconnect.subprocesses.inspect_environment import get_python_version, get_default_locale, filter_pip_freeze_output from .utils import get_dir From 96d15633cdf481a5ef780cf53f91390f2422130b Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Mon, 24 Mar 2025 14:33:47 +0100 Subject: [PATCH 22/24] Make detect_python_version_requirement test --- tests/test_pyproject.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/test_pyproject.py b/tests/test_pyproject.py index 8fd3fe57..eb5b3f28 100644 --- a/tests/test_pyproject.py +++ b/tests/test_pyproject.py @@ -7,6 +7,7 @@ parse_setupcfg_python_requires, parse_pyversion_python_requires, get_python_version_requirement_parser, + detect_python_version_requirement, ) import pytest @@ -104,7 +105,7 @@ def test_pyprojecttoml_python_requires(project_dir, expected): ], ids=["option-exists", "option-missing"], ) -def test_setupcfg_python_requires(tmp_path, project_dir, expected): +def test_setupcfg_python_requires(project_dir, expected): """Test that the python_requires field is correctly parsed from setup.cfg. Both when the option exists or when it missing in the file. @@ -120,7 +121,7 @@ def test_setupcfg_python_requires(tmp_path, project_dir, expected): ], ids=["option-exists"], ) -def test_pyversion_python_requires(tmp_path, project_dir, expected): +def test_pyversion_python_requires(project_dir, expected): """Test that the python version is correctly parsed from .python-version. We do not test the case where the option is missing, as an empty .python-version file @@ -128,3 +129,16 @@ def test_pyversion_python_requires(tmp_path, project_dir, expected): """ versionfile = pathlib.Path(project_dir) / ".python-version" assert parse_pyversion_python_requires(versionfile) == expected + + +def test_detect_python_version_requirement(): + """Test that the python version requirement is correctly detected from the metadata files. + + Given that we already know from the other tests that the metadata files are correctly parsed, + this test primarily checks that when there are multiple metadata files, the one with the most specific + version requirement is used. + """ + project_dir = os.path.join(PROJECTS_DIRECTORY, "allofthem") + assert detect_python_version_requirement(project_dir) == ">=3.8, <3.12" + + assert detect_python_version_requirement(os.path.join(PROJECTS_DIRECTORY, "empty")) is None From 334458665d778a41114fe9760d1e8d647a8013fd Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Wed, 26 Mar 2025 10:24:41 +0100 Subject: [PATCH 23/24] typo README.md Co-authored-by: Brian Smith --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 767310f1..692670d3 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ that requires Python 3.8. 2. A `pyproject.toml` with a `project.requires-python` field exists. In such case the requirement specified in the field will be used if no `.python-version` file exists. - 3. A `setup.cfg` with a `options.python_requirs` field exists. + 3. A `setup.cfg` with an `options.python_requires` field exists. In such case the requirement specified in the field will be used if no `.python-version` or `pyproject.toml` files exist. 4. If no other source of version requirement was found, then From cbab772b7ea12979157c21bb8e7040c79071b8bf Mon Sep 17 00:00:00 2001 From: Alessandro Molina Date: Wed, 26 Mar 2025 15:22:21 +0100 Subject: [PATCH 24/24] Update Python version detection section in README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 692670d3..24dc83c2 100644 --- a/README.md +++ b/README.md @@ -225,7 +225,7 @@ requirements. For example, a server with only Python 3.9 installed will fail to match content that requires Python 3.8. -`rsconnect` will supports detecting Python version requirements in 4 ways: +`rsconnect` supports detecting Python version requirements in several ways: 1. A `.python-version` file exists. In such case `rsconnect` will use its content to determine the python version requirement. 2. A `pyproject.toml` with a `project.requires-python` field exists. @@ -233,11 +233,11 @@ that requires Python 3.8. if no `.python-version` file exists. 3. A `setup.cfg` with an `options.python_requires` field exists. In such case the requirement specified in the field will be used - if no `.python-version` or `pyproject.toml` files exist. + if **1** or **2** were not already satisfied. 4. If no other source of version requirement was found, then the interpreter in use is considered the one required to run the content. -On newer Posit Connect versions the requirement detected by `rsconnect` is +On Posit Connect `>=2025.03.0` the requirement detected by `rsconnect` is always respected. Older Connect versions will instead rely only on the python version used to deploy the content to determine the requirement.