diff --git a/CHANGELOG.md b/CHANGELOG.md index 532abdd9..4830b427 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [??] - ?? + +### Added + +- `rsconnect` now detects Python interpreter version requirements from + `.python-version`, `pyproject.toml` and `setup.cfg` + ## [1.25.2] - 2025-02-26 ### Fixed diff --git a/README.md b/README.md index 19dcdcc5..24dc83c2 100644 --- a/README.md +++ b/README.md @@ -219,24 +219,34 @@ ensuring that you use the same Python that you use to run your Jupyter Notebook: #### Python Version When deploying Python content to Posit Connect, -the server will require matching `` versions of Python. For example, -a server with only Python 3.9 installed will fail to match content deployed with -Python 3.8. Your administrator may also enable exact Python version matching which -will be stricter and require matching major, minor, and patch versions. For more -information see the [Posit Connect Admin Guide chapter titled Python Version +the server will require a version of Python that matches the content +requirements. + +For example, a server with only Python 3.9 installed will fail to match content +that requires Python 3.8. + +`rsconnect` supports detecting Python version requirements in several ways: + 1. A `.python-version` file exists. In such case + `rsconnect` will use its content to determine the python version requirement. + 2. A `pyproject.toml` with a `project.requires-python` field exists. + In such case the requirement specified in the field will be used + if no `.python-version` file exists. + 3. A `setup.cfg` with an `options.python_requires` field exists. + In such case the requirement specified in the field will be used + if **1** or **2** were not already satisfied. + 4. If no other source of version requirement was found, then + the interpreter in use is considered the one required to run the content. + +On Posit Connect `>=2025.03.0` the requirement detected by `rsconnect` is +always respected. Older Connect versions will instead rely only on the +python version used to deploy the content to determine the requirement. + +For more information see the [Posit Connect Admin Guide chapter titled Python Version Matching](https://docs.posit.co/connect/admin/python/#python-version-matching). -We recommend installing a version of Python on your client that is also available -in your Connect installation. If that's not possible, you can override -rsconnect-python's detected Python version and request a version of Python -that is installed in Connect, For example, this command: - -```bash -rsconnect deploy api --override-python-version 3.11.5 my-api/ -``` - -will deploy the content in `my-api` while requesting that Connect -use Python version 3.11.5. +We recommend providing a `pyproject.toml` with a `project.requires-python` field +if the deployed content is an installable package and a `.python-version` file +for plain directories. > **Note** > The packages and package versions listed in `requirements.txt` must be diff --git a/rsconnect/actions.py b/rsconnect/actions.py index c2589d63..2c57b664 100644 --- a/rsconnect/actions.py +++ b/rsconnect/actions.py @@ -31,13 +31,12 @@ from . import api from .bundle import ( - create_python_environment, get_default_entrypoint, make_api_bundle, make_quarto_source_bundle, read_manifest_file, ) -from .environment import Environment, EnvironmentException +from .environment import Environment from .exception import RSConnectException from .log import VERBOSE, logger from .models import AppMode, AppModes @@ -78,8 +77,6 @@ def failed(err: str): passed() except RSConnectException as exc: failed("Error: " + exc.message) - except EnvironmentException as exc: - failed("Error: " + str(exc)) except Exception as exc: traceback.print_exc() failed("Internal error: " + str(exc)) @@ -375,7 +372,7 @@ def deploy_app( ) ) - environment = create_python_environment( + environment = Environment.create_python_environment( directory, # pyright: ignore force_generate, python, diff --git a/rsconnect/api.py b/rsconnect/api.py index adf01b3e..4d455876 100644 --- a/rsconnect/api.py +++ b/rsconnect/api.py @@ -51,7 +51,8 @@ from typing_extensions import NotRequired, TypedDict from . import validation -from .bundle import _default_title, fake_module_file_from_directory +from .bundle import _default_title +from .environment import fake_module_file_from_directory from .certificates import read_certificate_file from .exception import DeploymentFailedException, RSConnectException from .http_support import CookieJar, HTTPResponse, HTTPServer, JsonData, append_to_path diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py index df9ba659..274b7ee8 100644 --- a/rsconnect/bundle.py +++ b/rsconnect/bundle.py @@ -30,7 +30,6 @@ splitext, ) from pathlib import Path -from pprint import pformat from typing import ( IO, TYPE_CHECKING, @@ -53,7 +52,7 @@ import click -from .environment import Environment, MakeEnvironment +from .environment import Environment, list_environment_dirs, is_environment_dir from .exception import RSConnectException from .log import VERBOSE, logger from .models import AppMode, AppModes, GlobSet @@ -103,9 +102,14 @@ class ManifestDataQuarto(TypedDict): engines: list[str] +class ManifestDataEnvironmentPython(TypedDict): + requires: NotRequired[str] + + class ManifestDataEnvironment(TypedDict): image: NotRequired[str] environment_management: NotRequired[dict[Literal["python", "r"], bool]] + python: NotRequired[ManifestDataEnvironmentPython] class ManifestDataPython(TypedDict): @@ -191,16 +195,22 @@ def __init__( }, } + if environment.python_version_requirement: + # If the environment has a python version requirement, + # add it to the manifest as environment.python.requires + manifest_environment = self.data.setdefault("environment", {}) + manifest_environment["python"] = {"requires": environment.python_version_requirement} + if image or env_management_py is not None or env_management_r is not None: - self.data["environment"] = {} + manifest_environment = self.data.setdefault("environment", {}) if image: - self.data["environment"]["image"] = image + manifest_environment["image"] = image if env_management_py is not None or env_management_r is not None: - self.data["environment"]["environment_management"] = {} + manifest_environment["environment_management"] = {} if env_management_py is not None: - self.data["environment"]["environment_management"]["python"] = env_management_py + manifest_environment["environment_management"]["python"] = env_management_py if env_management_r is not None: - self.data["environment"]["environment_management"]["r"] = env_management_r + manifest_environment["environment_management"]["r"] = env_management_r self.data["files"] = {} if files: @@ -378,56 +388,16 @@ def make_source_manifest( env_management_py: Optional[bool] = None, env_management_r: Optional[bool] = None, ) -> ManifestData: - manifest: ManifestData = cast(ManifestData, {"version": 1}) - - # When adding locale, add it early so it is ordered immediately after - # version. - if environment: - manifest["locale"] = environment.locale - - manifest["metadata"] = { - "appmode": app_mode.name(), - } - - if entrypoint: - manifest["metadata"]["entrypoint"] = entrypoint - - if quarto_inspection: - manifest["quarto"] = { - "version": quarto_inspection.get("quarto", {}).get("version", "99.9.9"), - "engines": quarto_inspection.get("engines", []), - } - - files_data = quarto_inspection.get("files", {}) - files_input_data = files_data.get("input", []) - if len(files_input_data) > 1: - manifest["metadata"]["content_category"] = "site" - - if environment: - package_manager = environment.package_manager - manifest["python"] = { - "version": environment.python, - "package_manager": { - "name": package_manager, - "version": getattr(environment, package_manager), - "package_file": environment.filename, - }, - } - - if image or env_management_py is not None or env_management_r is not None: - manifest["environment"] = {} - if image: - manifest["environment"]["image"] = image - if env_management_py is not None or env_management_r is not None: - manifest["environment"]["environment_management"] = {} - if env_management_py is not None: - manifest["environment"]["environment_management"]["python"] = env_management_py - if env_management_r is not None: - manifest["environment"]["environment_management"]["r"] = env_management_r - - manifest["files"] = {} - - return manifest + manifest: Manifest = Manifest( + app_mode=app_mode, + environment=environment, + entrypoint=entrypoint, + quarto_inspection=quarto_inspection, + image=image, + env_management_py=env_management_py, + env_management_r=env_management_r, + ) + return manifest.data def manifest_add_file(manifest: ManifestData, rel_path: str, base_dir: str) -> None: @@ -703,15 +673,13 @@ def make_html_manifest( filename: str, ) -> ManifestData: # noinspection SpellCheckingInspection - manifest: ManifestData = { - "version": 1, - "metadata": { - "appmode": "static", - "primary_html": filename, - }, - "files": {}, - } - return manifest + manifest = Manifest( + metadata=ManifestDataMetadata( + appmode="static", + primary_html=filename, + ) + ) + return manifest.data def make_notebook_html_bundle( @@ -874,27 +842,6 @@ def create_glob_set(directory: str | Path, excludes: Sequence[str]) -> GlobSet: return GlobSet(work) -def is_environment_dir(directory: str | Path): - """Detect whether `directory` is a virtualenv""" - - # A virtualenv will have Python at ./bin/python - python_path = join(directory, "bin", "python") - # But on Windows, it's at Scripts\Python.exe - win_path = join(directory, "Scripts", "Python.exe") - return exists(python_path) or exists(win_path) - - -def list_environment_dirs(directory: str | Path): - """Returns a list of subdirectories in `directory` that appear to contain virtual environments.""" - envs: list[str] = [] - - for name in os.listdir(directory): - path = join(directory, name) - if is_environment_dir(path): - envs.append(name) - return envs - - def make_api_manifest( directory: str, entry_point: str, @@ -1611,167 +1558,6 @@ def _warn_on_ignored_entrypoint(entrypoint: Optional[str]) -> None: ) -def _warn_on_ignored_manifest(directory: str) -> None: - """ - Checks for the existence of a file called manifest.json in the given directory. - If it's there, a warning noting that it will be ignored will be printed. - - :param directory: the directory to check in. - """ - if exists(join(directory, "manifest.json")): - click.secho( - " Warning: the existing manifest.json file will not be used or considered.", - fg="yellow", - ) - - -def _warn_if_no_requirements_file(directory: str) -> None: - """ - Checks for the existence of a file called requirements.txt in the given directory. - If it's not there, a warning will be printed. - - :param directory: the directory to check in. - """ - if not exists(join(directory, "requirements.txt")): - click.secho( - " Warning: Capturing the environment using 'pip freeze'.\n" - " Consider creating a requirements.txt file instead.", - fg="yellow", - ) - - -def _warn_if_environment_directory(directory: str | Path) -> None: - """ - Issue a warning if the deployment directory is itself a virtualenv (yikes!). - - :param directory: the directory to check in. - """ - if is_environment_dir(directory): - click.secho( - " Warning: The deployment directory appears to be a python virtual environment.\n" - " Python libraries and binaries will be excluded from the deployment.", - fg="yellow", - ) - - -def _warn_on_ignored_requirements(directory: str, requirements_file_name: str) -> None: - """ - Checks for the existence of a file called manifest.json in the given directory. - If it's there, a warning noting that it will be ignored will be printed. - - :param directory: the directory to check in. - :param requirements_file_name: the name of the requirements file. - """ - if exists(join(directory, requirements_file_name)): - click.secho( - " Warning: the existing %s file will not be used or considered." % requirements_file_name, - fg="yellow", - ) - - -def fake_module_file_from_directory(directory: str) -> str: - """ - Takes a directory and invents a properly named file that though possibly fake, - can be used for other name/title derivation. - - :param directory: the directory to start with. - :return: the directory plus the (potentially) fake module file. - """ - app_name = abspath(directory) - app_name = dirname(app_name) if app_name.endswith(os.path.sep) else basename(app_name) - return join(directory, app_name + ".py") - - -def which_python(python: Optional[str] = None) -> str: - """Determines which Python executable to use. - - If the :param python: is provided, then validation is performed to check if the path is an executable file. If - None, the invoking system Python executable location is returned. - - :param python: (Optional) path to a python executable. - :return: :param python: or `sys.executable`. - """ - if python is None: - return sys.executable - if not exists(python): - raise RSConnectException(f"The path '{python}' does not exist. Expected a Python executable.") - if isdir(python): - raise RSConnectException(f"The path '{python}' is a directory. Expected a Python executable.") - if not os.access(python, os.X_OK): - raise RSConnectException(f"The path '{python}' is not executable. Expected a Python executable") - return python - - -def inspect_environment( - python: str, - directory: str, - force_generate: bool = False, - check_output: Callable[..., bytes] = subprocess.check_output, -) -> Environment: - """Run the environment inspector using the specified python binary. - - Returns a dictionary of information about the environment, - or containing an "error" field if an error occurred. - """ - flags: list[str] = [] - if force_generate: - flags.append("f") - - args = [python, "-m", "rsconnect.environment"] - if flags: - args.append("-" + "".join(flags)) - args.append(directory) - - try: - environment_json = check_output(args, text=True) - except Exception as e: - raise RSConnectException("Error inspecting environment (subprocess failed)") from e - - try: - environment_data = json.loads(environment_json) - except json.JSONDecodeError as e: - raise RSConnectException("Error parsing environment JSON") from e - - try: - return MakeEnvironment(**environment_data) - except TypeError as e: - system_error_message = environment_data.get("error") - if system_error_message: - raise RSConnectException(f"Error creating environment: {system_error_message}") from e - raise RSConnectException("Error constructing environment object") from e - - -def get_python_env_info( - file_name: str, - python: str | None, - force_generate: bool = False, - override_python_version: str | None = None, -) -> tuple[str, Environment]: - """ - Gathers the python and environment information relating to the specified file - with an eye to deploy it. - - :param file_name: the primary file being deployed. - :param python: the optional name of a Python executable. - :param force_generate: force generating "requirements.txt" or "environment.yml", - even if it already exists. - :return: information about the version of Python in use plus some environmental - stuff. - """ - python = which_python(python) - logger.debug("Python: %s" % python) - environment = inspect_environment(python, dirname(file_name), force_generate=force_generate) - if environment.error: - raise RSConnectException(environment.error) - logger.debug("Python: %s" % python) - logger.debug("Environment: %s" % pformat(environment._asdict())) - - if override_python_version: - environment = environment._replace(python=override_python_version) - - return python, environment - - def create_notebook_manifest_and_environment_file( entry_point_file: str, environment: Environment, @@ -2254,26 +2040,3 @@ def write_manifest_json(manifest_path: str | Path, manifest: ManifestData) -> No with open(manifest_path, "w") as f: json.dump(manifest, f, indent=2) f.write("\n") - - -def create_python_environment( - directory: str, - force_generate: bool = False, - python: Optional[str] = None, - override_python_version: Optional[str] = None, -) -> Environment: - module_file = fake_module_file_from_directory(directory) - - # click.secho(' Deploying %s to server "%s"' % (directory, connect_server.url)) - - _warn_on_ignored_manifest(directory) - _warn_if_no_requirements_file(directory) - _warn_if_environment_directory(directory) - - # with cli_feedback("Inspecting Python environment"): - _, environment = get_python_env_info(module_file, python, force_generate, override_python_version) - - if force_generate: - _warn_on_ignored_requirements(directory, environment.filename) - - return environment diff --git a/rsconnect/environment.py b/rsconnect/environment.py index 9858d392..ec0052bd 100644 --- a/rsconnect/environment.py +++ b/rsconnect/environment.py @@ -1,233 +1,343 @@ -#!/usr/bin/env python -""" -Environment data class abstraction that is usable as an executable module +"""Detects the configuration of a Python environment. + +Given a directory and a Python executable, this module inspects the environment +and returns information about the Python version and the environment itself. -```bash -python -m rsconnect.environment -``` +To inspect the environment it relies on a subprocess that runs the `rsconnect.subprocesses.inspect_environment` +module. This module is responsible for gathering the environment information and returning it in a JSON format. """ -from __future__ import annotations -import datetime -import json -import locale -import os -import re -import subprocess +import typing import sys -from dataclasses import asdict, dataclass, replace -from typing import Callable, Optional - -version_re = re.compile(r"\d+\.\d+(\.\d+)?") -exec_dir = os.path.dirname(sys.executable) +import dataclasses +import pprint +import subprocess +import json +import pathlib +import os.path +from . import pyproject +from .log import logger +from .exception import RSConnectException +from .subprocesses.inspect_environment import EnvironmentData, MakeEnvironmentData as _MakeEnvironmentData -@dataclass(frozen=True) -class Environment: - contents: str - filename: str - locale: str - package_manager: str - pip: str - python: str - source: str - error: str | None +import click - def _asdict(self): - return asdict(self) - - def _replace(self, **kwargs: object): - return replace(self, **kwargs) +class Environment: + """A Python project environment, -def MakeEnvironment( - contents: str, - filename: str, - locale: str, - package_manager: str, - pip: str, - python: str, - source: str, - error: Optional[str] = None, - **kwargs: object, # provides compatibility where we no longer support some older properties -) -> Environment: - return Environment(contents, filename, locale, package_manager, pip, python, source, error) + The data is loaded from a rsconnect.utils.environment json response, + the environment contains all the information provided by :class:`EnvironmentData` plus + the environment python interpreter and the python interpreter version requirement. + The goal is to capture all the information needed to replicate such environment. + """ -class EnvironmentException(Exception): - pass + DATA_FIELDS = {f.name for f in dataclasses.fields(EnvironmentData)} + + def __init__( + self, + data: EnvironmentData, + python_interpreter: typing.Optional[str] = None, + python_version_requirement: typing.Optional[str] = None, + ): + self._data = data + + # Fields that are not loaded from the environment subprocess + self.python_version_requirement = python_version_requirement + self.python_interpreter = python_interpreter + + def __getattr__(self, name: str) -> typing.Any: + # We directly proxy the attributes of the EnvironmentData object + # so that schema changes can be handled in EnvironmentData exclusively. + return getattr(self._data, name) + + def __setattr__(self, name: str, value: typing.Any) -> None: + if name in self.DATA_FIELDS: + # proxy the attribute to the underlying EnvironmentData object + self._data._replace(**{name: value}) + else: + super().__setattr__(name, value) + + def __eq__(self, other: typing.Any) -> bool: + if not isinstance(other, Environment): + return False + + return ( + self._data == other._data + and self.python_interpreter == other.python_interpreter + and self.python_version_requirement == other.python_version_requirement + ) + def __repr__(self) -> str: + data = self._data._asdict() + data.pop("contents", None) # Remove contents as it's too long to display + return ( + f"Environment({data}, " + f"python_interpreter={self.python_interpreter}, " + f"python_version_requirement={self.python_version_requirement})" + ) -def detect_environment(dirname: str, force_generate: bool = False) -> Environment: - """Determine the python dependencies in the environment. + @classmethod + def from_dict( + cls, + data: typing.Dict[str, typing.Any], + python_interpreter: typing.Optional[str] = None, + python_version_requirement: typing.Optional[str] = None, + ) -> "Environment": + """Create an Environment instance from the dictionary representation of EnvironmentData.""" + return cls( + _MakeEnvironmentData(**data), + python_interpreter=python_interpreter, + python_version_requirement=python_version_requirement, + ) - `pip freeze` will be used to introspect the environment. + @classmethod + def create_python_environment( + cls, + directory: str, + force_generate: bool = False, + python: typing.Optional[str] = None, + override_python_version: typing.Optional[str] = None, + app_file: typing.Optional[str] = None, + ) -> "Environment": + """Given a project directory and a Python executable, return Environment information. + + If no Python executable is provided, the current system Python executable is used. + + :param directory: the project directory to inspect. + :param force_generate: force generating "requirements.txt" to snapshot the environment + packages even if it already exists. + :param python: the Python executable of the environment to use for inspection. + :param override_python_version: the Python version required by the project. + :param app_file: the main application file to use for inspection. + + :return: a tuple containing the Python executable of the environment and the Environment object. + """ + if app_file is None: + module_file = fake_module_file_from_directory(directory) + else: + module_file = app_file + + # click.secho(' Deploying %s to server "%s"' % (directory, connect_server.url)) + _warn_on_ignored_manifest(directory) + _warn_if_no_requirements_file(directory) + _warn_if_environment_directory(directory) + + python_version_requirement = pyproject.detect_python_version_requirement(directory) + _warn_on_missing_python_version(python_version_requirement) + + if override_python_version: + # TODO: --override-python-version should be deprecated in the future + # and instead we should suggest the user sets it in .python-version + # or pyproject.toml + python_version_requirement = f"=={override_python_version}" + + # with cli_feedback("Inspecting Python environment"): + environment = cls._get_python_env_info(module_file, python, force_generate) + environment.python_version_requirement = python_version_requirement + + if override_python_version: + # Retaing backward compatibility with old Connect versions + # that didn't support environment.python.requires + environment.python = override_python_version + + if force_generate: + _warn_on_ignored_requirements(directory, environment.filename) + + return environment + + @classmethod + def _get_python_env_info( + cls, file_name: str, python: typing.Optional[str], force_generate: bool = False + ) -> "Environment": + """ + Gathers the python and environment information relating to the specified file + with an eye to deploy it. + + :param file_name: the primary file being deployed. + :param python: the optional name of a Python executable. + :param force_generate: force generating "requirements.txt" or "environment.yml", + even if it already exists. + :return: information about the version of Python in use plus some environmental + stuff. + """ + python = which_python(python) + logger.debug("Python: %s" % python) + environment = cls._inspect_environment(python, os.path.dirname(file_name), force_generate=force_generate) + if environment.error: + raise RSConnectException(environment.error) + logger.debug("Python: %s" % python) + logger.debug("Environment: %s" % pprint.pformat(environment._asdict())) + return environment + + @classmethod + def _inspect_environment( + cls, + python: str, + directory: str, + force_generate: bool = False, + check_output: typing.Callable[..., bytes] = subprocess.check_output, + ) -> "Environment": + """Run the environment inspector using the specified python binary. + + Returns a dictionary of information about the environment, + or containing an "error" field if an error occurred. + """ + flags: typing.List[str] = [] + if force_generate: + flags.append("f") + + args = [python, "-m", "rsconnect.subprocesses.inspect_environment"] + if flags: + args.append("-" + "".join(flags)) + args.append(directory) + + try: + environment_json = check_output(args, text=True) + except Exception as e: + raise RSConnectException("Error inspecting environment (subprocess failed)") from e + + try: + environment_data = json.loads(environment_json) + except json.JSONDecodeError as e: + raise RSConnectException("Error parsing environment JSON") from e + + if "error" in environment_data: + system_error_message = environment_data.get("error") + if system_error_message: + raise RSConnectException(f"Error creating environment: {system_error_message}") + + try: + return cls.from_dict(environment_data, python_interpreter=python) + except TypeError as e: + raise RSConnectException("Error constructing environment object") from e + + +def which_python(python: typing.Optional[str] = None) -> str: + """Determines which Python executable to use. + + If the :param python: is provided, then validation is performed to check if the path is an executable file. If + None, the invoking system Python executable location is returned. + + :param python: (Optional) path to a python executable. + :return: :param python: or `sys.executable`. + """ + if python is None: + return sys.executable + if not os.path.exists(python): + raise RSConnectException(f"The path '{python}' does not exist. Expected a Python executable.") + if os.path.isdir(python): + raise RSConnectException(f"The path '{python}' is a directory. Expected a Python executable.") + if not os.access(python, os.X_OK): + raise RSConnectException(f"The path '{python}' is not executable. Expected a Python executable") + return python + + +def fake_module_file_from_directory(directory: str) -> str: + """ + Takes a directory and invents a properly named file that though possibly fake, + can be used for other name/title derivation. - :param: dirname Directory name - :param: force_generate Force the generation of an environment - :return: a dictionary containing the package spec filename and contents if successful, - or a dictionary containing `error` on failure. + :param directory: the directory to start with. + :return: the directory plus the (potentially) fake module file. """ + app_name = os.path.abspath(directory) + app_name = os.path.dirname(app_name) if app_name.endswith(os.path.sep) else os.path.basename(app_name) + return os.path.join(directory, app_name + ".py") - if force_generate: - result = pip_freeze() - else: - result = output_file(dirname, "requirements.txt", "pip") or pip_freeze() - if result is not None: - result["python"] = get_python_version() - result["pip"] = get_version("pip") - result["locale"] = get_default_locale() +def is_environment_dir(directory: typing.Union[str, pathlib.Path]) -> bool: + """Detect whether `directory` is a virtualenv""" - return MakeEnvironment(**result) + # A virtualenv will have Python at ./bin/python + python_path = os.path.join(directory, "bin", "python") + # But on Windows, it's at Scripts\Python.exe + win_path = os.path.join(directory, "Scripts", "Python.exe") + return os.path.exists(python_path) or os.path.exists(win_path) -def get_python_version() -> str: - v = sys.version_info - return "%d.%d.%d" % (v[0], v[1], v[2]) +def list_environment_dirs(directory: typing.Union[str, pathlib.Path]) -> typing.List[str]: + """Returns a list of subdirectories in `directory` that appear to contain virtual environments.""" + envs: typing.List[str] = [] + for name in os.listdir(directory): + path = os.path.join(directory, name) + if is_environment_dir(path): + envs.append(name) + return envs -def get_default_locale(locale_source: Callable[..., tuple[str | None, str | None]] = locale.getlocale): - result = ".".join([item or "" for item in locale_source()]) - return "" if result == "." else result +def _warn_on_ignored_manifest(directory: str) -> None: + """ + Checks for the existence of a file called manifest.json in the given directory. + If it's there, a warning noting that it will be ignored will be printed. -def get_version(module: str): - try: - args = [sys.executable, "-m", module, "--version"] - proc = subprocess.Popen( - args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - stdout, _stderr = proc.communicate() - match = version_re.search(stdout) - if match: - return match.group() - - msg = "Failed to get version of '%s' from the output of: %s" % ( - module, - " ".join(args), + :param directory: the directory to check in. + """ + if os.path.exists(os.path.join(directory, "manifest.json")): + click.secho( + " Warning: the existing manifest.json file will not be used or considered.", + fg="yellow", ) - raise EnvironmentException(msg) - except Exception as exception: - raise EnvironmentException("Error getting '%s' version: %s" % (module, str(exception))) - -def output_file(dirname: str, filename: str, package_manager: str): - """Read an existing package spec file. - Returns a dictionary containing the filename and contents - if successful, None if the file does not exist, - or a dictionary containing 'error' on failure. +def _warn_if_no_requirements_file(directory: str) -> None: """ - try: - path = os.path.join(dirname, filename) - if not os.path.exists(path): - return None + Checks for the existence of a file called requirements.txt in the given directory. + If it's not there, a warning will be printed. - with open(path, "r") as f: - data = f.read() - - data = "\n".join([line for line in data.split("\n") if "rsconnect" not in line]) - - return { - "filename": filename, - "contents": data, - "source": "file", - "package_manager": package_manager, - } - except Exception as exception: - raise EnvironmentException("Error reading %s: %s" % (filename, str(exception))) - - -def pip_freeze(): - """Inspect the environment using `pip freeze --disable-pip-version-check version`. - - Returns a dictionary containing the filename - (always 'requirements.txt') and contents if successful, - or a dictionary containing 'error' on failure. + :param directory: the directory to check in. """ - try: - proc = subprocess.Popen( - [sys.executable, "-m", "pip", "freeze", "--disable-pip-version-check"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, + if not os.path.exists(os.path.join(directory, "requirements.txt")): + click.secho( + " Warning: Capturing the environment using 'pip freeze'.\n" + " Consider creating a requirements.txt file instead.", + fg="yellow", ) - pip_stdout, pip_stderr = proc.communicate() - pip_status = proc.returncode - except Exception as exception: - raise EnvironmentException("Error during pip freeze: %s" % str(exception)) - - if pip_status != 0: - msg = pip_stderr or ("exited with code %d" % pip_status) - raise EnvironmentException("Error during pip freeze: %s" % msg) - - pip_stdout = filter_pip_freeze_output(pip_stdout) - - pip_stdout = ( - "# requirements.txt generated by rsconnect-python on " - + str(datetime.datetime.now(datetime.timezone.utc)) - + "\n" - + pip_stdout - ) - - return { - "filename": "requirements.txt", - "contents": pip_stdout, - "source": "pip_freeze", - "package_manager": "pip", - } - -def filter_pip_freeze_output(pip_stdout: str): - # Filter out dependency on `rsconnect` and ignore output lines from pip which start with `[notice]` - return "\n".join( - [line for line in pip_stdout.split("\n") if (("rsconnect" not in line) and (line.find("[notice]") != 0))] - ) +def _warn_if_environment_directory(directory: typing.Union[str, pathlib.Path]) -> None: + """ + Issue a warning if the deployment directory is itself a virtualenv (yikes!). + :param directory: the directory to check in. + """ + if is_environment_dir(directory): + click.secho( + " Warning: The deployment directory appears to be a python virtual environment.\n" + " Python libraries and binaries will be excluded from the deployment.", + fg="yellow", + ) -def strip_ref(line: str): - # remove erroneous conda build paths that will break pip install - return line.split(" @ file:", 1)[0].strip() +def _warn_on_ignored_requirements(directory: str, requirements_file_name: str) -> None: + """ + Checks for the existence of a file called manifest.json in the given directory. + If it's there, a warning noting that it will be ignored will be printed. -def exclude(line: str): - return line and line.startswith("setuptools") and "post" in line + :param directory: the directory to check in. + :param requirements_file_name: the name of the requirements file. + """ + if os.path.exists(os.path.join(directory, requirements_file_name)): + click.secho( + " Warning: the existing %s file will not be used or considered." % requirements_file_name, + fg="yellow", + ) -def main(): +def _warn_on_missing_python_version(version_constraint: typing.Optional[str]) -> None: """ - Run `detect_environment` and dump the result as JSON. + Check that the project has a Python version constraint requested. + If it doesn't warn the user that it should be specified. + + :param version_constraint: the version constraint in the project. """ - try: - if len(sys.argv) < 2: - raise EnvironmentException("Usage: %s [-fc] DIRECTORY" % sys.argv[0]) - # directory is always the last argument - directory = sys.argv[len(sys.argv) - 1] - flags = "" - force_generate = False - if len(sys.argv) > 2: - flags = sys.argv[1] - if "f" in flags: - force_generate = True - envinfo = detect_environment(directory, force_generate)._asdict() - if "contents" in envinfo: - keepers = list(map(strip_ref, envinfo["contents"].split("\n"))) - keepers = [line for line in keepers if not exclude(line)] - envinfo["contents"] = "\n".join(keepers) - - json.dump( - envinfo, - sys.stdout, - indent=4, + if version_constraint is None: + click.secho( + " Warning: Python version constraint missing from pyproject.toml, setup.cfg or .python-version\n" + " Connect will guess the version to use based on local environment.\n" + " Consider specifying a Python version constraint.", + fg="yellow", ) - except EnvironmentException as exception: - json.dump(dict(error=str(exception)), sys.stdout, indent=4) - - -if __name__ == "__main__": - main() diff --git a/rsconnect/main.py b/rsconnect/main.py index 28470478..a979c0e4 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -48,13 +48,10 @@ get_content, search_content, ) +from .environment import Environment, fake_module_file_from_directory from .api import RSConnectClient, RSConnectExecutor, RSConnectServer from .bundle import ( - create_python_environment, default_title_from_manifest, - fake_module_file_from_directory, - get_python_env_info, - is_environment_dir, make_api_bundle, make_html_bundle, make_manifest_bundle, @@ -74,7 +71,6 @@ write_tensorflow_manifest_json, write_voila_manifest_json, ) -from .environment import EnvironmentException from .exception import RSConnectException from .json_web_token import ( TokenGenerator, @@ -116,8 +112,6 @@ def failed(err: str) -> Never: result = func(*args, **kwargs) except RSConnectException as exc: failed("Error: " + exc.message) - except EnvironmentException as exc: - failed("Error: " + str(exc)) except Exception as exc: traceback.print_exc() failed("Internal error: " + str(exc)) @@ -813,35 +807,6 @@ def _warn_on_ignored_manifest(directory: str): ) -def _warn_if_no_requirements_file(directory: str): - """ - Checks for the existence of a file called requirements.txt in the given directory. - If it's not there, a warning will be printed. - - :param directory: the directory to check in. - """ - if not exists(join(directory, "requirements.txt")): - click.secho( - " Warning: Capturing the environment using 'pip freeze --disable-pip-version-check'.\n" - " Consider creating a requirements.txt file instead.", - fg="yellow", - ) - - -def _warn_if_environment_directory(directory: str): - """ - Issue a warning if the deployment directory is itself a virtualenv (yikes!). - - :param directory: the directory to check in. - """ - if is_environment_dir(directory): - click.secho( - " Warning: The deployment directory appears to be a python virtual environment.\n" - " Python libraries and binaries will be excluded from the deployment.", - fg="yellow", - ) - - def _warn_on_ignored_requirements(directory: str, requirements_file_name: str): """ Checks for the existence of a file called manifest.json in the given directory. @@ -948,10 +913,13 @@ def deploy_notebook( app_mode = AppModes.JUPYTER_NOTEBOOK if not static else AppModes.STATIC base_dir = dirname(file) - _warn_on_ignored_manifest(base_dir) - _warn_if_no_requirements_file(base_dir) - _warn_if_environment_directory(base_dir) - python, environment = get_python_env_info(file, python, force_generate, override_python_version) + environment = Environment.create_python_environment( + base_dir, + app_file=file, + force_generate=force_generate, + python=python, + override_python_version=override_python_version, + ) if force_generate: _warn_on_ignored_requirements(base_dir, environment.filename) @@ -976,7 +944,7 @@ def deploy_notebook( ce.make_bundle( make_notebook_html_bundle, file, - python, + environment.python, hide_all_input, hide_tagged_input, ) @@ -1086,7 +1054,7 @@ def deploy_voila( set_verbosity(verbose) output_params(ctx, locals().items()) app_mode = AppModes.JUPYTER_VOILA - environment = create_python_environment( + environment = Environment.create_python_environment( path if isdir(path) else dirname(path), force_generate, python, override_python_version ) @@ -1290,7 +1258,6 @@ def deploy_quarto( base_dir = file_or_directory if not isdir(file_or_directory): base_dir = dirname(file_or_directory) - module_file = fake_module_file_from_directory(file_or_directory) extra_files = validate_extra_files(base_dir, extra_files) _warn_on_ignored_manifest(base_dir) @@ -1301,17 +1268,12 @@ def deploy_quarto( inspect = quarto_inspect(quarto, file_or_directory) engines = validate_quarto_engines(inspect) - python = None environment = None if "jupyter" in engines: - _warn_if_no_requirements_file(base_dir) - _warn_if_environment_directory(base_dir) - with cli_feedback("Inspecting Python environment"): - python, environment = get_python_env_info(module_file, python, force_generate, override_python_version) - - if force_generate: - _warn_on_ignored_requirements(base_dir, environment.filename) + environment = Environment.create_python_environment( + base_dir, force_generate=force_generate, override_python_version=override_python_version + ) ce = RSConnectExecutor( ctx=ctx, @@ -1652,7 +1614,7 @@ def deploy_app( set_verbosity(verbose) entrypoint = validate_entry_point(entrypoint, directory) extra_files_list = validate_extra_files(directory, extra_files) - environment = create_python_environment( + environment = Environment.create_python_environment( directory, force_generate, python, override_python_version=override_python_version ) @@ -1823,7 +1785,13 @@ def write_manifest_notebook( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - python, environment = get_python_env_info(file, python, force_generate, override_python_version) + environment = Environment.create_python_environment( + base_dir, + force_generate=force_generate, + python=python, + override_python_version=override_python_version, + app_file=file, + ) with cli_feedback("Creating manifest.json"): environment_file_exists = write_notebook_manifest_json( @@ -1929,7 +1897,13 @@ def write_manifest_voila( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - python, environment = get_python_env_info(path, python, force_generate, override_python_version) + environment = Environment.create_python_environment( + base_dir, + force_generate=force_generate, + override_python_version=override_python_version, + python=python, + app_file=path, + ) environment_file_exists = exists(join(base_dir, environment.filename)) @@ -2052,7 +2026,9 @@ def write_manifest_quarto( environment = None if "jupyter" in engines: with cli_feedback("Inspecting Python environment"): - python, environment = get_python_env_info(base_dir, python, force_generate, override_python_version) + environment = Environment.create_python_environment( + base_dir, force_generate=force_generate, override_python_version=override_python_version, python=python + ) environment_file_exists = exists(join(base_dir, environment.filename)) if environment_file_exists and not force_generate: @@ -2295,7 +2271,12 @@ def _write_framework_manifest( raise RSConnectException("manifest.json already exists. Use --overwrite to overwrite.") with cli_feedback("Inspecting Python environment"): - _, environment = get_python_env_info(directory, python, force_generate, override_python_version) + environment = Environment.create_python_environment( + directory, + force_generate=force_generate, + override_python_version=override_python_version, + python=python, + ) if app_mode == AppModes.PYTHON_SHINY: with cli_feedback("Inspecting Shiny for Python app"): diff --git a/rsconnect/pyproject.py b/rsconnect/pyproject.py index 553ece9c..0e3c0adc 100644 --- a/rsconnect/pyproject.py +++ b/rsconnect/pyproject.py @@ -16,6 +16,23 @@ import toml as tomllib # type: ignore[no-redef] +def detect_python_version_requirement(directory: typing.Union[str, pathlib.Path]) -> typing.Optional[str]: + """Detect the python version requirement for a project. + + The directory should contain a metadata file such as pyproject.toml, + setup.cfg, or .python-version. + + Returns the python version requirement as a string or None if not found. + """ + for _, metadata_file in lookup_metadata_file(directory): + parser = get_python_version_requirement_parser(metadata_file) + version_constraint = parser(metadata_file) + if version_constraint: + return version_constraint + + return None + + def lookup_metadata_file(directory: typing.Union[str, pathlib.Path]) -> typing.List[typing.Tuple[str, pathlib.Path]]: """Given the directory of a project return the path of a usable metadata file. @@ -38,7 +55,7 @@ def _generate(): def get_python_version_requirement_parser( metadata_file: pathlib.Path, -) -> typing.Optional[typing.Callable[[pathlib.Path], typing.Optional[str]]]: +) -> typing.Callable[[pathlib.Path], typing.Optional[str]]: """Given the metadata file, return the appropriate parser function. The returned function takes a pathlib.Path and returns the parsed value. @@ -50,7 +67,7 @@ def get_python_version_requirement_parser( elif metadata_file.name == ".python-version": return parse_pyversion_python_requires else: - return None + raise NotImplementedError(f"Unknown metadata file type: {metadata_file.name}") def parse_pyproject_python_requires(pyproject_file: pathlib.Path) -> typing.Optional[str]: diff --git a/rsconnect/subprocesses/__init__.py b/rsconnect/subprocesses/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rsconnect/subprocesses/inspect_environment.py b/rsconnect/subprocesses/inspect_environment.py new file mode 100644 index 00000000..97b2c9a4 --- /dev/null +++ b/rsconnect/subprocesses/inspect_environment.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python +""" +Environment data class abstraction that is usable as an executable module + +```bash +python -m rsconnect.subprocesses.inspect_environment +``` +""" +from __future__ import annotations + +import datetime +import json +import locale +import os +import re +import subprocess +import sys +from dataclasses import asdict, dataclass, replace +from typing import Callable, Optional + +version_re = re.compile(r"\d+\.\d+(\.\d+)?") +exec_dir = os.path.dirname(sys.executable) + + +@dataclass(frozen=True) +class EnvironmentData: + contents: str + filename: str + locale: str + package_manager: str + pip: str + python: str + source: str + python_requires: Optional[str] + error: Optional[str] + + def _asdict(self): + return asdict(self) + + def _replace(self, **kwargs: object): + return replace(self, **kwargs) + + +def MakeEnvironmentData( + contents: str, + filename: str, + locale: str, + package_manager: str, + pip: str, + python: str, + source: str, + python_requires: Optional[str] = None, + error: Optional[str] = None, + **kwargs: object, # provides compatibility where we no longer support some older properties +) -> EnvironmentData: + return EnvironmentData(contents, filename, locale, package_manager, pip, python, source, python_requires, error) + + +class EnvironmentException(Exception): + pass + + +def detect_environment(dirname: str, force_generate: bool = False) -> EnvironmentData: + """Determine the python dependencies in the environment. + + `pip freeze` will be used to introspect the environment. + + :param: dirname Directory name + :param: force_generate Force the generation of an environment + :return: a dictionary containing the package spec filename and contents if successful, + or a dictionary containing `error` on failure. + """ + + if force_generate: + result = pip_freeze() + else: + result = output_file(dirname, "requirements.txt", "pip") or pip_freeze() + + if result is not None: + result["python"] = get_python_version() + result["pip"] = get_version("pip") + result["locale"] = get_default_locale() + + return MakeEnvironmentData(**result) + + +def get_python_version() -> str: + v = sys.version_info + return "%d.%d.%d" % (v[0], v[1], v[2]) + + +def get_default_locale(locale_source: Callable[..., tuple[str | None, str | None]] = locale.getlocale): + result = ".".join([item or "" for item in locale_source()]) + return "" if result == "." else result + + +def get_version(module: str): + try: + args = [sys.executable, "-m", module, "--version"] + proc = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, _stderr = proc.communicate() + match = version_re.search(stdout) + if match: + return match.group() + + msg = "Failed to get version of '%s' from the output of: %s" % ( + module, + " ".join(args), + ) + raise EnvironmentException(msg) + except Exception as exception: + raise EnvironmentException("Error getting '%s' version: %s" % (module, str(exception))) + + +def output_file(dirname: str, filename: str, package_manager: str): + """Read an existing package spec file. + + Returns a dictionary containing the filename and contents + if successful, None if the file does not exist, + or a dictionary containing 'error' on failure. + """ + try: + path = os.path.join(dirname, filename) + if not os.path.exists(path): + return None + + with open(path, "r") as f: + data = f.read() + + data = "\n".join([line for line in data.split("\n") if "rsconnect" not in line]) + + return { + "filename": filename, + "contents": data, + "source": "file", + "package_manager": package_manager, + } + except Exception as exception: + raise EnvironmentException("Error reading %s: %s" % (filename, str(exception))) + + +def pip_freeze(): + """Inspect the environment using `pip freeze --disable-pip-version-check version`. + + Returns a dictionary containing the filename + (always 'requirements.txt') and contents if successful, + or a dictionary containing 'error' on failure. + """ + try: + proc = subprocess.Popen( + [sys.executable, "-m", "pip", "freeze", "--disable-pip-version-check"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + pip_stdout, pip_stderr = proc.communicate() + pip_status = proc.returncode + except Exception as exception: + raise EnvironmentException("Error during pip freeze: %s" % str(exception)) + + if pip_status != 0: + msg = pip_stderr or ("exited with code %d" % pip_status) + raise EnvironmentException("Error during pip freeze: %s" % msg) + + pip_stdout = filter_pip_freeze_output(pip_stdout) + + pip_stdout = ( + "# requirements.txt generated by rsconnect-python on " + + str(datetime.datetime.now(datetime.timezone.utc)) + + "\n" + + pip_stdout + ) + + return { + "filename": "requirements.txt", + "contents": pip_stdout, + "source": "pip_freeze", + "package_manager": "pip", + } + + +def filter_pip_freeze_output(pip_stdout: str): + # Filter out dependency on `rsconnect` and ignore output lines from pip which start with `[notice]` + return "\n".join( + [line for line in pip_stdout.split("\n") if (("rsconnect" not in line) and (line.find("[notice]") != 0))] + ) + + +def strip_ref(line: str): + # remove erroneous conda build paths that will break pip install + return line.split(" @ file:", 1)[0].strip() + + +def exclude(line: str): + return line and line.startswith("setuptools") and "post" in line + + +def main(): + """ + Run `detect_environment` and dump the result as JSON. + """ + try: + if len(sys.argv) < 2: + raise EnvironmentException("Usage: %s [-fc] DIRECTORY" % sys.argv[0]) + # directory is always the last argument + directory = sys.argv[len(sys.argv) - 1] + flags = "" + force_generate = False + if len(sys.argv) > 2: + flags = sys.argv[1] + if "f" in flags: + force_generate = True + envinfo = detect_environment(directory, force_generate)._asdict() + if "contents" in envinfo: + keepers = list(map(strip_ref, envinfo["contents"].split("\n"))) + keepers = [line for line in keepers if not exclude(line)] + envinfo["contents"] = "\n".join(keepers) + + json.dump( + envinfo, + sys.stdout, + indent=4, + ) + except EnvironmentException as exception: + json.dump(dict(error=str(exception)), sys.stdout, indent=4) + + +if __name__ == "__main__": + main() diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 30e727c2..e300954e 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import json import os -import subprocess import sys import tarfile import tempfile @@ -11,17 +10,13 @@ import pytest -import rsconnect.bundle from rsconnect.bundle import ( Manifest, _default_title, _default_title_from_manifest, create_html_manifest, - create_python_environment, create_voila_manifest, - get_python_env_info, guess_deploy_dir, - inspect_environment, keep_manifest_specified_file, list_files, make_api_bundle, @@ -40,9 +35,8 @@ to_bytes, validate_entry_point, validate_extra_files, - which_python, ) -from rsconnect.environment import Environment, MakeEnvironment, detect_environment +from rsconnect.environment import Environment from rsconnect.exception import RSConnectException from rsconnect.models import AppModes @@ -85,7 +79,7 @@ def test_make_notebook_source_bundle1(self): # the test environment. Don't do this in the production code, which # runs in the notebook server. We need the introspection to run in # the kernel environment and not the notebook server environment. - environment = detect_environment(directory) + environment = Environment.create_python_environment(directory) with make_notebook_source_bundle( nb_path, environment, @@ -155,7 +149,7 @@ def test_make_notebook_source_bundle2(self): # the test environment. Don't do this in the production code, which # runs in the notebook server. We need the introspection to run in # the kernel environment and not the notebook server environment. - environment = detect_environment(directory) + environment = Environment.create_python_environment(directory) with make_notebook_source_bundle( nb_path, @@ -255,7 +249,7 @@ def test_make_quarto_source_bundle_from_simple_project(self): # input file. create_fake_quarto_rendered_output(temp_proj, "myquarto") - environment = detect_environment(temp_proj) + environment = Environment.create_python_environment(temp_proj) # mock the result of running of `quarto inspect ` inspect = { @@ -352,7 +346,7 @@ def test_make_quarto_source_bundle_from_complex_project(self): create_fake_quarto_rendered_output(site_dir, "index") create_fake_quarto_rendered_output(site_dir, "about") - environment = detect_environment(temp_proj) + environment = Environment.create_python_environment(temp_proj) # mock the result of running of `quarto inspect ` inspect = { @@ -454,7 +448,7 @@ def test_make_quarto_source_bundle_from_project_with_requirements(self): fp.write("pandas\n") fp.close() - environment = detect_environment(temp_proj) + environment = Environment.create_python_environment(temp_proj) # mock the result of running of `quarto inspect ` inspect = { @@ -743,15 +737,17 @@ def test_make_source_manifest(self): # include environment parameter manifest = make_source_manifest( AppModes.PYTHON_API, - Environment( - contents="", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="22.0.4", - python="3.9.12", - source="file", + Environment.from_dict( + dict( + contents="", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="22.0.4", + python="3.9.12", + source="file", + ) ), None, None, @@ -922,15 +918,17 @@ def test_make_quarto_manifest_project_with_env(self): "config": {"project": {"title": "quarto-proj-py"}, "editor": "visual", "language": {}}, }, AppModes.SHINY_QUARTO, - Environment( - contents="", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="22.0.4", - python="3.9.12", - source="file", + Environment.from_dict( + dict( + contents="", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="22.0.4", + python="3.9.12", + source="file", + ) ), [], [], @@ -1207,132 +1205,6 @@ def test_default_title_from_manifest(self): m = {"metadata": {"entrypoint": "module:object"}} self.assertEqual(_default_title_from_manifest(m, "dir/to/manifest.json"), "0to") - def test_inspect_environment(self): - environment = inspect_environment(sys.executable, get_dir("pip1")) - assert environment is not None - assert environment.python != "" - - def test_inspect_environment_catches_type_error(self): - with pytest.raises(RSConnectException) as exec_info: - inspect_environment(sys.executable, None) # type: ignore - - assert isinstance(exec_info.value, RSConnectException) - assert isinstance(exec_info.value.__cause__, TypeError) - - -@pytest.mark.parametrize( - ( - "file_name", - "python", - "force_generate", - "expected_python", - "expected_environment", - ), - [ - pytest.param( - "path/to/file.py", - sys.executable, - False, - sys.executable, - MakeEnvironment( - contents=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip=None, - python=None, - source="pip_freeze", - error=None, - ), - id="basic", - ), - pytest.param( - "another/file.py", - basename(sys.executable), - False, - sys.executable, - MakeEnvironment( - contents=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip=None, - python=None, - source="pip_freeze", - error=None, - ), - id="which_python", - ), - pytest.param( - "will/the/files/never/stop.py", - "argh.py", - False, - "unused", - MakeEnvironment(None, None, None, None, None, None, None, error="Could not even do things"), - id="exploding", - ), - ], -) -def test_get_python_env_info( - monkeypatch, - file_name, - python, - force_generate, - expected_python, - expected_environment, -): - def fake_which_python(python, env=os.environ): - return expected_python - - def fake_inspect_environment( - python, - directory, - force_generate=False, - check_output=subprocess.check_output, - ): - return expected_environment - - monkeypatch.setattr(rsconnect.bundle, "inspect_environment", fake_inspect_environment) - - monkeypatch.setattr(rsconnect.bundle, "which_python", fake_which_python) - - if expected_environment.error is not None: - with pytest.raises(RSConnectException): - _, _ = get_python_env_info(file_name, python, force_generate=force_generate) - else: - python, environment = get_python_env_info(file_name, python, force_generate=force_generate) - - assert python == expected_python - assert environment == expected_environment - - -class WhichPythonTestCase(TestCase): - def test_default(self): - self.assertEqual(which_python(), sys.executable) - - def test_none(self): - self.assertEqual(which_python(None), sys.executable) - - def test_sys(self): - self.assertEqual(which_python(sys.executable), sys.executable) - - def test_does_not_exist(self): - with tempfile.NamedTemporaryFile() as tmpfile: - name = tmpfile.name - with self.assertRaises(RSConnectException): - which_python(name) - - def test_is_directory(self): - with tempfile.TemporaryDirectory() as tmpdir: - with self.assertRaises(RSConnectException): - which_python(tmpdir) - - @pytest.mark.skipif(sys.platform.startswith("win"), reason="os.X_OK always returns True") - def test_is_not_executable(self): - with tempfile.NamedTemporaryFile() as tmpfile: - with self.assertRaises(RSConnectException): - which_python(tmpfile.name) - cur_dir = os.path.dirname(__file__) bqplot_dir = os.path.join(cur_dir, "testdata", "voila", "bqplot", "") @@ -1396,15 +1268,17 @@ def test_guess_deploy_dir(self): ], ) def test_create_voila_manifest_1(path, entrypoint): - environment = Environment( - contents="bqplot\n", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0", - python="3.8.12", - source="file", + environment = Environment.from_dict( + dict( + contents="bqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) ) if sys.platform == "win32": @@ -1469,15 +1343,17 @@ def test_create_voila_manifest_1(path, entrypoint): ], ) def test_create_voila_manifest_2(path, entrypoint): - environment = Environment( - contents="numpy\nipywidgets\nbqplot\n", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0", - python="3.8.12", - source="file", + environment = Environment.from_dict( + dict( + contents="numpy\nipywidgets\nbqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) ) if sys.platform == "win32": @@ -1515,15 +1391,17 @@ def test_create_voila_manifest_2(path, entrypoint): def test_create_voila_manifest_extra(): - environment = Environment( - contents="numpy\nipywidgets\nbqplot\n", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0.1", - python="3.8.12", - source="file", + environment = Environment.from_dict( + dict( + contents="numpy\nipywidgets\nbqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0.1", + python="3.8.12", + source="file", + ) ) if sys.platform == "win32": @@ -1599,15 +1477,17 @@ def test_create_voila_manifest_extra(): ], ) def test_create_voila_manifest_multi_notebook(path, entrypoint): - environment = Environment( - contents="bqplot\n", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0", - python="3.8.12", - source="file", + environment = Environment.from_dict( + dict( + contents="bqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) ) if sys.platform == "win32": @@ -1704,15 +1584,17 @@ def test_make_voila_bundle( path, entrypoint, ): - environment = Environment( - contents="bqplot", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0", - python="3.8.12", - source="file", + environment = Environment.from_dict( + dict( + contents="bqplot", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) ) if sys.platform == "win32": @@ -1811,15 +1693,17 @@ def test_make_voila_bundle_multi_notebook( path, entrypoint, ): - environment = Environment( - contents="bqplot", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0", - python="3.8.12", - source="file", + environment = Environment.from_dict( + dict( + contents="bqplot", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) ) if sys.platform == "win32": @@ -1900,15 +1784,17 @@ def test_make_voila_bundle_2( path, entrypoint, ): - environment = Environment( - contents="numpy\nipywidgets\nbqplot\n", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0", - python="3.8.12", - source="file", + environment = Environment.from_dict( + dict( + contents="numpy\nipywidgets\nbqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0", + python="3.8.12", + source="file", + ) ) if sys.platform == "win32": @@ -1964,15 +1850,17 @@ def test_make_voila_bundle_extra(): requirements_hash = "d51994456975ff487749acc247ae6d63" - environment = Environment( - contents="numpy\nipywidgets\nbqplot\n", - error=None, - filename="requirements.txt", - locale="en_US.UTF-8", - package_manager="pip", - pip="23.0.1", - python="3.8.12", - source="file", + environment = Environment.from_dict( + dict( + contents="numpy\nipywidgets\nbqplot\n", + error=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip="23.0.1", + python="3.8.12", + source="file", + ) ) ans = { "version": 1, @@ -2437,7 +2325,7 @@ def test_make_api_manifest_fastapi(): "prices.csv": {"checksum": "012afa636c426748177b38160135307a"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( fastapi_dir, ) manifest, _ = make_api_manifest( @@ -2469,7 +2357,7 @@ def test_make_api_bundle_fastapi(): "prices.csv": {"checksum": "012afa636c426748177b38160135307a"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( fastapi_dir, ) with make_api_bundle( @@ -2513,7 +2401,7 @@ def test_make_api_manifest_flask(): "prices.csv": {"checksum": "012afa636c426748177b38160135307a"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( flask_dir, ) manifest, _ = make_api_manifest( @@ -2545,7 +2433,7 @@ def test_make_api_bundle_flask(): "prices.csv": {"checksum": "012afa636c426748177b38160135307a"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( flask_dir, ) with make_api_bundle( @@ -2589,7 +2477,7 @@ def test_make_api_manifest_streamlit(): "data.csv": {"checksum": "aabd9d1210246c69403532a6a9d24286"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( streamlit_dir, ) manifest, _ = make_api_manifest( @@ -2620,7 +2508,7 @@ def test_make_api_bundle_streamlit(): "data.csv": {"checksum": "aabd9d1210246c69403532a6a9d24286"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( streamlit_dir, ) with make_api_bundle( @@ -2665,7 +2553,7 @@ def test_make_api_manifest_dash(): "prices.csv": {"checksum": "3efb0ed7ad93bede9dc88f7a81ad4153"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( dash_dir, ) manifest, _ = make_api_manifest( @@ -2697,7 +2585,7 @@ def test_make_api_bundle_dash(): "prices.csv": {"checksum": "3efb0ed7ad93bede9dc88f7a81ad4153"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( dash_dir, ) with make_api_bundle( @@ -2741,7 +2629,7 @@ def test_make_api_manifest_bokeh(): "data.csv": {"checksum": "aabd9d1210246c69403532a6a9d24286"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( bokeh_dir, ) manifest, _ = make_api_manifest( @@ -2774,7 +2662,7 @@ def test_make_api_bundle_bokeh(): }, } - environment = create_python_environment( + environment = Environment.create_python_environment( bokeh_dir, ) with make_api_bundle( @@ -2818,7 +2706,7 @@ def test_make_api_manifest_shiny(): "data.csv": {"checksum": "aabd9d1210246c69403532a6a9d24286"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( shiny_dir, ) manifest, _ = make_api_manifest( @@ -2850,7 +2738,7 @@ def test_make_api_bundle_shiny(): "data.csv": {"checksum": "aabd9d1210246c69403532a6a9d24286"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( shiny_dir, ) with make_api_bundle( @@ -2928,7 +2816,7 @@ def test_make_api_manifest_gradio(): "app.py": {"checksum": "22feec76e9c02ac6b5a34a083e2983b6"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( gradio_dir, ) manifest, _ = make_api_manifest( @@ -2958,7 +2846,7 @@ def test_make_api_bundle_gradio(): "app.py": {"checksum": "22feec76e9c02ac6b5a34a083e2983b6"}, }, } - environment = create_python_environment( + environment = Environment.create_python_environment( gradio_dir, ) with make_api_bundle( diff --git a/tests/test_environment.py b/tests/test_environment.py index c168153e..52df3c35 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,18 +1,21 @@ import re import sys +import os +import tempfile +import subprocess from unittest import TestCase -from rsconnect.environment import ( - MakeEnvironment, - detect_environment, - filter_pip_freeze_output, - get_default_locale, - get_python_version, -) +import rsconnect.environment +from rsconnect.exception import RSConnectException +from rsconnect.environment import Environment, which_python +from rsconnect.subprocesses.inspect_environment import get_python_version, get_default_locale, filter_pip_freeze_output from .utils import get_dir +import pytest + version_re = re.compile(r"\d+\.\d+(\.\d+)?") +TESTDATA = os.path.join(os.path.dirname(__file__), "testdata") class TestEnvironment(TestCase): @@ -33,26 +36,29 @@ def test_get_default_locale(self): self.assertEqual(get_default_locale(lambda: (None, None)), "") def test_file(self): - result = detect_environment(get_dir("pip1")) + result = Environment.create_python_environment(get_dir("pip1")) self.assertTrue(version_re.match(result.pip)) self.assertIsInstance(result.locale, str) self.assertIn(".", result.locale) - expected = MakeEnvironment( - contents="numpy\npandas\nmatplotlib\n", - filename="requirements.txt", - locale=result.locale, - package_manager="pip", - pip=result.pip, - python=self.python_version(), - source="file", + expected = Environment.from_dict( + dict( + contents="numpy\npandas\nmatplotlib\n", + filename="requirements.txt", + locale=result.locale, + package_manager="pip", + pip=result.pip, + python=self.python_version(), + source="file", + ), + python_interpreter=sys.executable, ) self.assertEqual(expected, result) def test_pip_freeze(self): - result = detect_environment(get_dir("pip2")) + result = Environment.create_python_environment(get_dir("pip2")) # these are the dependencies declared in our pyproject.toml self.assertIn("six", result.contents) @@ -63,14 +69,17 @@ def test_pip_freeze(self): self.assertIsInstance(result.locale, str) self.assertIn(".", result.locale) - expected = MakeEnvironment( - contents=result.contents, - filename="requirements.txt", - locale=result.locale, - package_manager="pip", - pip=result.pip, - python=self.python_version(), - source="pip_freeze", + expected = Environment.from_dict( + dict( + contents=result.contents, + filename="requirements.txt", + locale=result.locale, + package_manager="pip", + pip=result.pip, + python=self.python_version(), + source="pip_freeze", + ), + python_interpreter=sys.executable, ) self.assertEqual(expected, result) @@ -94,3 +103,170 @@ def test_filter_pip_freeze_output(self): expected = "numpy\npandas\nnot at beginning [notice]" self.assertEqual(filtered, expected) + + +class WhichPythonTestCase(TestCase): + def test_default(self): + self.assertEqual(which_python(), sys.executable) + + def test_none(self): + self.assertEqual(which_python(None), sys.executable) + + def test_sys(self): + self.assertEqual(which_python(sys.executable), sys.executable) + + def test_does_not_exist(self): + with tempfile.NamedTemporaryFile() as tmpfile: + name = tmpfile.name + with self.assertRaises(RSConnectException): + which_python(name) + + def test_is_directory(self): + with tempfile.TemporaryDirectory() as tmpdir: + with self.assertRaises(RSConnectException): + which_python(tmpdir) + + @pytest.mark.skipif(sys.platform.startswith("win"), reason="os.X_OK always returns True") + def test_is_not_executable(self): + with tempfile.NamedTemporaryFile() as tmpfile: + with self.assertRaises(RSConnectException): + which_python(tmpfile.name) + + +class TestPythonVersionRequirements: + def test_pyproject_toml(self): + env = Environment.create_python_environment(os.path.join(TESTDATA, "python-project", "using_pyproject")) + assert env.python_interpreter == sys.executable + assert env.python_version_requirement == ">=3.8" + + def test_python_version(self): + env = Environment.create_python_environment(os.path.join(TESTDATA, "python-project", "using_pyversion")) + assert env.python_interpreter == sys.executable + assert env.python_version_requirement == ">=3.8, <3.12" + + def test_all_of_them(self): + env = Environment.create_python_environment(os.path.join(TESTDATA, "python-project", "allofthem")) + assert env.python_interpreter == sys.executable + assert env.python_version_requirement == ">=3.8, <3.12" + + def test_missing(self): + env = Environment.create_python_environment(os.path.join(TESTDATA, "python-project", "empty")) + assert env.python_interpreter == sys.executable + assert env.python_version_requirement is None + + +def test_inspect_environment(): + environment = Environment._inspect_environment(sys.executable, get_dir("pip1")) + assert environment is not None + assert environment.python != "" + + +def test_inspect_environment_catches_type_error(): + with pytest.raises(RSConnectException) as exec_info: + Environment._inspect_environment(sys.executable, None) # type: ignore + + assert isinstance(exec_info.value, RSConnectException) + assert isinstance(exec_info.value.__cause__, TypeError) + + +@pytest.mark.parametrize( + ( + "file_name", + "python", + "force_generate", + "expected_python", + "expected_environment", + ), + [ + pytest.param( + "path/to/file.py", + sys.executable, + False, + sys.executable, + Environment.from_dict( + dict( + contents=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip=None, + python=None, + source="pip_freeze", + error=None, + ), + python_interpreter=sys.executable, + ), + id="basic", + ), + pytest.param( + "another/file.py", + os.path.basename(sys.executable), + False, + sys.executable, + Environment.from_dict( + dict( + contents=None, + filename="requirements.txt", + locale="en_US.UTF-8", + package_manager="pip", + pip=None, + python=None, + source="pip_freeze", + error=None, + ), + python_interpreter=sys.executable, + ), + id="which_python", + ), + pytest.param( + "will/the/files/never/stop.py", + "argh.py", + False, + "unused", + Environment.from_dict( + dict( + contents=None, + filename=None, + locale=None, + package_manager=None, + pip=None, + python=None, + source=None, + error="Could not even do things", + ) + ), + id="exploding", + ), + ], +) +def test_get_python_env_info( + monkeypatch, + file_name, + python, + force_generate, + expected_python, + expected_environment, +): + def fake_which_python(python, env=os.environ): + return expected_python + + def fake_inspect_environment( + python, + directory, + force_generate=False, + check_output=subprocess.check_output, + ): + return expected_environment + + monkeypatch.setattr(Environment, "_inspect_environment", fake_inspect_environment) + + monkeypatch.setattr(rsconnect.environment, "which_python", fake_which_python) + + if expected_environment.error is not None: + with pytest.raises(RSConnectException): + _ = Environment._get_python_env_info(file_name, python, force_generate=force_generate) + else: + environment = Environment._get_python_env_info(file_name, python, force_generate=force_generate) + + assert environment.python_interpreter == expected_python + assert environment == expected_environment diff --git a/tests/test_pyproject.py b/tests/test_pyproject.py index a245988e..eb5b3f28 100644 --- a/tests/test_pyproject.py +++ b/tests/test_pyproject.py @@ -7,6 +7,7 @@ parse_setupcfg_python_requires, parse_pyversion_python_requires, get_python_version_requirement_parser, + detect_python_version_requirement, ) import pytest @@ -50,15 +51,20 @@ def test_python_project_metadata_detect(project_dir, expected): ("pyproject.toml", parse_pyproject_python_requires), ("setup.cfg", parse_setupcfg_python_requires), (".python-version", parse_pyversion_python_requires), - ("invalid.txt", None), + ("invalid.txt", NotImplementedError("Unknown metadata file type: invalid.txt")), ], ids=["pyproject.toml", "setup.cfg", ".python-version", "invalid"], ) def test_get_python_version_requirement_parser(filename, expected_parser): """Test that given a metadata file name, the correct parser is returned.""" metadata_file = pathlib.Path(PROJECTS_DIRECTORY) / filename - parser = get_python_version_requirement_parser(metadata_file) - assert parser == expected_parser + if isinstance(expected_parser, Exception): + with pytest.raises(expected_parser.__class__) as excinfo: + parser = get_python_version_requirement_parser(metadata_file) + assert str(excinfo.value) == expected_parser.args[0] + else: + parser = get_python_version_requirement_parser(metadata_file) + assert parser == expected_parser @pytest.mark.parametrize( @@ -99,7 +105,7 @@ def test_pyprojecttoml_python_requires(project_dir, expected): ], ids=["option-exists", "option-missing"], ) -def test_setupcfg_python_requires(tmp_path, project_dir, expected): +def test_setupcfg_python_requires(project_dir, expected): """Test that the python_requires field is correctly parsed from setup.cfg. Both when the option exists or when it missing in the file. @@ -115,7 +121,7 @@ def test_setupcfg_python_requires(tmp_path, project_dir, expected): ], ids=["option-exists"], ) -def test_pyversion_python_requires(tmp_path, project_dir, expected): +def test_pyversion_python_requires(project_dir, expected): """Test that the python version is correctly parsed from .python-version. We do not test the case where the option is missing, as an empty .python-version file @@ -123,3 +129,16 @@ def test_pyversion_python_requires(tmp_path, project_dir, expected): """ versionfile = pathlib.Path(project_dir) / ".python-version" assert parse_pyversion_python_requires(versionfile) == expected + + +def test_detect_python_version_requirement(): + """Test that the python version requirement is correctly detected from the metadata files. + + Given that we already know from the other tests that the metadata files are correctly parsed, + this test primarily checks that when there are multiple metadata files, the one with the most specific + version requirement is used. + """ + project_dir = os.path.join(PROJECTS_DIRECTORY, "allofthem") + assert detect_python_version_requirement(project_dir) == ">=3.8, <3.12" + + assert detect_python_version_requirement(os.path.join(PROJECTS_DIRECTORY, "empty")) is None