From dfe84c5856ca468abeaca589a4c2cdd4d4110d10 Mon Sep 17 00:00:00 2001 From: u Date: Sat, 17 May 2025 13:54:09 +0300 Subject: [PATCH] Fix metadata, system user, argument parser, add option to generate from archive file, add setopt option --- py2pack/__init__.py | 174 +++++++++++++++++++----------- py2pack/utils.py | 238 +++++++++++++++++++++++++++++++++++++++++- test/test_py2pack.py | 1 + test/test_template.py | 4 +- 4 files changed, 353 insertions(+), 64 deletions(-) diff --git a/py2pack/__init__.py b/py2pack/__init__.py index 37d39a8..65aef0a 100755 --- a/py2pack/__init__.py +++ b/py2pack/__init__.py @@ -37,9 +37,70 @@ from py2pack import version as py2pack_version from py2pack.utils import (_get_archive_filelist, get_pyproject_table, parse_pyproject, get_setuptools_scripts, - get_metadata) + get_metadata, CaselessDict, pypi_text_file, + pypi_json_file, pypi_archive_file, + parse_vars) -from email import parser +from packaging.requirements import Requirement + + +def get_user_name(): + return pwd.getpwuid(os.getuid()).pw_name + + +def _get_homepage(urls): + try: + urls = CaselessDict(urls) + for page in ('Homepage', 'Source', 'GitHub', 'Repository', 'GitLab'): + if page in urls: + return urls[page] + except Exception: + pass + return None + + +def fix_data(args): + # fix data fetched from pypi.org + data = args.fetched_data + # get info + data_info = data["info"] + # set version if absent + args.version = data_info['version'] + # set name if absent + if not args.name: + args.name = data_info['name'] + # fix requires_dist + requires_dist = data_info.get("requires_dist", None) or [] + # fix provides_extra + provides_extra = data_info.get("provides_extra", None) or [] + extra_from_req = re.compile(r'''\bextra\s+==\s+["']([^"']+)["']''') + # add additional provides_extra from requires_dist + for required_dist in requires_dist: + req = Requirement(required_dist) + if found := extra_from_req.search(str(req.marker)): + provides_extra.append(found.group(1)) + # provides_extra must be unique list + provides_extra = list(sorted(set(provides_extra))) + # fix classifiers + classifiers = data_info.get("classifiers", None) or [] + # get project_urls dictionary + try: + urls = dict(data_info.get('project_urls', None)) + except TypeError: + urls = {} + # fix homepage + if 'home_page' not in data_info: + home_page = _get_homepage(urls) or data_info.get('project_url', None) + if home_page: + data_info['home_page'] = home_page + # set fixed requires_dist + data_info["requires_dist"] = requires_dist + # set fixed provides_extra + data_info["provides_extra"] = provides_extra + # set fixed classifiers + data_info["classifiers"] = classifiers + # set fixed project_urls + data_info['project_urls'] = urls def replace_string(output_string, replaces): @@ -67,33 +128,6 @@ def pypi_json(project, release=None): return pypimeta -def pypi_text_file(pkg_info_path): - with open(pkg_info_path, 'r') as pkg_info_file: - pkg_info_lines = parser.Parser().parse(pkg_info_file) - pkg_info_dict = {} - for key, value in pkg_info_lines.items(): - key = key.lower().replace('-', '_') - if key in {'classifiers', 'requires_dist', 'provides_extra'}: - val = pkg_info_dict.get(key) - if val is None: - val = [] - pkg_info_dict[key] = val - val.append(value) - else: - pkg_info_dict[key] = value - return {'info': pkg_info_dict, 'urls': []} - - -def pypi_json_file(file_path): - with open(file_path, 'r') as json_file: - js = json.load(json_file) - if 'info' not in js: - js = {'info': js} - if 'urls' not in js: - js['urls'] = [] - return js - - def _get_template_dirs(): """existing directories where to search for jinja2 templates. The order is important. The first found template from the first found dir wins!""" @@ -128,7 +162,7 @@ def search(args): def show(args): - fetch_data(args) + fetch_data(args, trylocal=True) print('showing package {0}...'.format(args.fetched_data['info']['name'])) pprint.pprint(args.fetched_data) @@ -140,9 +174,10 @@ def fetch(args): print("unable to find a source release for {0}!".format(args.name)) sys.exit(1) print('downloading package {0}-{1}...'.format(args.name, args.version)) - print('from {0}'.format(url['url'])) + download_url = url['download_url'] + print('from {0}'.format(download_url)) - with requests.get(url['url']) as r: + with requests.get(download_url) as r: with open(url['filename'], 'wb') as f: f.write(r.content) @@ -239,11 +274,7 @@ def _canonicalize_setup_data(data): data["console_scripts"] = list(dict.fromkeys(console_scripts)) # Standards says, that keys must be lowercase but not even PyPA adheres to it - homepage = (get_pyproject_table(data, 'project.urls.homepage') or - get_pyproject_table(data, 'project.urls.Homepage') or - get_pyproject_table(data, 'project.urls.Source') or - get_pyproject_table(data, 'project.urls.GitHub') or - get_pyproject_table(data, 'project.urls.Repository') or + homepage = (_get_homepage(get_pyproject_table(data, 'project.urls')) or data.get('home_page', None)) if homepage: data['home_page'] = homepage @@ -363,7 +394,7 @@ def generate(args): warnings.warn("the '--run' switch is deprecated and a noop", DeprecationWarning) - fetch_local_data(args) + fetch_data(args, trylocal=True) if not args.template: args.template = file_template_list()[0] if not args.filename: @@ -371,9 +402,9 @@ def generate(args): print('generating spec file for {0}...'.format(args.name)) data = args.fetched_data['info'] durl = newest_download_url(args) - source_url = data['source_url'] = (args.source_url or (durl and durl['url'])) + source_url = data['source_url'] = (args.source_url or (durl and durl['download_url'])) data['year'] = datetime.datetime.now().year # set current year - data['user_name'] = pwd.getpwuid(os.getuid())[4] # set system user (packager) + data['user_name'] = get_user_name() # set system user (packager) data['summary_no_ending_dot'] = re.sub(r'(.*)\.', r'\g<1>', data.get('summary')) if data.get('summary') else "" # If package name supplied on command line differs in case from PyPI's one @@ -392,8 +423,18 @@ def generate(args): if tarball_file: break - if tarball_file: # get some more info from that + # localarchive argument was set by fetch_local_data method, and, if not empty, then exists in filesystem + localarchive = args.localarchive + + if tarball_file and not localarchive: # get some more info from that tarball_file = tarball_file[0] + else: + tarball_file = localarchive + + if not tarball_file: + tarball_file = args.name + '-' + args.version + '.tar.gz' + + if os.path.exists(tarball_file): _augment_data_from_tarball(args, tarball_file, data) else: @@ -409,6 +450,7 @@ def generate(args): env = _prepare_template_env(_get_template_dirs()) template = env.get_template(args.template) + data.update(parse_vars(args.setopt)) result = template.render(data).encode('utf-8') # render template and encode properly outfile = open(args.filename, 'wb') # write result to spec file try: @@ -420,28 +462,35 @@ def generate(args): def fetch_local_data(args): localfile = args.localfile local = args.local - + # set localarchive argument + args.localarchive = None if not localfile and local: localfile = os.path.join(f'{args.name}.egg-info', 'PKG-INFO') if os.path.isfile(localfile): try: - data = pypi_json_file(localfile) - except json.decoder.JSONDecodeError: - data = pypi_text_file(localfile) + data = pypi_archive_file(localfile) + args.localarchive = localfile + except TypeError: + try: + data = pypi_json_file(localfile) + except json.decoder.JSONDecodeError: + data = pypi_text_file(localfile) args.fetched_data = data - args.version = args.fetched_data['info']['version'] - return - fetch_data(args) + return True + return False -def fetch_data(args): - args.fetched_data = pypi_json(args.name, args.version) - urls = args.fetched_data.get('urls', []) - if len(urls) == 0: - print(f"unable to find a suitable release for {args.name}!") - sys.exit(1) - else: - args.version = args.fetched_data['info']['version'] # return current release number +def fetch_data(args, trylocal=False): + if trylocal: + trylocal = fetch_local_data(args) + + if not trylocal: + args.fetched_data = pypi_json(args.name, args.version) + urls = args.fetched_data.get('urls', []) + if len(urls) == 0: + print(f"unable to find a suitable release for {args.name}!") + sys.exit(1) + fix_data(args) def newest_download_url(args): @@ -452,14 +501,14 @@ def newest_download_url(args): if not hasattr(args, "fetched_data"): return {} for release in args.fetched_data['urls']: # Check download URLs in releases - if release['packagetype'] == 'sdist': # Found the source URL we care for - release['url'] = _get_source_url(args.name, release['filename']) + if release['packagetype'] == 'sdist' and not release.get('download_url'): # Found the source URL we care for + release['download_url'] = _get_source_url(args.name, release['filename']) return release # No PyPI tarball release, let's see if an upstream download URL is provided: data = args.fetched_data['info'] - if 'download_url' in data and data['download_url']: - url = data['download_url'] - return {'url': url, + url = data.get('download_url') + if url: + return {'download_url': url, 'filename': os.path.basename(url)} return {} # We're all out of bubblegum @@ -487,6 +536,8 @@ def main(): parser_show = subparsers.add_parser('show', help='show metadata for package') parser_show.add_argument('name', help='package name') parser_show.add_argument('version', nargs='?', help='package version (optional)') + parser_show.add_argument('--local', action='store_true', help='show metadata from local package') + parser_show.add_argument('--localfile', default='', help='path to the local PKG-INFO or json metadata') parser_show.set_defaults(func=show) parser_fetch = subparsers.add_parser('fetch', help='download package source tarball from PyPI') @@ -500,6 +551,7 @@ def main(): parser_generate.add_argument('version', nargs='?', help='package version (optional)') parser_generate.add_argument('--source-url', default=None, help='source url') parser_generate.add_argument('--source-glob', help='source glob template') + parser_generate.add_argument('--setopt', action="append", help='An KEY=VALUE option (optional)') parser_generate.add_argument('--local', action='store_true', help='build from local package') parser_generate.add_argument('--localfile', default='', help='path to the local PKG-INFO or json metadata') parser_generate.add_argument('-t', '--template', choices=file_template_list(), default='opensuse.spec', help='file template') diff --git a/py2pack/utils.py b/py2pack/utils.py index 3f24e3e..e4fd563 100644 --- a/py2pack/utils.py +++ b/py2pack/utils.py @@ -31,8 +31,27 @@ import tarfile import zipfile - +from sys import getsizeof, maxsize +from typing import Any +from typing import Collection +from typing import Dict +from typing import Hashable +from typing import ItemsView +from typing import Iterator +from typing import KeysView +from typing import Mapping +from typing import Optional +from typing import Tuple +from typing import Type +from typing import TypeVar +from typing import Union +from typing import ValuesView +from typing import cast from backports.entry_points_selectable import EntryPoint, EntryPoints +from email import parser +from importlib import metadata +import json +from io import StringIO def _get_archive_filelist(filename): @@ -195,3 +214,220 @@ def get_metadata(filename): data['install_requires'] = mdata.get_all('Requires-Dist') return data + + +K = TypeVar("K", bound=Hashable) +V = TypeVar("V") + + +class CaselessDict(Mapping[K, V]): + """A dictionary with case-insensitive string getters.""" + + def __init__(self, *args: Union[Mapping[K, V], Collection[Tuple[K, V]]], **kwargs: Mapping[K, V]) -> None: + self._map: Dict[K, V] = dict(*args, **kwargs) + self._caseless: Dict[str, str] = {k.lower(): k for k, v in self._map.items() if isinstance(k, str)} + self._hash: int = -1 + + def __contains__(self, key: object) -> bool: + """Test if is contained within this mapping.""" + return ( + self._caseless[key.lower()] in self._map + if (isinstance(key, str) and key.lower() in self._caseless) + else key in self._map + ) + + def __copy__(self) -> "CaselessDict": + """Return a shallow copy of this mapping.""" + return type(self)(self.items()) + + def __eq__(self, other: Any) -> bool: + """Test if is equal to this class instance.""" + ret = isinstance(other, type(self)) + ret = ret and hasattr(other, "__hash__") + ret = ret and hash(self) == hash(other) + ret = ret and hasattr(other, "__len__") + ret = ret and len(self) == len(other) + ret = ret and all([key in other and other[key] == value for key, value in self.items()]) + return ret + + def __getitem__(self, key: K) -> Any: + """Return a value indexed with .""" + if isinstance(key, str) and key.lower() in self._caseless: + return self._map[cast(K, self._caseless[key.lower()])] + else: + return self._map[key] + + def __hash__(self) -> int: + """Return a hash of this dictionary using all key-value pairs.""" + if self._hash == -1 and self: + current: int = 0 + for key, value in self.items(): + if isinstance(key, str): + current ^= hash((key.lower(), value)) + else: + current ^= hash((key, value)) + current ^= maxsize + self._hash = current + return self._hash + + def __iter__(self) -> Iterator[K]: + """Return an iterator over the keys.""" + return iter(self._map.keys()) + + def __len__(self) -> int: + """Return the length of the mapping.""" + return len(self._map) + + def __ne__(self, other: Any) -> bool: + return not self == other + + def __nonzero__(self) -> bool: + """Test if this mapping is of non-zero length.""" + return bool(self._map) + + def __reduce__(self) -> Tuple[Type["CaselessDict"], Tuple[List[Tuple[K, V]]]]: + """Return a recipe for pickling.""" + return type(self), (list(self.items()),) + + def __repr__(self) -> str: + """Return a representation of this class instance.""" + return f"{self.__class__.__qualname__}({repr(self._map)})" + + def __sizeof__(self) -> int: + """Return the size of this class instance.""" + return getsizeof(self._map) + + def __str__(self) -> str: + """Return a string representation of this class.""" + return self.__repr__() + + @classmethod + def fromkeys(cls, keys: Collection[K], default: V) -> "CaselessDict": + """Build a mapping from a set of keys with a default value.""" + return cls([(key, default) for key in keys]) + + def copy(self, mapping: Optional[Dict[K, V]] = None) -> "CaselessDict": + """Return a shallow copy of this mapping.""" + overrides: Dict[K, V] = {} + if mapping is not None: + for k, v in mapping.items(): + if isinstance(k, str) and k.lower() in self._caseless: + overrides[cast(K, self._caseless[k.lower()])] = v + else: + overrides[k] = v + return type(self)((list(self.items())) + list(overrides.items())) + + def get(self, key: K, default: Optional[Any] = None) -> Union[Any, V]: + """Return a value indexed with but if that key is not present, return .""" + if isinstance(key, str) and key.lower() in self._caseless: + caseless_key: K = cast(K, self._caseless[key.lower()]) + return self._map.get(caseless_key, default) + else: + return self._map.get(key, default) + + def items(self) -> ItemsView[K, V]: + """Return this mapping as a list of paired key-values.""" + return self._map.items() + + def keys(self) -> KeysView[K]: + """Return the keys in insertion order.""" + return self._map.keys() + + def updated(self, key: K, value: V) -> "CaselessDict": + """Return a shallow copy of this mapping with a key-value pair.""" + return self.copy({key: value}) + + def values(self) -> ValuesView[V]: + """Return the values in insertion order.""" + return self._map.values() + + +def pypi_text_file(pkg_info_path): + pkg_info_file = open(pkg_info_path, 'r') + text = pypi_text_stream(pkg_info_file) + pkg_info_file.close() + return text + + +def pypi_text_stream(pkg_info_stream): + pkg_info_lines = parser.Parser().parse(pkg_info_stream) + return pypi_text_items(pkg_info_lines.items()) + + +def pypi_text_metaextract(library): + pkg_info_lines = metadata.metadata(library) + return pypi_text_items(pkg_info_lines.items()) + + +def pypi_text_items(pkg_info_items): + pkg_info_dict = {} + for key, value in pkg_info_items: + key = key.lower().replace('-', '_') + if key in {'requires_dist', 'provides_extra'}: + val = dict.setdefault(pkg_info_dict, key, []) + val.append(value) + elif key in {'classifier'}: + val = dict.setdefault(pkg_info_dict, key + 's', []) + val.append(value) + elif key in {'project_url'}: + key1, val = value.split(',', 1) + pkg_info_dict.setdefault(key + 's', {})[key1.strip()] = val.strip() + else: + pkg_info_dict[key] = value + return {'info': pkg_info_dict, 'urls': []} + + +def pypi_json_file(file_path): + json_file = open(file_path, 'r') + js = pypi_json_stream(json_file) + json_file.close() + return js + + +def pypi_json_stream(json_stream): + js = json.load(json_stream) + if 'info' not in js: + js = {'info': js} + if 'urls' not in js: + js['urls'] = [] + return js + + +def _check_if_pypi_archive_file(path): + return path.count('/') == 1 and os.path.basename(path) == 'PKG-INFO' + + +def pypi_archive_file(file_path): + if tarfile.is_tarfile(file_path): + with tarfile.open(file_path, 'r') as archive: + for member in archive.getmembers(): + if _check_if_pypi_archive_file(member.name): + return pypi_text_stream(StringIO(archive.extractfile(member).read().decode())) + elif zipfile.is_zipfile(file_path): + with zipfile.ZipFile(file_path, 'r') as archive: + for member in archive.namelist(): + if _check_if_pypi_archive_file(member): + return pypi_text_stream(StringIO(archive.open(member).read().decode())) + else: + raise TypeError("Can not extract '%s'. Not a tar or zip file" % file_path) + raise KeyError('PKG-INFO not found on archive ' + file_path) + + +def parse_vars(items): + """ + Parse a series of key-value pairs and return a dictionary and + a success boolean for whether each item was successfully parsed. + """ + if not items: + return {} + + d = {} + for item in items: + index = item.find('=') + if index > 0: + split_string = (item[:index], item[(index + 1):]) + d[split_string[0]] = split_string[1] + else: + d[item] = True + + return d diff --git a/test/test_py2pack.py b/test/test_py2pack.py index 9726a6a..3c4b11f 100644 --- a/test/test_py2pack.py +++ b/test/test_py2pack.py @@ -28,6 +28,7 @@ class Py2packTestCase(unittest.TestCase): def setUp(self): class Args: + setopt = ['py2pack_test_option=test', 'py2pack_test_flag'] name = "py2pack" version = "0.4.4" source_url = None diff --git a/test/test_template.py b/test/test_template.py index 3d3d487..ef8c563 100644 --- a/test/test_template.py +++ b/test/test_template.py @@ -19,7 +19,6 @@ import datetime import os import os.path -import pwd import sys import pytest @@ -29,6 +28,7 @@ class Args(object): run = False + setopt = ['py2pack_test_option=test', 'py2pack_test_flag'] template = '' filename = '' name = '' @@ -41,7 +41,7 @@ class Args(object): compare_dir = os.path.join(os.path.dirname(__file__), 'examples') maxDiff = None -username = pwd.getpwuid(os.getuid())[4] +username = py2pack.get_user_name() @pytest.mark.parametrize('template, fetch_tarball',