diff --git a/py2pack/__init__.py b/py2pack/__init__.py index 37d39a8..2f1caf3 100755 --- a/py2pack/__init__.py +++ b/py2pack/__init__.py @@ -23,13 +23,12 @@ import json import os import pprint -import pwd import re import sys import warnings import jinja2 -import pypi_search.search + import requests from metaextract import utils as meta_utils @@ -39,7 +38,8 @@ parse_pyproject, get_setuptools_scripts, get_metadata) -from email import parser +from py2pack.parse import (fetch_local_data, fix_info, get_homepage, + get_user_name) def replace_string(output_string, replaces): @@ -67,33 +67,6 @@ def pypi_json(project, release=None): return pypimeta -def pypi_text_file(pkg_info_path): - with open(pkg_info_path, 'r') as pkg_info_file: - pkg_info_lines = parser.Parser().parse(pkg_info_file) - pkg_info_dict = {} - for key, value in pkg_info_lines.items(): - key = key.lower().replace('-', '_') - if key in {'classifiers', 'requires_dist', 'provides_extra'}: - val = pkg_info_dict.get(key) - if val is None: - val = [] - pkg_info_dict[key] = val - val.append(value) - else: - pkg_info_dict[key] = value - return {'info': pkg_info_dict, 'urls': []} - - -def pypi_json_file(file_path): - with open(file_path, 'r') as json_file: - js = json.load(json_file) - if 'info' not in js: - js = {'info': js} - if 'urls' not in js: - js['urls'] = [] - return js - - def _get_template_dirs(): """existing directories where to search for jinja2 templates. The order is important. The first found template from the first found dir wins!""" @@ -121,14 +94,8 @@ def list_packages(args=None): print(package) -def search(args): - print('searching for package {0}...'.format(args.name)) - for hit in pypi_search.search.find_packages(args.name): - print('found {0}-{1}'.format(hit['name'], hit['version'])) - - def show(args): - fetch_data(args) + fetch_data(args, trylocal=True) print('showing package {0}...'.format(args.fetched_data['info']['name'])) pprint.pprint(args.fetched_data) @@ -239,11 +206,7 @@ def _canonicalize_setup_data(data): data["console_scripts"] = list(dict.fromkeys(console_scripts)) # Standards says, that keys must be lowercase but not even PyPA adheres to it - homepage = (get_pyproject_table(data, 'project.urls.homepage') or - get_pyproject_table(data, 'project.urls.Homepage') or - get_pyproject_table(data, 'project.urls.Source') or - get_pyproject_table(data, 'project.urls.GitHub') or - get_pyproject_table(data, 'project.urls.Repository') or + homepage = (get_homepage(get_pyproject_table(data, 'project.urls')) or data.get('home_page', None)) if homepage: data['home_page'] = homepage @@ -363,7 +326,7 @@ def generate(args): warnings.warn("the '--run' switch is deprecated and a noop", DeprecationWarning) - fetch_local_data(args) + fetch_data(args, trylocal=True) if not args.template: args.template = file_template_list()[0] if not args.filename: @@ -373,7 +336,7 @@ def generate(args): durl = newest_download_url(args) source_url = data['source_url'] = (args.source_url or (durl and durl['url'])) data['year'] = datetime.datetime.now().year # set current year - data['user_name'] = pwd.getpwuid(os.getuid())[4] # set system user (packager) + data['user_name'] = get_user_name() # set system user (packager) data['summary_no_ending_dot'] = re.sub(r'(.*)\.', r'\g<1>', data.get('summary')) if data.get('summary') else "" # If package name supplied on command line differs in case from PyPI's one @@ -392,15 +355,24 @@ def generate(args): if tarball_file: break - if tarball_file: # get some more info from that + # localarchive argument was set by fetch_local_data method, and, if not empty, then exists in filesystem + localarchive = args.localarchive + + if tarball_file and not localarchive: # get some more info from that tarball_file = tarball_file[0] + else: + tarball_file = localarchive + + if not tarball_file: + tarball_file = args.name + '-' + args.version + '.tar.gz' + + if os.path.exists(tarball_file): _augment_data_from_tarball(args, tarball_file, data) else: warnings.warn("No tarball for {} in version {} found. Valuable " "information for the generation might be missing." "".format(args.name, args.version)) - tarball_file = args.name + '-' + args.version + '.zip' if not source_url: data['source_url'] = os.path.basename(tarball_file) @@ -409,6 +381,7 @@ def generate(args): env = _prepare_template_env(_get_template_dirs()) template = env.get_template(args.template) + data.update(args.options) # update data with custom options result = template.render(data).encode('utf-8') # render template and encode properly outfile = open(args.filename, 'wb') # write result to spec file try: @@ -417,31 +390,22 @@ def generate(args): outfile.close() -def fetch_local_data(args): - localfile = args.localfile - local = args.local - - if not localfile and local: - localfile = os.path.join(f'{args.name}.egg-info', 'PKG-INFO') - if os.path.isfile(localfile): - try: - data = pypi_json_file(localfile) - except json.decoder.JSONDecodeError: - data = pypi_text_file(localfile) - args.fetched_data = data - args.version = args.fetched_data['info']['version'] - return - fetch_data(args) - - -def fetch_data(args): - args.fetched_data = pypi_json(args.name, args.version) - urls = args.fetched_data.get('urls', []) - if len(urls) == 0: - print(f"unable to find a suitable release for {args.name}!") - sys.exit(1) - else: - args.version = args.fetched_data['info']['version'] # return current release number +def fetch_data(args, trylocal=False): + if trylocal: + trylocal = fetch_local_data(args) + if not trylocal: + args.fetched_data = pypi_json(args.name, args.version) + urls = args.fetched_data.get('urls', []) + if len(urls) == 0: + print(f"unable to find a suitable release for {args.name}!") + sys.exit(1) + data_info = args.fetched_data["info"] + fix_info(data_info) + # set version if absent + args.version = data_info['version'] + # set name if absent + if not args.name: + args.name = data_info['name'] def newest_download_url(args): @@ -480,13 +444,11 @@ def main(): parser_list = subparsers.add_parser('list', help='list all packages on PyPI') parser_list.set_defaults(func=list_packages) - parser_search = subparsers.add_parser('search', help='search for packages on PyPI') - parser_search.add_argument('name', help='package name (with optional version)') - parser_search.set_defaults(func=search) - parser_show = subparsers.add_parser('show', help='show metadata for package') parser_show.add_argument('name', help='package name') parser_show.add_argument('version', nargs='?', help='package version (optional)') + parser_show.add_argument('--local', action='store_true', help='show metadata from local package') + parser_show.add_argument('--localfile', default='', help='path to the local PKG-INFO or json metadata') parser_show.set_defaults(func=show) parser_fetch = subparsers.add_parser('fetch', help='download package source tarball from PyPI') @@ -500,6 +462,7 @@ def main(): parser_generate.add_argument('version', nargs='?', help='package version (optional)') parser_generate.add_argument('--source-url', default=None, help='source url') parser_generate.add_argument('--source-glob', help='source glob template') + parser_generate.add_argument('--setopt', action="append", help='An KEY=VALUE option (optional)', default=[]) parser_generate.add_argument('--local', action='store_true', help='build from local package') parser_generate.add_argument('--localfile', default='', help='path to the local PKG-INFO or json metadata') parser_generate.add_argument('-t', '--template', choices=file_template_list(), default='opensuse.spec', help='file template') @@ -526,6 +489,14 @@ def main(): if 'func' not in args: sys.exit(parser.print_help()) + if args.func == generate: + options = args.options = {} + for opt in args.setopt: + if '=' in opt: + key, value = opt.split('=', 1) + options[key] = value + else: + options[opt] = True args.func(args) diff --git a/py2pack/parse.py b/py2pack/parse.py new file mode 100644 index 0000000..f93f725 --- /dev/null +++ b/py2pack/parse.py @@ -0,0 +1,177 @@ +from email import parser +from importlib import metadata +import json +from io import TextIOWrapper +import tarfile +import os +import pwd +import zipfile +from os.path import join, basename, isfile +import re +from packaging.requirements import Requirement + + +def lowercase_dict(d): + ret = {} + for key, value in d.items(): + ret[str(key).lower()] = value + return ret + + +def get_homepage(urls): + try: + urls = lowercase_dict(urls) + for page in ('homepage', 'source', 'github', 'repository', 'gitlab'): + if page in urls: + return urls[page] + except Exception: + return None + + +def pypi_text_file(pkg_info_path): + # open PKG-INFO file and parse + pkg_info_file = open(pkg_info_path, 'r') + text = pypi_text_stream(pkg_info_file) + pkg_info_file.close() + return text + + +def pypi_text_stream(pkg_info_stream): + # parse PKG-INFO stream + pkg_info_lines = parser.Parser().parse(pkg_info_stream) + return pypi_text_items(pkg_info_lines.items()) + + +def pypi_text_metaextract(library): + # parse metadata from python module which already exists + pkg_info_lines = metadata.metadata(library) + return pypi_text_items(pkg_info_lines.items()) + + +def pypi_text_items(pkg_info_items): + # parse PKG-INFO lines + pkg_info_dict = {} + for key, value in pkg_info_items: + key = key.lower().replace('-', '_') + if key in {'requires_dist', 'provides_extra'}: + val = dict.setdefault(pkg_info_dict, key, []) + val.append(value) + elif key in {'classifier'}: + val = dict.setdefault(pkg_info_dict, key + 's', []) + val.append(value) + elif key in {'project_url'}: + key1, val = value.split(',', 1) + pkg_info_dict.setdefault(key + 's', {})[key1.strip()] = val.strip() + else: + pkg_info_dict[key] = value + return {'info': pkg_info_dict, 'urls': []} + + +def pypi_json_file(file_path): + # parse pypi json file + json_file = open(file_path, 'r') + js = pypi_json_stream(json_file) + json_file.close() + return js + + +def pypi_json_stream(json_stream): + # parse pypi json stream + js = json.load(json_stream) + if 'info' not in js: + js = {'info': js} + if 'urls' not in js: + js['urls'] = [] + return js + + +def _check_if_pypi_archive_file(path): + # check if archive is python source + return path.count('/') == 1 and basename(path) == 'PKG-INFO' + + +def pypi_archive_file(file_path): + # try to extract metadata from tar archive + if tarfile.is_tarfile(file_path): + with tarfile.open(file_path, 'r') as archive: + for member in archive.getmembers(): + if _check_if_pypi_archive_file(member.name): + return pypi_text_stream(TextIOWrapper(archive.extractfile(member), encoding='utf-8')) + # try to extract metadata from zip archive + elif zipfile.is_zipfile(file_path): + with zipfile.ZipFile(file_path, 'r') as archive: + for member in archive.namelist(): + if _check_if_pypi_archive_file(member): + return pypi_text_stream(TextIOWrapper(archive.open(member), encoding='utf-8')) + else: + raise TypeError("Can not extract '%s'. Not a tar or zip file" % file_path) + raise KeyError('PKG-INFO not found on archive ' + file_path) + + +def fetch_local_data(args): + # autodetect localfile name and type and parse + localfile = args.localfile + local = args.local + # set localarchive argument + args.localarchive = None + if not localfile and local: + try: + args.fetched_data = pypi_text_metaextract(args.name) + return True + except metadata.PackageNotFoundError: + localfile = join(f'{args.name}.egg-info', 'PKG-INFO') + if isfile(localfile): + try: + data = pypi_archive_file(localfile) + args.localarchive = localfile + except TypeError: + try: + data = pypi_json_file(localfile) + except json.decoder.JSONDecodeError: + data = pypi_text_file(localfile) + args.fetched_data = data + return True + return False + + +def fix_info(data_info): + # fix requires_dist + requires_dist = data_info.get("requires_dist", None) or [] + # fix provides_extra + provides_extra = data_info.get("provides_extra", None) or [] + extra_from_req = re.compile(r'''\bextra\s+==\s+["']([^"']+)["']''') + # add additional provides_extra from requires_dist + for required_dist in requires_dist: + req = Requirement(required_dist) + if found := extra_from_req.search(str(req.marker)): + provides_extra.append(found.group(1)) + # provides_extra must be unique list + provides_extra = list(sorted(set(provides_extra))) + # fix classifiers + classifiers = data_info.get("classifiers", None) or [] + # get project_urls dictionary + try: + urls = dict(data_info.get('project_urls', None)) + except TypeError: + urls = {} + # fix homepage + if 'home_page' not in data_info: + home_page = get_homepage(urls) or data_info.get('project_url', None) + if home_page: + data_info['home_page'] = home_page + # set fixed requires_dist + data_info["requires_dist"] = requires_dist + # set fixed provides_extra + data_info["provides_extra"] = provides_extra + # set fixed classifiers + data_info["classifiers"] = classifiers + # set fixed project_urls + data_info['project_urls'] = urls + + +# get username +def get_user_name(): + pwuid = pwd.getpwuid(os.getuid()) + gecos = pwuid.pw_gecos # or pwd.getpwuid(os.getuid())[4] + name = pwuid.pw_name # or pwd.getpwuid(os.getuid())[0] + return gecos or name diff --git a/py2pack/templates/fedora.spec b/py2pack/templates/fedora.spec index 611cbe8..07018bc 100644 --- a/py2pack/templates/fedora.spec +++ b/py2pack/templates/fedora.spec @@ -17,6 +17,8 @@ BuildRequires: python-devel %define python_module() python3dist(%1) %endif +{%- set archive_prefix = name|replace('-', '_')|replace('.', '_') %} +{%- set module_prefix = name|replace('-', '_')|replace('.', '/') %} {%- set build_requires_plus_pip = ((build_requires if build_requires and build_requires is not none else []) + ['pip']) %} {%- for req in build_requires_plus_pip |sort %} @@ -71,7 +73,7 @@ Summary: %{summary} %prep -%autosetup -p1 -n %{pypi_name}-%{version} +%autosetup -p1 -n {{ archive_prefix }}-%{version} %build %pyproject_wheel diff --git a/py2pack/templates/opensuse.spec b/py2pack/templates/opensuse.spec index 5b7cd16..f5f4985 100644 --- a/py2pack/templates/opensuse.spec +++ b/py2pack/templates/opensuse.spec @@ -32,6 +32,8 @@ License: {{ license }} URL: {{ home_page }} Source: {{ source_url|replace(version, '%{version}') }} BuildRequires: python-rpm-macros +{%- set archive_prefix = name|replace('-', '_')|replace('.', '_') %} +{%- set module_prefix = name|replace('-', '_')|replace('.', '/') %} {%- set build_requires_plus_pip = ((build_requires if build_requires and build_requires is not none else []) + ['pip']) %} {%- for req in build_requires_plus_pip |sort %} @@ -80,7 +82,7 @@ BuildArch: noarch {{ description }} %prep -%autosetup -p1 -n {{ name }}-%{version} +%autosetup -p1 -n {{ archive_prefix }}-%{version} %build {%- if has_ext_modules %} @@ -120,11 +122,11 @@ CHOOSE: %pytest OR %pyunittest -v OR CUSTOM %python_alternative %{_bindir}/{{ script }} {%- endfor %} {%- if has_ext_modules %} -%{python_sitearch}/{{name}} -%{python_sitearch}/{{name}}-%{version}.dist-info +%{python_sitearch}/{{ module_prefix }} +%{python_sitearch}/{{ archive_prefix }}-%{version}.dist-info {%- else %} -%{python_sitelib}/{{name}} -%{python_sitelib}/{{name}}-%{version}.dist-info +%{python_sitelib}/{{ module_prefix }} +%{python_sitelib}/{{ archive_prefix }}-%{version}.dist-info {%- endif %} {%- if data_files and data_files is not none %} {%- for dir, files in data_files %} diff --git a/pyproject.toml b/pyproject.toml index fe090ff..81f5e1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,6 @@ dependencies = [ "metaextract", "platformdirs", "packaging", - "pypi-search", "requests", "tomli; python_version < '3.11'", ] diff --git a/test/examples/py2pack-fedora-augmented.spec b/test/examples/py2pack-fedora-augmented.spec index edec47d..ab744b4 100644 --- a/test/examples/py2pack-fedora-augmented.spec +++ b/test/examples/py2pack-fedora-augmented.spec @@ -48,7 +48,7 @@ Summary: %{summary} %prep -%autosetup -p1 -n %{pypi_name}-%{version} +%autosetup -p1 -n py2pack-%{version} %build %pyproject_wheel diff --git a/test/examples/sampleproject-fedora-augmented.spec b/test/examples/sampleproject-fedora-augmented.spec index 5ccce9d..0801d9d 100644 --- a/test/examples/sampleproject-fedora-augmented.spec +++ b/test/examples/sampleproject-fedora-augmented.spec @@ -62,7 +62,7 @@ Summary: %{summary} %prep -%autosetup -p1 -n %{pypi_name}-%{version} +%autosetup -p1 -n sampleproject-%{version} %build %pyproject_wheel diff --git a/test/test_py2pack.py b/test/test_py2pack.py index 9726a6a..dcd65f5 100644 --- a/test/test_py2pack.py +++ b/test/test_py2pack.py @@ -34,6 +34,7 @@ class Args: source_glob = None local = False localfile = "" + options = {} self.args = Args() @@ -59,9 +60,6 @@ def test_replace_text(self): def test_list(self): py2pack.list_packages(self.args) - def test_search(self): - py2pack.search(self.args) - def test_show(self): py2pack.show(self.args) diff --git a/test/test_template.py b/test/test_template.py index f3c0177..0f3366a 100644 --- a/test/test_template.py +++ b/test/test_template.py @@ -19,7 +19,6 @@ import datetime import os import os.path -import pwd import sys import pytest @@ -37,11 +36,12 @@ class Args(object): source_glob = None local = False localfile = '' + options = {} compare_dir = os.path.join(os.path.dirname(__file__), 'examples') maxDiff = None -username = pwd.getpwuid(os.getuid())[4] +username = py2pack.get_user_name() @pytest.mark.parametrize('template, fetch_tarball',