Add archiving option, fix exceptions, caused by multiline content in rpm spec files, remove shebangs, fix missing provides_extra field

py2pack/__init__.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
 # Copyright (c) 2013, Sascha Peilicke <sascha@peilicke.de>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -40,6 +37,22 @@
                            get_metadata)
 
 from email import parser
+import tarfile
+import zipfile
+from packaging.requirements import Requirement
+from os.path import basename
+from io import StringIO
+
+
+try:
+    import distro
+    DEFAULT_TEMPLATE = {
+        'fedora': 'fedora.spec',
+        'debian': 'opensuse.dsc',
+        'mageia': 'mageia.spec'
+    }.get(distro.id(), 'opensuse.spec')
+except ModuleNotFoundError:
+    DEFAULT_TEMPLATE = 'opensuse.spec'
 
 
 def replace_string(output_string, replaces):
@@ -68,8 +81,14 @@ def pypi_json(project, release=None):
 
 
 def pypi_text_file(pkg_info_path):
-    with open(pkg_info_path, 'r') as pkg_info_file:
-        pkg_info_lines = parser.Parser().parse(pkg_info_file)
+    pkg_info_file = open(pkg_info_path, 'r')
+    text = pypi_text_stream(pkg_info_file)
+    pkg_info_file.close()
+    return text
+
+
+def pypi_text_stream(pkg_info_stream):
+    pkg_info_lines = parser.Parser().parse(pkg_info_stream)
     pkg_info_dict = {}
     for key, value in pkg_info_lines.items():
         key = key.lower().replace('-', '_')
@@ -85,15 +104,44 @@ def pypi_text_file(pkg_info_path):
 
 
 def pypi_json_file(file_path):
-    with open(file_path, 'r') as json_file:
-        js = json.load(json_file)
+    json_file = open(file_path, 'r')
+    js = pypi_json_stream(json_file)
+    json_file.close()
+    return js
+
+
+def pypi_json_stream(json_stream):
+    js = json.load(json_stream)
     if 'info' not in js:
         js = {'info': js}
     if 'urls' not in js:
         js['urls'] = []
     return js
 
 
+def pypi_archive_file_tar(file_path):
+    with tarfile.open(file_path, 'r') as archive:
+        for member in archive.getmembers():
+            if os.path.basename(member.name) == 'PKG-INFO':
+                return pypi_text_stream(StringIO(archive.extractfile(member).read().decode()))
+    raise KeyError('PKG-INFO not found on archive '+file_path)
+
+
+def pypi_archive_file_zip(file_path):
+    with zipfile.ZipFile(file_path, 'r') as archive:
+        for member in archive.namelist():
+            if os.path.basename(member) == 'PKG-INFO':
+                return pypi_text_stream(StringIO(archive.open(member).read().decode()))
+    raise KeyError('PKG-INFO not found on archive '+file_path)
+
+
+def pypi_archive_file(file_path):
+    try:
+        return pypi_archive_file_tar(file_path)
+    except tarfile.ReadError:
+        return pypi_archive_file_zip(file_path)
+
+
 def _get_template_dirs():
     """existing directories where to search for jinja2 templates. The order
     is important. The first found template from the first found dir wins!"""
@@ -111,6 +159,22 @@ def _get_template_dirs():
     ])
 
 
+def fix_data(data):
+    extra_from_req = re.compile(r'''\bextra\s+==\s+["']([^"']+)["']''')
+    extras = []
+    data_info = data["info"]
+    requires_dist = data_info.get("requires_dist", []) or []
+    provides_extra = data_info.get("provides_extra", []) or []
+    for required_dist in requires_dist:
+        req = Requirement(required_dist)
+        if found := extra_from_req.search(str(req.marker)):
+            extras.append(found.group(1))
+    provides_extra = list(sorted(set([*extras, *provides_extra])))
+    data_info["requires_dist"] = requires_dist
+    data_info["provides_extra"] = provides_extra
+    data_info["classifiers"] = (data_info.get("classifiers", []) or [])
+
+
 def list_packages(args=None):
     """query the "Simple API" of PYPI for all packages and print them."""
     print('listing all PyPI packages...')
@@ -407,6 +471,12 @@ def generate(args):
 
     _normalize_license(data)
 
+    for field in ['summary', 'license', 'home_page', 'source_url', 'summary_no_ending_dot']:
+        if field not in data:
+            continue
+        # remove line breaks to avoid multiline rpm spec file
+        data[field + '_singleline'] = str(data[field]).replace('\n', ' ')
+
     env = _prepare_template_env(_get_template_dirs())
     template = env.get_template(args.template)
     result = template.render(data).encode('utf-8')                          # render template and encode properly
@@ -425,23 +495,28 @@ def fetch_local_data(args):
         localfile = os.path.join(f'{args.name}.egg-info', 'PKG-INFO')
     if os.path.isfile(localfile):
         try:
-            data = pypi_json_file(localfile)
-        except json.decoder.JSONDecodeError:
-            data = pypi_text_file(localfile)
+            data = pypi_archive_file(localfile)
+        except Exception:
+            try:
+                data = pypi_json_file(localfile)
+            except json.decoder.JSONDecodeError:
+                data = pypi_text_file(localfile)
         args.fetched_data = data
         args.version = args.fetched_data['info']['version']
-        return
-    fetch_data(args)
+        fix_data(data)
+    else:
+        fetch_data(args)
 
 
 def fetch_data(args):
-    args.fetched_data = pypi_json(args.name, args.version)
-    urls = args.fetched_data.get('urls', [])
+    data = args.fetched_data = pypi_json(args.name, args.version)
+    urls = data.get('urls', [])
     if len(urls) == 0:
         print(f"unable to find a suitable release for {args.name}!")
         sys.exit(1)
     else:
         args.version = args.fetched_data['info']['version']                 # return current release number
+    fix_data(data)
 
 
 def newest_download_url(args):
@@ -502,7 +577,7 @@ def main():
     parser_generate.add_argument('--source-glob', help='source glob template')
     parser_generate.add_argument('--local', action='store_true', help='build from local package')
     parser_generate.add_argument('--localfile', default='', help='path to the local PKG-INFO or json metadata')
-    parser_generate.add_argument('-t', '--template', choices=file_template_list(), default='opensuse.spec', help='file template')
+    parser_generate.add_argument('-t', '--template', choices=file_template_list(), default=DEFAULT_TEMPLATE, help='file template')
     parser_generate.add_argument('-f', '--filename', help='spec filename (optional)')
     # TODO (toabctl): remove this is a later release
     parser_generate.add_argument(

py2pack/templates/fedora.spec

@@ -3,61 +3,31 @@
 Name:           python-%{pypi_name}
 Version:        {{ version }}
 Release:        %autorelease
-Summary:        {{ summary }}
+# Fill in the actual package summary to submit package to Fedora
+Summary:        {{ summary_singleline }}
 
 # Check if the automatically generated License and its spelling is correct for Fedora
 # https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/
-License:        {{ license }}
-URL:            {{ home_page }}
-Source:         {{ source_url|replace(version, '%{version}') }}
+License:        {{ license_singleline }}
+URL:            {{ home_page_singleline }}
+Source:         {{ source_url_singleline|replace(version, '%{version}') }}
 
 BuildRequires:  pyproject-rpm-macros
-BuildRequires:  python-devel
-%if %{undefined python_module}
-%define python_module() python3dist(%1)
-%endif
+BuildRequires:  python3-devel
 
-{%- set build_requires_plus_pip = ((build_requires if build_requires and build_requires is not none else []) +
-                                   ['pip']) %}
-{%- for req in build_requires_plus_pip |sort %}
-BuildRequires:  %{python_module {{ req }}}
-{%- endfor %}
-{%- if (install_requires and install_requires is not none) or (tests_require and tests_require is not none) %}
-# SECTION test requirements
-%if %{with test}
-{%- if install_requires and install_requires is not none %}
-{%- for req in install_requires|reject("in",build_requires)|sort %}
-BuildRequires:  %{python_module {{ req }}}
-{%- endfor %}
-{%- endif %}
-{%- if tests_require and tests_require is not none %}
-{%- for req in tests_require|sort|reject("in",build_requires|sort) %}
-BuildRequires:  %{python_module {{ req }}}
-{%- endfor %}
-{%- endif %}
-%endif
-# /SECTION
-{%- endif %}
 {%- if source_url.endswith('.zip') %}
 BuildRequires:  unzip
 {%- endif %}
 BuildRequires:  fdupes
-{%- if install_requires and install_requires is not none %}
-{%- for req in install_requires|sort %}
-Requires:       %{python_module {{ req }}}
-{%- endfor %}
-{%- endif %}
-{%- if extras_require and extras_require is not none %}
-{%- for reqlist in extras_require.values() %}
-{%- for req in reqlist %}
-Suggests:       %{python_module {{ req }}}
-{%- endfor %}
-{%- endfor %}
-{%- endif %}
 {%- if not has_ext_modules %}
 BuildArch:      noarch
 {%- endif %}
 
+{%- if provides_extra and provides_extra is not none %}
+{%- set provides_extra_comma_separated_list = ','.join(provides_extra)  %}
+{%- set have_provides_extra = 1 %}
+{%- endif %}
+
 # Fill in the actual package description to submit package to Fedora
 %global _description %{expand:
 {{ description }}}
@@ -69,22 +39,27 @@ Summary:        %{summary}
 
 %description -n %{python_name} %_description
 
+{%- if have_provides_extra %}
+# For official Fedora packages, review which extras should be actually packaged
+# See: https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/#Extras
+%pyproject_extras_subpkg -n %{python_name} {{ provides_extra_comma_separated_list }}
+{%- endif %}
 
 %prep
 %autosetup -p1 -n %{pypi_name}-%{version}
 
+
+%generate_buildrequires
+# Keep only those extras which you actually want to package or use during tests
+%pyproject_buildrequires {% if have_provides_extra %}-x {{ provides_extra_comma_separated_list }}{% endif %}
+
+
 %build
 %pyproject_wheel
 
 
 %install
 %pyproject_install
-{%- set scripts_or_console_scripts = (
-            (scripts|map('basename')|list if scripts and scripts is not none else []) +
-            (console_scripts if console_scripts and console_scripts is not none else [])) %}
-#{%- for script in scripts_or_console_scripts %}
-#%python_clone -a %{buildroot}%{_bindir}/{{ script }}
-#{%- endfor %}
 # For official Fedora packages, including files with '*' +auto is not allowed
 # Replace it with a list of relevant Python modules/globs and list extra files in %%files
 %pyproject_save_files '*' +auto
@@ -93,6 +68,7 @@ Summary:        %{summary}
 {%- if testsuite or test_suite %}
 %if %{with test}
 %check
+%pyproject_check_import
 {%- if has_ext_modules %}
 %pytest_arch
 {%- else %}

py2pack/templates/mageia.spec

@@ -3,11 +3,11 @@
 Name:           python-%{mod_name}
 Version:        {{ version }}
 Release:        %mkrel 1
-Url:            {{ home_page }}
-Summary:        {{ summary }}
-License:        {{ license }}
+Url:            {{ home_page_singleline }}
+Summary:        {{ summary_singleline }}
+License:        {{ license_singleline }}
 Group:          Development/Python
-Source:         {{ source_url|replace(version, '%{version}') }}
+Source:         {{ source_url_singleline|replace(version, '%{version}') }}
 BuildRoot:      %{_tmppath}/%{name}-%{version}-buildroot
 BuildRequires:  python-devel
 {%- for req in requires %}

py2pack/templates/opensuse-legacy.spec

@@ -18,10 +18,10 @@
 Name:           python-{{ name }}
 Version:        {{ version }}
 Release:        0
-Summary:        {{ summary_no_ending_dot|default(summary, true) }}
-License:        {{ license }}
-URL:            {{ home_page }}
-Source:         {{ source_url|replace(version, '%{version}') }}
+Summary:        {{ summary_no_ending_dot_singleline|default(summary, true) }}
+License:        {{ license_singleline }}
+URL:            {{ home_page_singleline }}
+Source:         {{ source_url_singleline|replace(version, '%{version}') }}
 BuildRequires:  python-setuptools
 {%- if install_requires and install_requires is not none %}
 {%- for req in install_requires|sort %}

py2pack/templates/opensuse.spec

@@ -19,10 +19,10 @@
 Name:           python-{{ name }}
 Version:        {{ version }}
 Release:        0
-Summary:        {{ summary_no_ending_dot|default(summary, true) }}
-License:        {{ license }}
-URL:            {{ home_page }}
-Source:         {{ source_url|replace(version, '%{version}') }}
+Summary:        {{ summary_no_ending_dot_singleline|default(summary, true) }}
+License:        {{ license_singleline }}
+URL:            {{ home_page_singleline }}
+Source:         {{ source_url_singleline|replace(version, '%{version}') }}
 BuildRequires:  python-rpm-macros
 {%- set build_requires_plus_pip = ((build_requires if build_requires and build_requires is not none else []) +
                                    ['pip']) %}

python-setuptools-scm.conf

@@ -0,0 +1,15 @@
+license = "MIT"
+archful = false
+summary = "the blessed package to manage your versions by scm tags"
+pypi_version = "8.1.0"
+pypi_name = "setuptools-scm"
+python_name = "python-setuptools-scm"
+url = "https://pypi.org/project/setuptools-scm/"
+source = "PyPI"
+archive_name = "setuptools_scm-8.1.0.tar.gz"
+extras = [
+    "docs",
+    "rich",
+    "test",
+]
+license_files_present = true