From 4f6c33fa7911879b01616b4fb151120e385bda38 Mon Sep 17 00:00:00 2001 From: git-hyagi <45576767+git-hyagi@users.noreply.github.com> Date: Tue, 11 Nov 2025 14:17:49 -0300 Subject: [PATCH] Implements the vulnerability report closes: #1012 --- CHANGES/1012.feature | 1 + docs/user/guides/_SUMMARY.md | 1 + docs/user/guides/vulnerability_report.md | 109 ++++++++++++++++++ pulp_python/app/tasks/__init__.py | 1 + pulp_python/app/tasks/vulnerability_report.py | 30 +++++ pulp_python/app/viewsets.py | 29 ++++- .../api/test_vulnerability_report.py | 48 ++++++++ pulp_python/tests/functional/constants.py | 5 + pyproject.toml | 2 +- 9 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 CHANGES/1012.feature create mode 100644 docs/user/guides/vulnerability_report.md create mode 100644 pulp_python/app/tasks/vulnerability_report.py create mode 100644 pulp_python/tests/functional/api/test_vulnerability_report.py diff --git a/CHANGES/1012.feature b/CHANGES/1012.feature new file mode 100644 index 00000000..19ca50a7 --- /dev/null +++ b/CHANGES/1012.feature @@ -0,0 +1 @@ +Added the new /scan endpoint to the RepositoryVersion viewset to generate vulnerability reports. diff --git a/docs/user/guides/_SUMMARY.md b/docs/user/guides/_SUMMARY.md index 0a315ce5..8f2baba2 100644 --- a/docs/user/guides/_SUMMARY.md +++ b/docs/user/guides/_SUMMARY.md @@ -2,3 +2,4 @@ * [Sync from Remote Repositories](sync.md) * [Upload and Manage Content](upload.md) * [Publish and Host Python Content](publish.md) +* [Vulnerability Report](vulnerability_report.md) diff --git a/docs/user/guides/vulnerability_report.md b/docs/user/guides/vulnerability_report.md new file mode 100644 index 00000000..234b7f8a --- /dev/null +++ b/docs/user/guides/vulnerability_report.md @@ -0,0 +1,109 @@ +# Vulnerability Report + +Pulp Python provides vulnerability scanning capabilities to help you identify known security +vulnerabilities in your Python packages. This feature integrates with the [Open Source Vulnerabilities (OSV)](https://osv.dev/) +database to scan Pulp `RepositoryVersions` for vulnerable packages. + +## Prerequisites + +Before generating the vulnerability report, ensure that: + +1. You have a Python repository with [synced or uploaded content](site:pulp_python/docs/user/guides/sync/) +2. Pulp has connectivity to the [OSV API](https://api.osv.dev/v1/query) + +## Generating a vulnerability report + +To scan a `RepositoryVersion` for vulnerabilities, you need to pass the name of the repository and +optionally the version: + +```bash +pulp vulnerability-report create --repository my-repo --version 1 +``` + +## Understanding Scan Results + +After a scan completes, vulnerability information is available in two places: + +### 1. Repository Version Level + +The `RepositoryVersion` includes a `vuln_report` field that references a vulnerability report +containing all vulnerabilities found in that version: + +```bash +pulp python repository version show --repository my-repo +``` + +The response includes: + +```json +{ + "pulp_href": "/pulp/api/v3/repositories/python/python/.../versions/1/", + "number": 1, + ... + "vuln_report": "/pulp/api/v3/vuln-reports/..." +} +``` + +### 2. Content Level + +Individual Python package content units also include vulnerability report references: + +```bash +pulp python content list +``` + +Each package in the response includes: + +```json +{ + "pulp_href": "/pulp/api/v3/content/python/packages/.../", + "name": "Django", + ... + "vuln_report": "/pulp/api/v3/vuln-reports/...", + ... +} +``` + +### Viewing Vulnerability Details + +To view the actual vulnerability data, retrieve the vulnerability report: + +```bash +# Get vulnerability report details +pulp vulnerability-report show --href ${VULN_REPORT_HREF} +``` + +The report contains detailed information about each vulnerability, including: + +- **CVE identifiers**: Common Vulnerabilities and Exposures identifiers +- **Affected versions**: Which package versions are vulnerable +- **Fixed versions**: Which versions contain fixes +- **References**: Links to advisories and patches +- **Repository and Content**: Pulp `RepositoryVersion` and `Content` impacted + +## Example Workflow + +Here's a complete example of scanning a repository for vulnerabilities: + +```bash +# 1. Create a repository +pulp python repository create --name security-scan-repo + +# 2. Create a remote pointing to PyPI +pulp python remote create \ + --name pypi-remote \ + --url https://pypi.org/ \ + --includes '["django==5.2.1"]' + +# 3. Sync the repository +pulp python repository sync \ + --name security-scan-repo \ + --remote pypi-remote + +# 4. Scan for vulnerabilities +pulp vulnerability-report create --repository security-scan-repo + +# 5. View the vulnerability report +VULN_REPORT=$(pulp python repository version show --repository security-scan-repo | jq -r '.vuln_report') +pulp vulnerability-report show --href $VULN_REPORT +``` diff --git a/pulp_python/app/tasks/__init__.py b/pulp_python/app/tasks/__init__.py index b40ec245..6c2949eb 100644 --- a/pulp_python/app/tasks/__init__.py +++ b/pulp_python/app/tasks/__init__.py @@ -6,3 +6,4 @@ from .repair import repair # noqa:F401 from .sync import sync # noqa:F401 from .upload import upload, upload_group # noqa:F401 +from .vulnerability_report import get_repo_version_content # noqa:F401 diff --git a/pulp_python/app/tasks/vulnerability_report.py b/pulp_python/app/tasks/vulnerability_report.py new file mode 100644 index 00000000..8d5352ca --- /dev/null +++ b/pulp_python/app/tasks/vulnerability_report.py @@ -0,0 +1,30 @@ +from pulpcore.plugin.models import RepositoryVersion +from pulpcore.plugin.sync import sync_to_async_iterable + +from pulp_python.app.models import PythonPackageContent + + +async def get_repo_version_content(repo_version_pk: str): + """ + Retrieve Python package content from a repository version for vulnerability scanning. + """ + repo_version = await RepositoryVersion.objects.aget(pk=repo_version_pk) + content_units = PythonPackageContent.objects.filter(pk__in=repo_version.content).only( + "name", "version" + ) + ecosystem = "PyPI" + async for content in sync_to_async_iterable(content_units): + repo_content_osv_data = _build_osv_data(content.name, ecosystem, content.version) + repo_content_osv_data["repo_version"] = repo_version + repo_content_osv_data["content"] = content + yield repo_content_osv_data + + +def _build_osv_data(name, ecosystem, version=None): + """ + Build an OSV data structure for vulnerability queries. + """ + osv_data = {"package": {"name": name, "ecosystem": ecosystem}} + if version: + osv_data["version"] = version + return osv_data diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 8aabdfd1..e303ca44 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -13,7 +13,7 @@ AsyncOperationResponseSerializer, RepositorySyncURLSerializer, ) -from pulpcore.plugin.tasking import dispatch +from pulpcore.plugin.tasking import check_content, dispatch from pulp_python.app import models as python_models from pulp_python.app import serializers as python_serializers @@ -206,9 +206,36 @@ class PythonRepositoryVersionViewSet(core_viewsets.RepositoryVersionViewSet): "has_repository_model_or_domain_or_obj_perms:python.view_pythonrepository", ], }, + { + "action": ["scan"], + "principal": "authenticated", + "effect": "allow", + "condition": [ + "has_repository_model_or_domain_or_obj_perms:python.view_pythonrepository", + ], + }, ], } + @extend_schema( + summary="Generate vulnerability report", responses={202: AsyncOperationResponseSerializer} + ) + @action(detail=True, methods=["post"], serializer_class=None) + def scan(self, request, repository_pk, **kwargs): + """ + Scan a repository version for vulnerabilities. + """ + repository_version = self.get_object() + func = ( + f"{tasks.get_repo_version_content.__module__}.{tasks.get_repo_version_content.__name__}" + ) + task = dispatch( + check_content, + shared_resources=[repository_version.repository], + args=[func, [repository_version.pk]], + ) + return core_viewsets.OperationPostponedResponse(task, request) + class PythonDistributionViewSet(core_viewsets.DistributionViewSet, core_viewsets.RolesMixin): """ diff --git a/pulp_python/tests/functional/api/test_vulnerability_report.py b/pulp_python/tests/functional/api/test_vulnerability_report.py new file mode 100644 index 00000000..1630c147 --- /dev/null +++ b/pulp_python/tests/functional/api/test_vulnerability_report.py @@ -0,0 +1,48 @@ +import pytest + +from pulp_python.tests.functional.constants import ( + PYPI_URL, + VULNERABILITY_REPORT_TEST_PACKAGE_NAME, + VULNERABILITY_REPORT_TEST_PACKAGES, +) + + +@pytest.mark.parallel +def test_vulnerability_report( + pulpcore_bindings, python_bindings, python_repo, python_remote_factory, monitor_task +): + + # Sync the test repository. + remote = python_remote_factory(url=PYPI_URL, includes=VULNERABILITY_REPORT_TEST_PACKAGES) + sync_data = dict(remote=remote.pulp_href) + response = python_bindings.RepositoriesPythonApi.sync(python_repo.pulp_href, sync_data) + monitor_task(response.task) + + # get repo latest version + repo = python_bindings.RepositoriesPythonApi.read(python_repo.pulp_href) + latest_version_href = repo.latest_version_href + + # scan + response = python_bindings.RepositoriesPythonVersionsApi.scan( + python_python_repository_version_href=latest_version_href + ) + monitor_task(response.task) + + # checks + vulns_list = pulpcore_bindings.VulnReportApi.list() + assert len(vulns_list.results) > 0 + for results in vulns_list.results: + assert len(results.vulns) > 0 + for vuln in results.vulns: + assert VULNERABILITY_REPORT_TEST_PACKAGE_NAME.lower() in ( + affected["package"]["name"] for affected in vuln["affected"] + ) + + repo_version = python_bindings.RepositoriesPythonVersionsApi.read(latest_version_href) + assert repo_version.vuln_report is not None + + python_packages = python_bindings.ContentPackagesApi.list( + name=VULNERABILITY_REPORT_TEST_PACKAGE_NAME, repository_version=latest_version_href + ) + for content in python_packages.results: + assert content.vuln_report is not None diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index d65e62f6..cadbbc12 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -345,3 +345,8 @@ "releases": {"0.1": SHELF_0DOT1_RELEASE}, "urls": SHELF_0DOT1_RELEASE, } + +VULNERABILITY_REPORT_TEST_PACKAGE_NAME = "Django" +VULNERABILITY_REPORT_TEST_PACKAGES = [ + "django==5.2.1", +] diff --git a/pyproject.toml b/pyproject.toml index ef1935c7..41217018 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers=[ ] requires-python = ">=3.11" dependencies = [ - "pulpcore>=3.85.0,<3.100", + "pulpcore>=3.85.3,<3.100", "pkginfo>=1.12.0,<1.13.0", "bandersnatch>=6.6.0,<6.7", "pypi-simple>=1.5.0,<2.0",