From ca2fc64d3d8cdf473a86783d3634f3062b3e2f3e Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 12 Feb 2026 14:29:46 +0200 Subject: [PATCH 1/4] Add support for Reference Fix Commits improver Update the pipeline and fix the test Signed-off-by: ziad hany --- vulnerabilities/importers/__init__.py | 2 + .../v2_improvers/reference_collect_commits.py | 76 +++++++++++++ .../test_ref_collect_commits_v2.py | 105 ++++++++++++++++++ 3 files changed, 183 insertions(+) create mode 100644 vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py create mode 100644 vulnerabilities/tests/pipelines/v2_improvers/test_ref_collect_commits_v2.py diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 594021092..9cbf91320 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -82,6 +82,7 @@ from vulnerabilities.pipelines.v2_importers import ubuntu_osv_importer as ubuntu_osv_importer_v2 from vulnerabilities.pipelines.v2_importers import vulnrichment_importer as vulnrichment_importer_v2 from vulnerabilities.pipelines.v2_importers import xen_importer as xen_importer_v2 +from vulnerabilities.pipelines.v2_improvers import reference_collect_commits from vulnerabilities.utils import create_registry IMPORTERS_REGISTRY = create_registry( @@ -127,6 +128,7 @@ nginx_importer.NginxImporterPipeline, pysec_importer.PyPIImporterPipeline, fireeye_importer_v2.FireeyeImporterPipeline, + reference_collect_commits.CollectReferencesFixCommitsPipeline, apache_tomcat.ApacheTomcatImporter, postgresql.PostgreSQLImporter, debian.DebianImporter, diff --git a/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py b/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py new file mode 100644 index 000000000..80a266b5d --- /dev/null +++ b/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py @@ -0,0 +1,76 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from aboutcode.pipeline import LoopProgress +from packageurl.contrib.purl2url import purl2url +from packageurl.contrib.url2purl import url2purl + +from aboutcode.federated import get_core_purl +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageCommitPatch +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 +from vulnerabilities.pipes.advisory import VCS_URLS_SUPPORTED_TYPES +from vulnerabilities.tests.test_export import package +from vulnerabilities.utils import is_commit + + +class CollectReferencesFixCommitsPipeline(VulnerableCodeBaseImporterPipelineV2): + """ + Improver pipeline to scout References/Patch and create PackageCommitPatch entries. + """ + + pipeline_id = "collect_ref_fix_commits_v2" + license_expression = None + + @classmethod + def steps(cls): + return (cls.collect_and_store_fix_commits,) + + def get_vcs_commit(self, url): + """Extracts and VCS URL and commit hash from URL. + >> get_vcs_commit('https://github.com/aboutcode-org/vulnerablecode/commit/98e516011d6e096e25247b82fc5f196bbeecff10') + ('https://github.com/aboutcode-org/vulnerablecode', '98e516011d6e096e25247b82fc5f196bbeecff10') + >> get_vcs_commit('https://github.com/aboutcode-org/vulnerablecode/pull/1974') + None + """ + purl = url2purl(url) + if not purl or purl.type not in VCS_URLS_SUPPORTED_TYPES: + return None + + version = getattr(purl, "version", None) + if not version or not is_commit(version): + return None + + vcs_url = purl2url(get_core_purl(purl).to_string()) + return (vcs_url, version) if vcs_url else None + + def collect_and_store_fix_commits(self): + impacted_packages_advisories = ( + AdvisoryV2.objects.filter(impacted_packages__isnull=False) + .prefetch_related("references", "patches", "impacted_packages") + .distinct() + ) + + progress = LoopProgress( + total_iterations=impacted_packages_advisories.count(), logger=self.log + ) + for adv in progress.iter(impacted_packages_advisories.paginated(per_page=500)): + urls = {r.url for r in adv.references.all()} | {p.patch_url for p in adv.patches.all()} + impacted_packages = list(adv.impacted_packages.all()) + + for url in urls: + vcs_data = self.get_vcs_commit(url) + if not vcs_data: + continue + + vcs_url, commit_hash = vcs_data + package_commit_obj, _ = PackageCommitPatch.objects.get_or_create( + vcs_url=vcs_url, commit_hash=commit_hash + ) + package_commit_obj.fixed_impacted_packages.add(*impacted_packages) diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_ref_collect_commits_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_ref_collect_commits_v2.py new file mode 100644 index 000000000..51f3d9fe8 --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_ref_collect_commits_v2.py @@ -0,0 +1,105 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. + +from datetime import datetime + +import pytest + +from vulnerabilities.models import AdvisoryReference +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import PackageCommitPatch +from vulnerabilities.models import PackageV2 +from vulnerabilities.pipelines.v2_improvers.reference_collect_commits import ( + CollectReferencesFixCommitsPipeline, +) + + +@pytest.mark.django_db +def test_is_vcs_url_already_processed_true(): + advisory = AdvisoryV2.objects.create( + advisory_id="CVE-2025-9999", + datasource_id="test-ds", + avid="test-ds/CVE-2025-9999", + url="https://example.com/advisory/CVE-2025-9999", + unique_content_id="11111", + date_collected=datetime.now(), + ) + package = PackageV2.objects.create( + type="bar", + name="foo", + version="1.0", + ) + impact = ImpactedPackage.objects.create(advisory=advisory) + impact.affecting_packages.add(package) + package_commit_patch = PackageCommitPatch.objects.create( + vcs_url="https://github.com/user/repo/commit/6bd301819f8f69331a55ae2336c8b111fc933f3d", + commit_hash="6bd301819f8f69331a55ae2336c8b111fc933f3d", + ) + impact.fixed_by_package_commit_patches.add(package_commit_patch) + + +@pytest.mark.django_db +def test_collect_fix_commits_pipeline_creates_entry(): + advisory = AdvisoryV2.objects.create( + advisory_id="CVE-2025-1000", + datasource_id="test-ds", + avid="test-ds/CVE-2025-1000", + url="https://example.com/advisory/CVE-2025-1000", + unique_content_id="11111", + date_collected=datetime.now(), + ) + package = PackageV2.objects.create( + type="foo", + name="testpkg", + version="1.0", + ) + reference = AdvisoryReference.objects.create( + url="https://github.com/test/testpkg/commit/6bd301819f8f69331a55ae2336c8b111fc933f3d" + ) + impact = ImpactedPackage.objects.create(advisory=advisory) + impact.affecting_packages.add(package) + advisory.references.add(reference) + + pipeline = CollectReferencesFixCommitsPipeline() + pipeline.collect_and_store_fix_commits() + + package_commit_patch = PackageCommitPatch.objects.all() + + assert package_commit_patch.count() == 1 + fix = package_commit_patch.first() + assert fix.commit_hash == "6bd301819f8f69331a55ae2336c8b111fc933f3d" + assert fix.vcs_url == "https://github.com/test/testpkg" + + +@pytest.mark.django_db +def test_collect_fix_commits_pipeline_skips_non_commit_urls(): + advisory = AdvisoryV2.objects.create( + advisory_id="CVE-2025-2000", + datasource_id="test-ds", + avid="test-ds/CVE-2025-2000", + url="https://example.com/advisory/CVE-2025-2000", + unique_content_id="11111", + date_collected=datetime.now(), + ) + package = PackageV2.objects.create( + type="pypi", + name="otherpkg", + version="2.0", + ) + impact = ImpactedPackage.objects.create(advisory=advisory) + impact.affecting_packages.add(package) + + reference = AdvisoryReference.objects.create( + url="https://github.com/test/testpkg/issues/12" + ) # invalid reference 1 + advisory.references.add(reference) + + pipeline = CollectReferencesFixCommitsPipeline() + pipeline.collect_and_store_fix_commits() + assert PackageCommitPatch.objects.count() == 0 From 374de5e2d412fccf150ceee0b9eecdd021815a6b Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 20 Feb 2026 04:05:39 +0200 Subject: [PATCH 2/4] Improve the pipeline structure and reduce the number of database queries Signed-off-by: ziad hany --- .../pipelines/v2_improvers/reference_collect_commits.py | 3 +-- ...lect_commits_v2.py => test_reference_collect_commits_v2.py} | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) rename vulnerabilities/tests/pipelines/v2_improvers/{test_ref_collect_commits_v2.py => test_reference_collect_commits_v2.py} (98%) diff --git a/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py b/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py index 80a266b5d..0fb9af8cd 100644 --- a/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py +++ b/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py @@ -16,7 +16,6 @@ from vulnerabilities.models import PackageCommitPatch from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 from vulnerabilities.pipes.advisory import VCS_URLS_SUPPORTED_TYPES -from vulnerabilities.tests.test_export import package from vulnerabilities.utils import is_commit @@ -73,4 +72,4 @@ def collect_and_store_fix_commits(self): package_commit_obj, _ = PackageCommitPatch.objects.get_or_create( vcs_url=vcs_url, commit_hash=commit_hash ) - package_commit_obj.fixed_impacted_packages.add(*impacted_packages) + package_commit_obj.fixed_in_impacts.add(*impacted_packages) diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_ref_collect_commits_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_reference_collect_commits_v2.py similarity index 98% rename from vulnerabilities/tests/pipelines/v2_improvers/test_ref_collect_commits_v2.py rename to vulnerabilities/tests/pipelines/v2_improvers/test_reference_collect_commits_v2.py index 51f3d9fe8..2b65c07ba 100644 --- a/vulnerabilities/tests/pipelines/v2_improvers/test_ref_collect_commits_v2.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_reference_collect_commits_v2.py @@ -75,6 +75,7 @@ def test_collect_fix_commits_pipeline_creates_entry(): fix = package_commit_patch.first() assert fix.commit_hash == "6bd301819f8f69331a55ae2336c8b111fc933f3d" assert fix.vcs_url == "https://github.com/test/testpkg" + assert impact.fixed_by_package_commit_patches.count() == 1 @pytest.mark.django_db From 14d5b4faecc2afb7c2c3b36a645b8168d4b77ebf Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 20 Feb 2026 05:13:45 +0200 Subject: [PATCH 3/4] Delete the unnecessary license_expression var. Signed-off-by: ziad hany --- .../pipelines/v2_improvers/reference_collect_commits.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py b/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py index 0fb9af8cd..64c605cd8 100644 --- a/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py +++ b/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py @@ -25,7 +25,6 @@ class CollectReferencesFixCommitsPipeline(VulnerableCodeBaseImporterPipelineV2): """ pipeline_id = "collect_ref_fix_commits_v2" - license_expression = None @classmethod def steps(cls): From 5f066786a0ee7eef5c86cbf48e9321229954f7a8 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 20 Feb 2026 14:09:50 +0200 Subject: [PATCH 4/4] Drop unused test Signed-off-by: ziad hany --- .../test_reference_collect_commits_v2.py | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_reference_collect_commits_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_reference_collect_commits_v2.py index 2b65c07ba..b07b610c2 100644 --- a/vulnerabilities/tests/pipelines/v2_improvers/test_reference_collect_commits_v2.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_reference_collect_commits_v2.py @@ -20,30 +20,6 @@ ) -@pytest.mark.django_db -def test_is_vcs_url_already_processed_true(): - advisory = AdvisoryV2.objects.create( - advisory_id="CVE-2025-9999", - datasource_id="test-ds", - avid="test-ds/CVE-2025-9999", - url="https://example.com/advisory/CVE-2025-9999", - unique_content_id="11111", - date_collected=datetime.now(), - ) - package = PackageV2.objects.create( - type="bar", - name="foo", - version="1.0", - ) - impact = ImpactedPackage.objects.create(advisory=advisory) - impact.affecting_packages.add(package) - package_commit_patch = PackageCommitPatch.objects.create( - vcs_url="https://github.com/user/repo/commit/6bd301819f8f69331a55ae2336c8b111fc933f3d", - commit_hash="6bd301819f8f69331a55ae2336c8b111fc933f3d", - ) - impact.fixed_by_package_commit_patches.add(package_commit_patch) - - @pytest.mark.django_db def test_collect_fix_commits_pipeline_creates_entry(): advisory = AdvisoryV2.objects.create(