diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 594021092..9cbf91320 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -82,6 +82,7 @@ from vulnerabilities.pipelines.v2_importers import ubuntu_osv_importer as ubuntu_osv_importer_v2 from vulnerabilities.pipelines.v2_importers import vulnrichment_importer as vulnrichment_importer_v2 from vulnerabilities.pipelines.v2_importers import xen_importer as xen_importer_v2 +from vulnerabilities.pipelines.v2_improvers import reference_collect_commits from vulnerabilities.utils import create_registry IMPORTERS_REGISTRY = create_registry( @@ -127,6 +128,7 @@ nginx_importer.NginxImporterPipeline, pysec_importer.PyPIImporterPipeline, fireeye_importer_v2.FireeyeImporterPipeline, + reference_collect_commits.CollectReferencesFixCommitsPipeline, apache_tomcat.ApacheTomcatImporter, postgresql.PostgreSQLImporter, debian.DebianImporter, diff --git a/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py b/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py new file mode 100644 index 000000000..64c605cd8 --- /dev/null +++ b/vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py @@ -0,0 +1,74 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from aboutcode.pipeline import LoopProgress +from packageurl.contrib.purl2url import purl2url +from packageurl.contrib.url2purl import url2purl + +from aboutcode.federated import get_core_purl +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageCommitPatch +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 +from vulnerabilities.pipes.advisory import VCS_URLS_SUPPORTED_TYPES +from vulnerabilities.utils import is_commit + + +class CollectReferencesFixCommitsPipeline(VulnerableCodeBaseImporterPipelineV2): + """ + Improver pipeline to scout References/Patch and create PackageCommitPatch entries. + """ + + pipeline_id = "collect_ref_fix_commits_v2" + + @classmethod + def steps(cls): + return (cls.collect_and_store_fix_commits,) + + def get_vcs_commit(self, url): + """Extracts and VCS URL and commit hash from URL. + >> get_vcs_commit('https://github.com/aboutcode-org/vulnerablecode/commit/98e516011d6e096e25247b82fc5f196bbeecff10') + ('https://github.com/aboutcode-org/vulnerablecode', '98e516011d6e096e25247b82fc5f196bbeecff10') + >> get_vcs_commit('https://github.com/aboutcode-org/vulnerablecode/pull/1974') + None + """ + purl = url2purl(url) + if not purl or purl.type not in VCS_URLS_SUPPORTED_TYPES: + return None + + version = getattr(purl, "version", None) + if not version or not is_commit(version): + return None + + vcs_url = purl2url(get_core_purl(purl).to_string()) + return (vcs_url, version) if vcs_url else None + + def collect_and_store_fix_commits(self): + impacted_packages_advisories = ( + AdvisoryV2.objects.filter(impacted_packages__isnull=False) + .prefetch_related("references", "patches", "impacted_packages") + .distinct() + ) + + progress = LoopProgress( + total_iterations=impacted_packages_advisories.count(), logger=self.log + ) + for adv in progress.iter(impacted_packages_advisories.paginated(per_page=500)): + urls = {r.url for r in adv.references.all()} | {p.patch_url for p in adv.patches.all()} + impacted_packages = list(adv.impacted_packages.all()) + + for url in urls: + vcs_data = self.get_vcs_commit(url) + if not vcs_data: + continue + + vcs_url, commit_hash = vcs_data + package_commit_obj, _ = PackageCommitPatch.objects.get_or_create( + vcs_url=vcs_url, commit_hash=commit_hash + ) + package_commit_obj.fixed_in_impacts.add(*impacted_packages) diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_reference_collect_commits_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_reference_collect_commits_v2.py new file mode 100644 index 000000000..b07b610c2 --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_reference_collect_commits_v2.py @@ -0,0 +1,82 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. + +from datetime import datetime + +import pytest + +from vulnerabilities.models import AdvisoryReference +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import PackageCommitPatch +from vulnerabilities.models import PackageV2 +from vulnerabilities.pipelines.v2_improvers.reference_collect_commits import ( + CollectReferencesFixCommitsPipeline, +) + + +@pytest.mark.django_db +def test_collect_fix_commits_pipeline_creates_entry(): + advisory = AdvisoryV2.objects.create( + advisory_id="CVE-2025-1000", + datasource_id="test-ds", + avid="test-ds/CVE-2025-1000", + url="https://example.com/advisory/CVE-2025-1000", + unique_content_id="11111", + date_collected=datetime.now(), + ) + package = PackageV2.objects.create( + type="foo", + name="testpkg", + version="1.0", + ) + reference = AdvisoryReference.objects.create( + url="https://github.com/test/testpkg/commit/6bd301819f8f69331a55ae2336c8b111fc933f3d" + ) + impact = ImpactedPackage.objects.create(advisory=advisory) + impact.affecting_packages.add(package) + advisory.references.add(reference) + + pipeline = CollectReferencesFixCommitsPipeline() + pipeline.collect_and_store_fix_commits() + + package_commit_patch = PackageCommitPatch.objects.all() + + assert package_commit_patch.count() == 1 + fix = package_commit_patch.first() + assert fix.commit_hash == "6bd301819f8f69331a55ae2336c8b111fc933f3d" + assert fix.vcs_url == "https://github.com/test/testpkg" + assert impact.fixed_by_package_commit_patches.count() == 1 + + +@pytest.mark.django_db +def test_collect_fix_commits_pipeline_skips_non_commit_urls(): + advisory = AdvisoryV2.objects.create( + advisory_id="CVE-2025-2000", + datasource_id="test-ds", + avid="test-ds/CVE-2025-2000", + url="https://example.com/advisory/CVE-2025-2000", + unique_content_id="11111", + date_collected=datetime.now(), + ) + package = PackageV2.objects.create( + type="pypi", + name="otherpkg", + version="2.0", + ) + impact = ImpactedPackage.objects.create(advisory=advisory) + impact.affecting_packages.add(package) + + reference = AdvisoryReference.objects.create( + url="https://github.com/test/testpkg/issues/12" + ) # invalid reference 1 + advisory.references.add(reference) + + pipeline = CollectReferencesFixCommitsPipeline() + pipeline.collect_and_store_fix_commits() + assert PackageCommitPatch.objects.count() == 0