From 0f248c9b513d12046d2cea0b10640aaffa596dc5 Mon Sep 17 00:00:00 2001 From: Nachiket Date: Wed, 18 Nov 2026 09:33:44 +0530 Subject: [PATCH] Split long Bugzilla buglist references into per-bug references Some advisories reference a buglist.cgi URL that lists dozens of bug ids and is longer than the 1024 characters allowed for a reference URL, so inserting the advisory references failed with a DataError. When such a URL is too long, store one show_bug.cgi reference per bug instead. Reference: https://github.com/aboutcode-org/vulnerablecode/issues/2500 Signed-off-by: Nachiket --- vulnerabilities/pipes/advisory.py | 39 ++++++++++++++++++++ vulnerabilities/tests/pipes/test_advisory.py | 30 +++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index fec72e66f..6a1e8998d 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import dataclasses import hashlib import logging from datetime import datetime @@ -15,6 +16,9 @@ from typing import Callable from typing import List from typing import Union +from urllib.parse import parse_qs +from urllib.parse import urlsplit +from urllib.parse import urlunsplit from django.core.exceptions import ValidationError from django.db import transaction @@ -67,7 +71,42 @@ def get_or_create_advisory_aliases(aliases: List[str]) -> List[AdvisoryAlias]: return list(AdvisoryAlias.objects.filter(alias__in=aliases)) +def split_long_bugzilla_buglist(reference): + """ + Return a list of references for ``reference``. A Bugzilla buglist URL longer + than the reference URL field is split into one show_bug.cgi reference per bug, + and any other reference is returned unchanged. + """ + max_length = AdvisoryReference._meta.get_field("url").max_length + if len(reference.url) <= max_length: + return [reference] + + scheme, netloc, path, query, _ = urlsplit(reference.url) + if not path.endswith("/buglist.cgi"): + return [reference] + + bug_ids = [ + bug_id.strip() + for value in parse_qs(query).get("bug_id", []) + for bug_id in value.split(",") + if bug_id.strip() + ] + if not bug_ids: + return [reference] + + show_bug_path = path[: -len("buglist.cgi")] + "show_bug.cgi" + return [ + dataclasses.replace( + reference, + reference_id=bug_id, + url=urlunsplit((scheme, netloc, show_bug_path, f"id={bug_id}", "")), + ) + for bug_id in bug_ids + ] + + def get_or_create_advisory_references(references: List) -> List[AdvisoryReference]: + references = [ref for reference in references for ref in split_long_bugzilla_buglist(reference)] reference_urls = [ref.url for ref in references] existing = AdvisoryReference.objects.filter(url__in=reference_urls) existing_urls = {r.url for r in existing} diff --git a/vulnerabilities/tests/pipes/test_advisory.py b/vulnerabilities/tests/pipes/test_advisory.py index ad98e915a..732294525 100644 --- a/vulnerabilities/tests/pipes/test_advisory.py +++ b/vulnerabilities/tests/pipes/test_advisory.py @@ -23,6 +23,7 @@ from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import PackageCommitPatchData from vulnerabilities.importer import Reference +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.models import AdvisoryAlias from vulnerabilities.models import AdvisoryReference from vulnerabilities.models import AdvisorySeverity @@ -235,6 +236,35 @@ def test_get_or_create_advisory_references(advisory_references): assert ref.url in [r.url for r in advisory_references] +@pytest.mark.django_db +def test_get_or_create_advisory_references_splits_long_bugzilla_buglist(): + bug_ids = [str(1935995 + i) for i in range(120)] + buglist_url = "https://bugzilla.mozilla.org/buglist.cgi?bug_id=" + "%2C".join(bug_ids) + assert len(buglist_url) > AdvisoryReference._meta.get_field("url").max_length + nvd_url = "https://nvd.nist.gov/vuln/detail/CVE-2026-6785" + references = [ + ReferenceV2(url=nvd_url), + ReferenceV2(url=buglist_url, reference_type=AdvisoryReference.BUG), + ] + + refs = get_or_create_advisory_references(references) + + expected_bug_urls = { + "https://bugzilla.mozilla.org/show_bug.cgi?id=" + bug_id for bug_id in bug_ids + } + assert {ref.url for ref in refs} == expected_bug_urls | {nvd_url} + bug_refs = [ref for ref in refs if ref.url != nvd_url] + assert all(ref.reference_type == AdvisoryReference.BUG for ref in bug_refs) + assert {ref.reference_id for ref in bug_refs} == set(bug_ids) + + +@pytest.mark.django_db +def test_get_or_create_advisory_references_keeps_short_bugzilla_buglist(): + buglist_url = "https://bugzilla.mozilla.org/buglist.cgi?bug_id=1935995%2C1999158" + refs = get_or_create_advisory_references([ReferenceV2(url=buglist_url)]) + assert [ref.url for ref in refs] == [buglist_url] + + @pytest.mark.django_db def test_get_or_create_advisory_severities(advisory_severities): sevs = get_or_create_advisory_severities(advisory_severities)