From f8d6397d4ee34c8e27818ce45b50bfa8501f6e93 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 3 Nov 2026 14:59:04 +0530 Subject: [PATCH] Add golang importer pipeline Signed-off-by: Tushar Goel --- vulnerabilities/importers/__init__.py | 4 + .../v2_importers/golang_vulndb_importer.py | 362 ++++++++++++++++++ vulnerabilities/pipes/fetchcode_utils.py | 240 +++++++++++- vulnerabilities/pipes/osv_v2.py | 63 ++- vulnerabilities/pipes/univers_utils.py | 27 +- .../test_golang_vulndb_importer_v2.py | 297 ++++++++++++++ .../tests/pipes/test_fetchcode_utils.py | 103 +++++ vulnerabilities/tests/pipes/test_osv_v2.py | 11 + 8 files changed, 1093 insertions(+), 14 deletions(-) create mode 100644 vulnerabilities/pipelines/v2_importers/golang_vulndb_importer.py create mode 100644 vulnerabilities/tests/pipelines/v2_importers/test_golang_vulndb_importer_v2.py create mode 100644 vulnerabilities/tests/pipes/test_fetchcode_utils.py diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index a33e6d714..a128811b2 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -26,6 +26,9 @@ from vulnerabilities.pipelines.v2_importers import github_osv_importer as github_osv_importer_v2 from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2 from vulnerabilities.pipelines.v2_importers import glibc_importer as glibc_importer_v2 +from vulnerabilities.pipelines.v2_importers import ( + golang_vulndb_importer as golang_vulndb_importer_v2, +) from vulnerabilities.pipelines.v2_importers import istio_importer as istio_importer_v2 from vulnerabilities.pipelines.v2_importers import linux_kernel_importer as linux_kernel_importer_v2 from vulnerabilities.pipelines.v2_importers import mattermost_importer as mattermost_importer_v2 @@ -64,6 +67,7 @@ apache_httpd_v2.ApacheHTTPDImporterPipeline, pypa_importer_v2.PyPaImporterPipeline, gitlab_importer_v2.GitLabImporterPipeline, + golang_vulndb_importer_v2.GolangVulndbImporterPipeline, pysec_importer_v2.PyPIImporterPipeline, xen_importer_v2.XenImporterPipeline, curl_importer_v2.CurlImporterPipeline, diff --git a/vulnerabilities/pipelines/v2_importers/golang_vulndb_importer.py b/vulnerabilities/pipelines/v2_importers/golang_vulndb_importer.py new file mode 100644 index 000000000..b4d0b2b56 --- /dev/null +++ b/vulnerabilities/pipelines/v2_importers/golang_vulndb_importer.py @@ -0,0 +1,362 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import re +from pathlib import Path +from typing import Iterable +from typing import List +from typing import Optional +from typing import Tuple + +import dateparser +import saneyaml +from fetchcode.vcs import fetch_via_vcs +from packageurl import PackageURL +from univers.version_constraint import VersionConstraint +from univers.version_range import GolangVersionRange +from univers.versions import InvalidVersion + +from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 +from vulnerabilities.pipes.osv_v2 import parse_advisory_data_v3 +from vulnerabilities.utils import build_description +from vulnerabilities.utils import get_advisory_url +from vulnerabilities.utils import load_json + +CWE_ID_RE = re.compile(r"CWE-(\d+)", re.IGNORECASE) + + +class GolangVulndbImporterPipeline(VulnerableCodeBaseImporterPipelineV2): + """ + Go Vulnerability Database importer pipeline. + + Clone https://github.com/golang/vulndb and import: + - OSV JSON from data/osv + - YAML reports from data/reports + - CVE 5.0 records from data/cve/v5 + """ + + pipeline_id = "golang_vulndb_importer_v2" + datasource_id = "golang_vulndb" + spdx_license_expression = "CC-BY-4.0" + license_url = "https://github.com/golang/vulndb/blob/master/LICENSE" + repo_url = "git+https://github.com/golang/vulndb" + github_blob_base_url = "https://github.com/golang/vulndb/blob/master/" + + precedence = 500 + + @classmethod + def steps(cls): + return ( + cls.clone, + cls.collect_and_store_advisories, + cls.clean_downloads, + ) + + def clone(self): + self.log(f"Cloning `{self.repo_url}`") + self.vcs_response = fetch_via_vcs(self.repo_url) + + def _repo_root(self) -> Path: + return Path(self.vcs_response.dest_dir) + + def _osv_dir(self) -> Path: + return self._repo_root() / "data" / "osv" + + def _reports_dir(self) -> Path: + return self._repo_root() / "data" / "reports" + + def _cve_v5_dir(self) -> Path: + return self._repo_root() / "data" / "cve" / "v5" + + def advisories_count(self): + return ( + sum(1 for _ in self._osv_dir().glob("*.json")) + + sum(1 for _ in self._reports_dir().glob("*.yaml")) + + sum(1 for _ in self._cve_v5_dir().glob("*.json")) + ) + + def collect_advisories(self) -> Iterable[AdvisoryDataV2]: + for advisory in self._osv_dir().glob("*.json"): + yield self.to_advisory_data_from_osv(advisory) + + for advisory in self._reports_dir().glob("*.yaml"): + yield self.to_advisory_data_from_report(advisory) + + for advisory in self._cve_v5_dir().glob("*.json"): + yield self.to_advisory_data_from_cve5(advisory) + + def to_advisory_data(self, file: Path) -> Optional[AdvisoryDataV2]: + return self.to_advisory_data_from_osv(file) + + def _advisory_url(self, file: Path, data: Optional[dict] = None) -> str: + advisory_url = (data or {}).get("database_specific", {}).get("url") if data else None + if advisory_url: + return advisory_url + return get_advisory_url( + file=file, + base_path=self._repo_root(), + url=self.github_blob_base_url, + ) + + def to_advisory_data_from_osv(self, file: Path) -> Optional[AdvisoryDataV2]: + data = load_json(file) + advisory_id = data.get("id") + if not advisory_id: + self.log(f"Advisory ID not found in {file}") + return + + advisory = parse_advisory_data_v3( + raw_data=data, + supported_ecosystems=["golang"], + advisory_url=self._advisory_url(file, data), + advisory_text=file.read_text(), + advisory_id=advisory_id, + ) + if not advisory: + return + + # Generated OSV files sometimes use the zero date as a placeholder. + if advisory.date_published and advisory.date_published.year < 1970: + advisory.date_published = None + return advisory + + def to_advisory_data_from_report(self, file: Path) -> Optional[AdvisoryDataV2]: + advisory_text = file.read_text() + data = saneyaml.load(advisory_text) or {} + advisory_id = data.get("id") + if not advisory_id: + self.log(f"Advisory ID not found in {file}") + return + + aliases = [] + aliases.extend(data.get("cves") or []) + aliases.extend(data.get("ghsas") or []) + cve_metadata = data.get("cve_metadata") or {} + if cve_id := cve_metadata.get("id"): + aliases.append(cve_id) + aliases = [alias for alias in aliases if alias and alias != advisory_id] + + references = [] + for ref in data.get("references") or []: + if isinstance(ref, dict): + for url in ref.values(): + if url: + references.append(ReferenceV2(url=url)) + elif isinstance(ref, str) and ref: + references.append(ReferenceV2(url=ref)) + + pkg_go_dev_url = f"https://pkg.go.dev/vuln/{advisory_id}" + if pkg_go_dev_url not in [ref.url for ref in references]: + references.append(ReferenceV2(reference_id=advisory_id, url=pkg_go_dev_url)) + + weaknesses = extract_cwe_ids(cve_metadata.get("cwe") or "") + date_published = parse_go_date(data.get("published")) + + return AdvisoryDataV2( + advisory_id=advisory_id, + aliases=aliases, + summary=build_description( + summary=data.get("summary") or "", + description=data.get("description") or "", + ), + affected_packages=affected_packages_from_report(data.get("modules") or []), + references=references, + date_published=date_published, + weaknesses=weaknesses, + url=self._advisory_url(file), + original_advisory_text=advisory_text, + ) + + def to_advisory_data_from_cve5(self, file: Path) -> Optional[AdvisoryDataV2]: + data = load_json(file) + cve_metadata = data.get("cveMetadata") or {} + cve_id = cve_metadata.get("cveId") + go_id = file.stem + advisory_id = cve_id or go_id + if not advisory_id: + self.log(f"Advisory ID not found in {file}") + return + + cna = (data.get("containers") or {}).get("cna") or {} + summary = "" + for description in cna.get("descriptions") or []: + if description.get("lang") in ("en", "en-US"): + summary = description.get("value") or "" + break + if not summary: + summary = cna.get("title") or "" + + aliases = [] + if go_id and go_id != advisory_id: + aliases.append(go_id) + + references = [] + for ref in cna.get("references") or []: + url = ref.get("url") if isinstance(ref, dict) else ref + if url: + references.append(ReferenceV2(url=url)) + + weaknesses = [] + for problem_type in cna.get("problemTypes") or []: + for description in problem_type.get("descriptions") or []: + weaknesses.extend(extract_cwe_ids(description.get("cweId") or "")) + weaknesses.extend(extract_cwe_ids(description.get("description") or "")) + weaknesses = sorted(set(weaknesses)) + + date_published = parse_go_date(cve_metadata.get("datePublished")) + + return AdvisoryDataV2( + advisory_id=advisory_id, + aliases=aliases, + summary=summary, + affected_packages=affected_packages_from_cve5(cna.get("affected") or []), + references=references, + date_published=date_published, + weaknesses=weaknesses, + url=self._advisory_url(file), + original_advisory_text=file.read_text(), + ) + + def clean_downloads(self): + if self.vcs_response: + self.log(f"Removing cloned repository") + self.vcs_response.delete() + + def on_failure(self): + self.clean_downloads() + + +def parse_go_date(value): + if not value or not isinstance(value, str): + return None + parsed = dateparser.parse(value) + if parsed and parsed.year < 1970: + return None + return parsed + + +def extract_cwe_ids(value: str) -> List[int]: + if not value: + return [] + return [int(match) for match in CWE_ID_RE.findall(value)] + + +def golang_module_to_purl(module_path: str) -> PackageURL: + if module_path == "std": + module_path = "stdlib" + purl = PackageURL(type="golang", name=module_path) + return PackageURL.from_string(str(purl)) + + +def golang_ranges_from_constraints( + affected_constraints: List[VersionConstraint], + fixed_constraints: List[VersionConstraint], +) -> Tuple[Optional[GolangVersionRange], Optional[GolangVersionRange]]: + affected_range = None + fixed_range = None + if affected_constraints: + affected_range = GolangVersionRange( + constraints=VersionConstraint.simplify(affected_constraints) + ) + if fixed_constraints: + fixed_range = GolangVersionRange(constraints=VersionConstraint.simplify(fixed_constraints)) + return affected_range, fixed_range + + +def semver_constraint(comparator: str, version: str) -> Optional[VersionConstraint]: + if not version or version in ("0", "*"): + return None + try: + return VersionConstraint( + comparator=comparator, version=GolangVersionRange.version_class(version) + ) + except (InvalidVersion, ValueError, TypeError): + return None + + +def affected_packages_from_report(modules: List[dict]) -> List[AffectedPackageV2]: + affected_packages = [] + for module in modules: + module_path = module.get("module") + if not module_path: + continue + + affected_constraints = [] + fixed_constraints = [] + for version in module.get("versions") or []: + introduced = semver_constraint(">=", version.get("introduced")) + if introduced: + affected_constraints.append(introduced) + fixed = version.get("fixed") + if fixed_lt := semver_constraint("<", fixed): + affected_constraints.append(fixed_lt) + if fixed_eq := semver_constraint("=", fixed): + fixed_constraints.append(fixed_eq) + + affected_range, fixed_range = golang_ranges_from_constraints( + affected_constraints, fixed_constraints + ) + if not affected_range and not fixed_range: + continue + + try: + affected_packages.append( + AffectedPackageV2( + package=golang_module_to_purl(module_path), + affected_version_range=affected_range, + fixed_version_range=fixed_range, + ) + ) + except Exception: + continue + return affected_packages + + +def affected_packages_from_cve5(affected: List[dict]) -> List[AffectedPackageV2]: + affected_packages = [] + for item in affected: + module_path = item.get("packageName") or item.get("product") + if not module_path: + continue + + affected_constraints = [] + fixed_constraints = [] + for version in item.get("versions") or []: + if version.get("status") not in (None, "affected"): + continue + introduced = semver_constraint(">=", version.get("version")) + if introduced: + affected_constraints.append(introduced) + if less_than := semver_constraint("<", version.get("lessThan")): + affected_constraints.append(less_than) + if fixed_eq := semver_constraint("=", version.get("lessThan")): + fixed_constraints.append(fixed_eq) + if less_equal := semver_constraint("<=", version.get("lessThanOrEqual")): + affected_constraints.append(less_equal) + + affected_range, fixed_range = golang_ranges_from_constraints( + affected_constraints, fixed_constraints + ) + if not affected_range and not fixed_range: + continue + + try: + affected_packages.append( + AffectedPackageV2( + package=golang_module_to_purl(module_path), + affected_version_range=affected_range, + fixed_version_range=fixed_range, + ) + ) + except Exception: + continue + return affected_packages diff --git a/vulnerabilities/pipes/fetchcode_utils.py b/vulnerabilities/pipes/fetchcode_utils.py index 93b6dfe21..4647ea191 100644 --- a/vulnerabilities/pipes/fetchcode_utils.py +++ b/vulnerabilities/pipes/fetchcode_utils.py @@ -7,24 +7,74 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +""" +Helpers to list published package versions for version-range unfurl. + +Most ecosystems go through fetchcode. Go is handled here because fetchcode's +goproxy client is too strict for unfurl: + +1. fetchcode GETs ``/@v/{version}.info`` for every version in ``/@v/list``. + Some listed versions 404 on ``.info`` (retracted or never fully published). + fetchcode then raises and we lose the rest of the list. + + Example: ``pkg:golang/github.com/traefik/traefik/v3`` lists + ``v3.0.0-beta1``, but + ``https://proxy.golang.org/github.com/traefik/traefik/v3/@v/v3.0.0-beta1.info`` + is 404. ``v3.0.0.info`` is fine. Unfurl only needs the version strings from + ``/@v/list``. + +2. fetchcode trims Go paths to three segments, which drops major-version + suffixes such as ``/v3`` and nested modules such as + ``github.com/mattermost/mattermost/server/v8``. + +3. The Go vulnerability database uses ``stdlib``, ``cmd/go``, ``runtime``, + and similar names. Those are not proxy modules (first path element has no + dot). Their versions are Go releases from ``https://go.dev/dl/``. + +4. Vanity modules can be a single path segment with a dot, for example + ``gitea.dev`` or ``tailscale.com``. Those are valid goproxy module paths. +""" + import logging from traceback import format_exc as traceback_format_exc from typing import Callable +from typing import Optional +from typing import Set from typing import Union +import requests from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS as FETCHCODE_SUPPORTED_ECOSYSTEMS from fetchcode.package_versions import versions from packageurl import PackageURL +GOPROXY_URL = "https://proxy.golang.org" +GO_RELEASES_URL = "https://go.dev/dl/?mode=json&include=all" +# Hosting sites that are never themselves Go modules. +GO_VCS_ROOTS = {"github.com", "gitlab.com", "bitbucket.org", "gopkg.in"} + +_GO_RELEASE_VERSIONS = None + def get_versions(purl: Union[PackageURL, str], logger: Callable = None): - """Return set of known versions for the given purl.""" + """ + Return a set of known version strings for ``purl``, or None if the + ecosystem is not supported. + + Non-Go packages use fetchcode. Go packages use ``get_golang_versions``. + A leading ``v`` / ``V`` is stripped so versions match univers ranges. + + For example a Traefik PURL yields values such as ``{"3.0.0", "3.1.0"}`` + rather than ``{"v3.0.0", "v3.1.0"}``. + """ if isinstance(purl, str): purl = PackageURL.from_string(purl) if purl.type not in FETCHCODE_SUPPORTED_ECOSYSTEMS: return + if purl.type == "golang": + return get_golang_versions(purl, logger) + try: return {v.value.lstrip("vV") for v in versions(str(purl))} except Exception as e: @@ -33,3 +83,191 @@ def get_versions(purl: Union[PackageURL, str], logger: Callable = None): f"Error while fetching known versions for {purl!s}: {e!r} \n {traceback_format_exc()}", level=logging.ERROR, ) + + +def golang_module_from_purl(purl: PackageURL) -> str: + """ + Return the Go module path encoded in a golang PackageURL. + + PackageURL splits ``pkg:golang/github.com/traefik/traefik/v3`` into + namespace ``github.com/traefik/traefik`` and name ``v3``. The proxy + speaks in a single module path, so those parts are joined again:: + + >>> from packageurl import PackageURL + >>> golang_module_from_purl(PackageURL.from_string( + ... "pkg:golang/github.com/traefik/traefik/v3")) + 'github.com/traefik/traefik/v3' + >>> golang_module_from_purl(PackageURL.from_string("pkg:golang/stdlib")) + 'stdlib' + >>> golang_module_from_purl(PackageURL.from_string("pkg:golang/gitea.dev")) + 'gitea.dev' + """ + return "/".join(part for part in (purl.namespace, purl.name) if part) + + +def is_go_standard_library_module(module: str) -> bool: + """ + Return True if ``module`` is the Go standard library (or a std package). + + Go module paths must have a dot in the first element (``github.com/...``, + ``gitea.dev``). Names without that dot are stdlib in vulndb data: + ``stdlib``, ``std``, ``cmd``, ``cmd/go``, ``runtime``, ``net``. + + Those are not on proxy.golang.org (``malformed module path: missing dot + in first path element``). Their versions are Go toolchain releases. + + >>> is_go_standard_library_module("stdlib") + True + >>> is_go_standard_library_module("cmd/go") + True + >>> is_go_standard_library_module("runtime") + True + >>> is_go_standard_library_module("github.com/traefik/traefik/v3") + False + >>> is_go_standard_library_module("gitea.dev") + False + """ + first = (module.split("/")[0] or "").lower() + if first in {"std", "stdlib"}: + return True + # Real Go modules must have a dot in the first path element. + return bool(first) and "." not in first + + +def escape_go_module_path(path: str) -> str: + """ + Case-encode a Go module path for the goproxy protocol. + + Uppercase letters become ``!`` plus lowercase so the proxy can be served + from a case-insensitive filesystem. See + https://go.dev/ref/mod#goproxy-protocol + + >>> escape_go_module_path("github.com/Azure/go-autorest") + 'github.com/!azure/go-autorest' + >>> escape_go_module_path("gitea.dev") + 'gitea.dev' + """ + escaped = [] + for char in path: + if "A" <= char <= "Z": + escaped.append("!") + escaped.append(char.lower()) + else: + escaped.append(char) + return "".join(escaped) + + +def get_golang_versions(purl: PackageURL, logger: Callable = None) -> Optional[Set[str]]: + """ + Return published version strings for a golang ``purl``. + + Standard library PURLs (``pkg:golang/stdlib``, ``pkg:golang/cmd/go``) + use Go releases from ``https://go.dev/dl/?mode=json&include=all``. + Values look like ``1.25.11`` (the ``go`` prefix is stripped) so they + match vulndb ranges such as ``vers:golang/<1.25.11|>=1.26.0-0``. + + Other modules use ``GET https://proxy.golang.org/{module}/@v/list`` + and do not fetch ``.info``. A listed tag with no ``.info`` file, for + example Traefik ``v3.0.0-beta1``, is kept instead of aborting the list. + + Returned strings have a leading ``v`` stripped:: + + list line "v3.0.0" -> "3.0.0" + """ + module = golang_module_from_purl(purl) + if not module: + return set() + + if is_go_standard_library_module(module): + try: + return fetch_go_release_versions() + except Exception as e: + if logger: + logger( + f"Error while fetching Go releases for {purl!s}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + return set() + + try: + list_text = fetch_golang_version_list(module) + except Exception as e: + if logger: + logger( + f"Error while fetching known versions for {purl!s}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + return set() + + if list_text is None: + if logger: + logger( + f"Go module not found on proxy.golang.org: {module}", + level=logging.WARNING, + ) + return set() + + versions_found = set() + for line in list_text.splitlines(): + version = line.split()[0].strip() if line.strip() else "" + if version: + versions_found.add(version.lstrip("vV")) + return versions_found + + +def fetch_go_release_versions() -> Set[str]: + """ + Return published Go toolchain versions from go.dev. + + Used for ``stdlib``, ``cmd/go``, ``runtime``, and other std packages. + The JSON looks like ``{"version": "go1.25.11"}``; we store ``1.25.11``. + Results are cached for the process so unfurl does not refetch for every + stdlib impact. + """ + global _GO_RELEASE_VERSIONS + if _GO_RELEASE_VERSIONS is not None: + return _GO_RELEASE_VERSIONS + + response = requests.get(GO_RELEASES_URL, timeout=30) + response.raise_for_status() + versions_found = set() + for release in response.json() or []: + version = (release.get("version") or "").strip() + if version.lower().startswith("go"): + version = version[2:] + if version: + versions_found.add(version) + _GO_RELEASE_VERSIONS = versions_found + return versions_found + + +def fetch_golang_version_list(module: str) -> Optional[str]: + """ + Return the ``/@v/list`` body for ``module``, or None if it is not on the proxy. + + The first request uses the full path. On 404, trailing segments are + dropped so a package path can resolve to its module. Walk stops at VCS + hosts (``github.com``) which are not modules. + + Examples: + + - ``github.com/traefik/traefik/v3`` -> that module's list (major version + suffix is kept; fetchcode used to trim it). + - ``github.com/foo/mod/internal/pkg`` 404s, then ``.../mod/internal``, + then ``github.com/foo/mod`` until a list is found. + - ``gitea.dev`` is a one-segment vanity module and is queried as-is. + - ``github.com`` alone is never queried. + """ + parts = [part for part in module.split("/") if part] + while parts: + candidate = "/".join(parts) + if candidate in GO_VCS_ROOTS: + return None + url = f"{GOPROXY_URL}/{escape_go_module_path(candidate)}/@v/list" + response = requests.get(url, timeout=30) + if response.status_code == 404: + parts.pop() + continue + response.raise_for_status() + return response.text + return None diff --git a/vulnerabilities/pipes/osv_v2.py b/vulnerabilities/pipes/osv_v2.py index 9186c43c0..4b8ce47c7 100644 --- a/vulnerabilities/pipes/osv_v2.py +++ b/vulnerabilities/pipes/osv_v2.py @@ -152,7 +152,11 @@ def parse_advisory_data_v3( affected_version_range = None if affected_constraints: try: - valid_affected_constraints = VersionConstraint.simplify(affected_constraints) + valid_affected_constraints = VersionConstraint.simplify( + coerce_constraints_to_version_class( + affected_constraints, version_range_class.version_class + ) + ) validate_comparators(valid_affected_constraints) affected_version_range = version_range_class(constraints=valid_affected_constraints) except Exception as e: @@ -161,7 +165,11 @@ def parse_advisory_data_v3( fixed_version_range = None if fixed_constraints: try: - valid_fixed_constraints = VersionConstraint.simplify(fixed_constraints) + valid_fixed_constraints = VersionConstraint.simplify( + coerce_constraints_to_version_class( + fixed_constraints, version_range_class.version_class + ) + ) validate_comparators(valid_fixed_constraints) fixed_version_range = version_range_class(constraints=valid_fixed_constraints) except Exception as e: @@ -369,6 +377,31 @@ def get_affected_purl(affected_pkg, raw_id): return None +def coerce_constraints_to_version_class(constraints, version_class): + """Return constraints whose versions match ``version_class``.""" + if not constraints or not version_class: + return constraints + + coerced = [] + for constraint in constraints: + version = constraint.version + if version is None or isinstance(version, version_class): + coerced.append(constraint) + continue + try: + coerced.append( + VersionConstraint( + comparator=constraint.comparator, + version=version_class(str(version)), + ) + ) + except InvalidVersion: + logger.error( + f"Could not coerce {version!r} to {version_class!r} for constraint {constraint!r}" + ) + return coerced + + def get_explicit_affected_range(affected_pkg, raw_id, supported_ecosystem): """ Return an explicit affected version range for the affected package data. @@ -468,10 +501,16 @@ def get_version_ranges_constraints( logger.error(f"Invalid Commit: {event_value!r} for OSV id: {raw_id!r}") elif range_type in ("ECOSYSTEM", "SEMVER"): - if range_type == "ECOSYSTEM": - version_class = version_range_class.version_class if version_range_class else None - else: - version_class = SemverVersion # range_type = "SEMVER" + # OSV SEMVER is still ecosystem versions. Using SemverVersion inside + # e.g. GolangVersionRange later fails membership checks during unfurl. + version_class = version_range_class.version_class if version_range_class else None + if not version_class and range_type == "SEMVER": + version_class = SemverVersion + if not version_class: + logger.error( + f"Unsupported ecosystem {supported_ecosystem!r} for OSV id: {raw_id!r}" + ) + continue try: v_obj = version_class(event_value) @@ -488,11 +527,15 @@ def get_version_ranges_constraints( elif event_type == "fixed": # If database specific last known version is available then it's assumed that such last known # version is applicable to all ranges of current affected block. + last_known = db_specific_explicit_last_known + if last_known and not isinstance(last_known, version_class): + try: + last_known = version_class(str(last_known)) + except InvalidVersion: + last_known = None affected_constraint = VersionConstraint(comparator="<", version=v_obj) - if db_specific_explicit_last_known: - affected_constraint = VersionConstraint( - comparator="<=", version=db_specific_explicit_last_known - ) + if last_known: + affected_constraint = VersionConstraint(comparator="<=", version=last_known) affected_constraints.append(affected_constraint) diff --git a/vulnerabilities/pipes/univers_utils.py b/vulnerabilities/pipes/univers_utils.py index 296ebc12f..3cd0fc46a 100644 --- a/vulnerabilities/pipes/univers_utils.py +++ b/vulnerabilities/pipes/univers_utils.py @@ -20,6 +20,19 @@ from vulnerabilities.utils import update_purl_version +def coerce_version_to_range_class(version, version_range: VersionRange): + """Return ``version`` as ``version_range.version_class``, or None.""" + if not version or not version_range: + return version + version_class = version_range.version_class + if isinstance(version, version_class): + return version + try: + return version_class(str(version)) + except Exception: + return None + + def get_exact_purl_from_vers( version_range: VersionRange, base_purl: PackageURL ) -> Tuple[List[PackageURL], List[PackageURL]]: @@ -27,11 +40,19 @@ def get_exact_purl_from_vers( if not version_range: return [], [] - all_versions = [c.version for c in version_range.constraints if c] + all_versions = [ + v + for c in version_range.constraints + if c and (v := coerce_version_to_range_class(c.version, version_range)) + ] version_not_in_range = [ - c.version for c in version_range.constraints if c and c.comparator == "!=" + v + for c in version_range.constraints + if c + and c.comparator == "!=" + and (v := coerce_version_to_range_class(c.version, version_range)) ] - version_in_range = [v for v in all_versions if v and v in version_range] + version_in_range = [v for v in all_versions if v in version_range] purl_in_range = [ update_purl_version(purl=base_purl, version=str(version)) for version in version_in_range diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_golang_vulndb_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_golang_vulndb_importer_v2.py new file mode 100644 index 000000000..6d1363be2 --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_importers/test_golang_vulndb_importer_v2.py @@ -0,0 +1,297 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +from types import SimpleNamespace + +import saneyaml +from packageurl import PackageURL +from univers.versions import GolangVersion + +from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.pipelines.v2_importers.golang_vulndb_importer import ( + GolangVulndbImporterPipeline, +) +from vulnerabilities.pipes.univers_utils import get_exact_purls_v2 + +SAMPLE_OSV = { + "schema_version": "1.3.1", + "id": "GO-2024-2618", + "modified": "0001-01-01T00:00:00Z", + "published": "0001-01-01T00:00:00Z", + "aliases": ["CVE-2024-28110", "GHSA-5pf6-2qwx-pxm2"], + "summary": "Authentication token leak in github.com/cloudevents/sdk-go/v2", + "details": "Using cloudevents.WithRoundTripper leaks credentials.", + "affected": [ + { + "package": { + "name": "github.com/cloudevents/sdk-go/v2", + "ecosystem": "Go", + }, + "ranges": [ + { + "type": "SEMVER", + "events": [ + {"introduced": "0"}, + {"fixed": "2.15.2"}, + ], + } + ], + } + ], + "references": [ + { + "type": "ADVISORY", + "url": "https://github.com/cloudevents/sdk-go/security/advisories/GHSA-5pf6-2qwx-pxm2", + }, + { + "type": "FIX", + "url": "https://github.com/cloudevents/sdk-go/commit/de2f28370b0d2a0f64f92c0c6139fa4b8a7c3851", + }, + ], + "database_specific": { + "url": "https://pkg.go.dev/vuln/GO-2024-2618", + "review_status": "REVIEWED", + }, +} + +SAMPLE_STDLIB_OSV = { + "schema_version": "1.3.1", + "id": "GO-2021-0067", + "published": "2021-04-14T20:04:52Z", + "aliases": ["CVE-2021-27919"], + "summary": "Panic when opening archives in archive/zip", + "details": "Using Reader.Open on an archive can cause a panic.", + "affected": [ + { + "package": { + "name": "stdlib", + "ecosystem": "Go", + }, + "ranges": [ + { + "type": "SEMVER", + "events": [ + {"introduced": "1.16.0-0"}, + {"fixed": "1.16.1"}, + ], + } + ], + } + ], + "references": [{"type": "FIX", "url": "https://go.dev/cl/300489"}], +} + +SAMPLE_REPORT = { + "id": "GO-2020-0001", + "modules": [ + { + "module": "github.com/gin-gonic/gin", + "versions": [{"fixed": "1.6.0"}], + "vulnerable_at": "1.5.0", + } + ], + "summary": "Arbitrary log line injection in github.com/gin-gonic/gin", + "description": "Unsanitized input in the default logger allows log injection.", + "published": "2021-04-14T20:04:52Z", + "ghsas": ["GHSA-6vm3-jj99-7229"], + "references": [ + {"fix": "https://github.com/gin-gonic/gin/pull/2237"}, + {"fix": "https://github.com/gin-gonic/gin/commit/a71af9c144f9579f6dbe945341c1df37aaf09c0d"}, + ], + "cve_metadata": { + "id": "CVE-2020-36567", + "cwe": "CWE-117 Improper Output Neutralization for Logs", + }, +} + +SAMPLE_CVE5 = { + "dataType": "CVE_RECORD", + "dataVersion": "5.0", + "cveMetadata": {"cveId": "CVE-2020-36567"}, + "containers": { + "cna": { + "title": "Arbitrary log line injection in github.com/gin-gonic/gin", + "descriptions": [ + { + "lang": "en", + "value": "Unsanitized input in the default logger in github.com/gin-gonic/gin before v1.6.0 allows remote attackers to inject arbitrary log lines.", + } + ], + "affected": [ + { + "packageName": "github.com/gin-gonic/gin", + "versions": [ + { + "version": "0", + "lessThan": "1.6.0", + "status": "affected", + "versionType": "semver", + } + ], + } + ], + "problemTypes": [ + { + "descriptions": [ + { + "lang": "en", + "description": "CWE-117 Improper Output Neutralization for Logs", + } + ] + } + ], + "references": [{"url": "https://pkg.go.dev/vuln/GO-2020-0001"}], + } + }, +} + + +def test_clone(monkeypatch): + import vulnerabilities.pipelines.v2_importers.golang_vulndb_importer as golang_mod + + dummy = SimpleNamespace(dest_dir="dummy", delete=lambda: None) + monkeypatch.setattr(golang_mod, "fetch_via_vcs", lambda url: dummy) + + p = GolangVulndbImporterPipeline() + p.clone() + + assert p.vcs_response is dummy + + +def test_clean_downloads_and_on_failure(): + called = {} + + def delete(): + called["deleted"] = True + + dummy = SimpleNamespace(dest_dir="dummy", delete=delete) + p = GolangVulndbImporterPipeline() + p.vcs_response = dummy + p.clean_downloads() + assert called.get("deleted", False) + called.clear() + p.on_failure() + assert called.get("deleted", False) + + +def test_advisories_count_and_collect(tmp_path): + osv_dir = tmp_path / "data" / "osv" + reports_dir = tmp_path / "data" / "reports" + cve_dir = tmp_path / "data" / "cve" / "v5" + osv_dir.mkdir(parents=True) + reports_dir.mkdir(parents=True) + cve_dir.mkdir(parents=True) + (osv_dir / "GO-2024-2618.json").write_text(json.dumps(SAMPLE_OSV)) + (osv_dir / "missing-id.json").write_text(json.dumps({"summary": "no id"})) + (reports_dir / "GO-2020-0001.yaml").write_text(saneyaml.dump(SAMPLE_REPORT)) + (cve_dir / "GO-2020-0001.json").write_text(json.dumps(SAMPLE_CVE5)) + + p = GolangVulndbImporterPipeline() + p.vcs_response = SimpleNamespace(dest_dir=str(tmp_path), delete=lambda: None) + assert p.advisories_count() == 4 + + advisories = list(p.collect_advisories()) + real = [a for a in advisories if isinstance(a, AdvisoryDataV2)] + assert {a.advisory_id for a in real} == {"GO-2024-2618", "GO-2020-0001", "CVE-2020-36567"} + + +def test_to_advisory_data_skips_missing_id(tmp_path): + p = GolangVulndbImporterPipeline() + file = tmp_path / "missing.json" + file.write_text(json.dumps({"summary": "no id"})) + assert p.to_advisory_data(file) is None + + +def test_to_advisory_data_full(tmp_path): + file = tmp_path / "GO-2024-2618.json" + file.write_text(json.dumps(SAMPLE_OSV)) + p = GolangVulndbImporterPipeline() + p.vcs_response = SimpleNamespace(dest_dir=str(tmp_path), delete=lambda: None) + + adv = p.to_advisory_data(file) + assert isinstance(adv, AdvisoryDataV2) + assert adv.advisory_id == "GO-2024-2618" + assert "Authentication token leak" in adv.summary + assert adv.date_published is None + assert adv.url == "https://pkg.go.dev/vuln/GO-2024-2618" + assert set(adv.aliases) == {"CVE-2024-28110", "GHSA-5pf6-2qwx-pxm2"} + urls = [r.url for r in adv.references] + assert "https://github.com/cloudevents/sdk-go/security/advisories/GHSA-5pf6-2qwx-pxm2" in urls + + pkg = adv.affected_packages[0] + assert pkg.package == PackageURL( + type="golang", namespace="github.com/cloudevents/sdk-go", name="v2" + ) + assert pkg.fixed_version_range is not None + assert "2.15.2" in str(pkg.fixed_version_range) + assert all( + isinstance(constraint.version, GolangVersion) + for constraint in pkg.fixed_version_range.constraints + ) + affected_purls, fixed_purls = get_exact_purls_v2(pkg) + assert [str(purl.version) for purl in fixed_purls] == ["2.15.2"] + assert affected_purls == [] + + +def test_to_advisory_data_stdlib_and_published_date(tmp_path): + osv_dir = tmp_path / "data" / "osv" + osv_dir.mkdir(parents=True) + file = osv_dir / "GO-2021-0067.json" + file.write_text(json.dumps(SAMPLE_STDLIB_OSV)) + p = GolangVulndbImporterPipeline() + p.vcs_response = SimpleNamespace(dest_dir=str(tmp_path), delete=lambda: None) + + adv = p.to_advisory_data(file) + assert adv.advisory_id == "GO-2021-0067" + assert adv.date_published is not None + assert adv.date_published.year == 2021 + assert adv.affected_packages[0].package == PackageURL(type="golang", name="stdlib") + assert adv.url.endswith("data/osv/GO-2021-0067.json") + + +def test_to_advisory_data_from_report(tmp_path): + reports_dir = tmp_path / "data" / "reports" + reports_dir.mkdir(parents=True) + file = reports_dir / "GO-2020-0001.yaml" + file.write_text(saneyaml.dump(SAMPLE_REPORT)) + p = GolangVulndbImporterPipeline() + p.vcs_response = SimpleNamespace(dest_dir=str(tmp_path), delete=lambda: None) + + adv = p.to_advisory_data_from_report(file) + assert adv.advisory_id == "GO-2020-0001" + assert set(adv.aliases) == {"GHSA-6vm3-jj99-7229", "CVE-2020-36567"} + assert adv.weaknesses == [117] + assert adv.date_published.year == 2021 + assert adv.url.endswith("data/reports/GO-2020-0001.yaml") + urls = [r.url for r in adv.references] + assert "https://github.com/gin-gonic/gin/pull/2237" in urls + assert "https://pkg.go.dev/vuln/GO-2020-0001" in urls + pkg = adv.affected_packages[0] + assert pkg.package == PackageURL(type="golang", namespace="github.com/gin-gonic", name="gin") + assert "1.6.0" in str(pkg.fixed_version_range) + + +def test_to_advisory_data_from_cve5(tmp_path): + cve_dir = tmp_path / "data" / "cve" / "v5" + cve_dir.mkdir(parents=True) + file = cve_dir / "GO-2020-0001.json" + file.write_text(json.dumps(SAMPLE_CVE5)) + p = GolangVulndbImporterPipeline() + p.vcs_response = SimpleNamespace(dest_dir=str(tmp_path), delete=lambda: None) + + adv = p.to_advisory_data_from_cve5(file) + assert adv.advisory_id == "CVE-2020-36567" + assert adv.aliases == ["GO-2020-0001"] + assert adv.weaknesses == [117] + assert "Unsanitized input" in adv.summary + pkg = adv.affected_packages[0] + assert pkg.package == PackageURL(type="golang", namespace="github.com/gin-gonic", name="gin") + assert "1.6.0" in str(pkg.affected_version_range) + assert "1.6.0" in str(pkg.fixed_version_range) diff --git a/vulnerabilities/tests/pipes/test_fetchcode_utils.py b/vulnerabilities/tests/pipes/test_fetchcode_utils.py new file mode 100644 index 000000000..ec08ca54c --- /dev/null +++ b/vulnerabilities/tests/pipes/test_fetchcode_utils.py @@ -0,0 +1,103 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from unittest.mock import Mock +from unittest.mock import patch + +from django.test import SimpleTestCase +from packageurl import PackageURL + +from vulnerabilities.pipes.fetchcode_utils import get_versions + + +class TestGetGolangVersions(SimpleTestCase): + @patch("vulnerabilities.pipes.fetchcode_utils.requests.get") + def test_uses_list_and_skips_info(self, mock_get): + response = Mock() + response.status_code = 200 + response.text = "v3.0.0-beta1\nv3.0.0\nv3.1.0\n" + response.raise_for_status = Mock() + mock_get.return_value = response + + versions = get_versions(PackageURL.from_string("pkg:golang/github.com/traefik/traefik/v3")) + + self.assertEqual(versions, {"3.0.0-beta1", "3.0.0", "3.1.0"}) + mock_get.assert_called_once() + self.assertTrue( + mock_get.call_args.args[0].endswith("/github.com/traefik/traefik/v3/@v/list") + ) + + @patch("vulnerabilities.pipes.fetchcode_utils.requests.get") + def test_walks_up_module_path_on_404(self, mock_get): + missing = Mock() + missing.status_code = 404 + found = Mock() + found.status_code = 200 + found.text = "v1.2.3\n" + found.raise_for_status = Mock() + mock_get.side_effect = [missing, found] + + versions = get_versions( + PackageURL.from_string("pkg:golang/github.com/example/mod/internal/pkg") + ) + + self.assertEqual(versions, {"1.2.3"}) + urls = [call.args[0] for call in mock_get.call_args_list] + self.assertTrue(urls[0].endswith("/github.com/example/mod/internal/pkg/@v/list")) + self.assertTrue(urls[1].endswith("/github.com/example/mod/internal/@v/list")) + + @patch("vulnerabilities.pipes.fetchcode_utils.requests.get") + def test_one_segment_vanity_module(self, mock_get): + response = Mock() + response.status_code = 200 + response.text = "v1.27.0\nv1.26.1\n" + response.raise_for_status = Mock() + mock_get.return_value = response + + versions = get_versions(PackageURL.from_string("pkg:golang/gitea.dev")) + + self.assertEqual(versions, {"1.27.0", "1.26.1"}) + self.assertTrue(mock_get.call_args.args[0].endswith("/gitea.dev/@v/list")) + + @patch("vulnerabilities.pipes.fetchcode_utils.requests.get") + def test_stdlib_uses_go_releases(self, mock_get): + response = Mock() + response.status_code = 200 + response.json.return_value = [ + {"version": "go1.25.11"}, + {"version": "go1.26.4"}, + {"version": "go1.27.0-rc.3"}, + ] + response.raise_for_status = Mock() + mock_get.return_value = response + + import vulnerabilities.pipes.fetchcode_utils as fetchcode_utils + + fetchcode_utils._GO_RELEASE_VERSIONS = None + versions = get_versions(PackageURL.from_string("pkg:golang/stdlib")) + fetchcode_utils._GO_RELEASE_VERSIONS = None + + self.assertEqual(versions, {"1.25.11", "1.26.4", "1.27.0-rc.3"}) + self.assertTrue(mock_get.call_args.args[0].startswith("https://go.dev/dl/")) + + @patch("vulnerabilities.pipes.fetchcode_utils.requests.get") + def test_cmd_go_uses_go_releases(self, mock_get): + response = Mock() + response.status_code = 200 + response.json.return_value = [{"version": "go1.24.0"}] + response.raise_for_status = Mock() + mock_get.return_value = response + + import vulnerabilities.pipes.fetchcode_utils as fetchcode_utils + + fetchcode_utils._GO_RELEASE_VERSIONS = None + versions = get_versions(PackageURL.from_string("pkg:golang/cmd/go")) + fetchcode_utils._GO_RELEASE_VERSIONS = None + + self.assertEqual(versions, {"1.24.0"}) diff --git a/vulnerabilities/tests/pipes/test_osv_v2.py b/vulnerabilities/tests/pipes/test_osv_v2.py index a6c3a380c..2fb276fd3 100644 --- a/vulnerabilities/tests/pipes/test_osv_v2.py +++ b/vulnerabilities/tests/pipes/test_osv_v2.py @@ -15,6 +15,7 @@ from univers.version_constraint import VersionConstraint from univers.version_range import MavenVersionRange from univers.version_range import PypiVersionRange +from univers.versions import GolangVersion from univers.versions import MavenVersion from univers.versions import PypiVersion @@ -77,6 +78,16 @@ def test_get_version_ranges_constraints(): [], ) + affected, fixed, intro_commits, fixed_commits = get_version_ranges_constraints( + ranges={"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "0.3.8"}]}, + raw_id="GO-2020-0001", + supported_ecosystem="golang", + ) + assert affected == [VersionConstraint(comparator="<", version=GolangVersion(string="0.3.8"))] + assert fixed == [VersionConstraint(comparator="=", version=GolangVersion(string="0.3.8"))] + assert intro_commits == [] + assert fixed_commits == [] + def test_get_explicit_affected_constraints(): assert get_explicit_affected_range(