diff --git a/vulntotal/datasources/__init__.py b/vulntotal/datasources/__init__.py index fb269a201..cfda30216 100644 --- a/vulntotal/datasources/__init__.py +++ b/vulntotal/datasources/__init__.py @@ -22,7 +22,7 @@ "github": github.GithubDataSource, "safetydb": safetydb.SafetydbDataSource, "gitlab": gitlab.GitlabDataSource, - "oss_index": oss_index.OSSDataSource, + "sonatype_guide": oss_index.SonatypeGuideDataSource, "osv": osv.OSVDataSource, "snyk": snyk.SnykDataSource, "vulnerablecode": vulnerablecode.VulnerableCodeDataSource, diff --git a/vulntotal/datasources/oss_index.py b/vulntotal/datasources/oss_index.py index a18e65d21..7fce64a6a 100644 --- a/vulntotal/datasources/oss_index.py +++ b/vulntotal/datasources/oss_index.py @@ -12,6 +12,7 @@ from typing import Iterable import requests +from dotenv import load_dotenv from packageurl import PackageURL from vulntotal.validator import DataSource @@ -20,26 +21,56 @@ logger = logging.getLogger(__name__) -class OSSDataSource(DataSource): +class SonatypeGuideDataSource(DataSource): + """ + Datasource for Sonatype Guide (formerly OSS Index). + + Authentication: + Configure your Sonatype Guide Personal Access Token (PAT) in the + environment or ``.env`` file:: + + SONATYPE_GUIDE_TOKEN="your-sonatype-guide-pat" + + When authenticating with a PAT, the username is optional and automatically + defaults to ``"token"``, so users do not need to invent or provide a username. + An explicit username can optionally be configured via ``SONATYPE_GUIDE_USERNAME``:: + + SONATYPE_GUIDE_USERNAME="user@example.com" + SONATYPE_GUIDE_TOKEN="your-sonatype-guide-pat" + + Legacy ``OSS_TOKEN`` and ``OSS_USERNAME`` environment variables are also + supported for backward compatibility. + """ + spdx_license_expression = "TODO" license_url = "TODO" - api_unauthenticated = "https://ossindex.sonatype.org/api/v3/component-report" - api_authenticated = "https://ossindex.sonatype.org/api/v3/authorized/component-report" + api_unauthenticated = "https://api.guide.sonatype.com/api/v3/component-report" + api_authenticated = "https://api.guide.sonatype.com/api/v3/authorized/component-report" def fetch_json_response(self, coordinates): - """Fetch JSON response from OSS Index API for a given list of coordinates. + """Fetch JSON response from Sonatype Guide API for a given list of coordinates. + + Authentication uses ``SONATYPE_GUIDE_TOKEN`` (or legacy ``OSS_TOKEN``). + When only ``SONATYPE_GUIDE_TOKEN`` is set, username defaults to ``"token"``, + so users only need to provide their PAT without inventing a username. Parameters: coordinates: A list of strings representing the package coordinates. Returns: - A dictionary containing the JSON response from the OSS Index API, or None if the response is unsuccessful or an error occurs while fetching data. + A dictionary containing the JSON response from the Sonatype Guide API, + or None if the response is unsuccessful or an error occurs while fetching data. """ - username = os.environ.get("OSS_USERNAME", None) - token = os.environ.get("OSS_TOKEN", None) + load_dotenv() + token = os.environ.get("SONATYPE_GUIDE_TOKEN") or os.environ.get("OSS_TOKEN") + username = ( + os.environ.get("SONATYPE_GUIDE_USERNAME") + or os.environ.get("OSS_USERNAME") + or ("token" if token else None) + ) auth = None url = self.api_unauthenticated - if username and token: + if token: auth = (username, token) url = self.api_authenticated response = requests.post(url, auth=auth, json={"coordinates": coordinates}) @@ -48,12 +79,17 @@ def fetch_json_response(self, coordinates): return response.json() except requests.exceptions.HTTPError as e: if e.response.status_code == 401: - logger.error("Invalid credentials") + if auth: + logger.error("Invalid credentials") + else: + logger.error( + "Sonatype Guide requires a token: set SONATYPE_GUIDE_TOKEN in your environment or .env" + ) elif e.response.status_code == 429: msg = ( "Too many requests" if auth - else "Too many requests: add OSS_USERNAME and OSS_TOKEN in .env file" + else "Too many requests: add SONATYPE_GUIDE_TOKEN (or legacy OSS_TOKEN) in .env file" ) logger.error(msg) else: @@ -89,6 +125,10 @@ def supported_ecosystem(cls): } +# Maintain backward compatibility for existing imports and usages +OSSDataSource = SonatypeGuideDataSource + + def parse_advisory(component, purl) -> Iterable[VendorData]: """ Parse component from OSS Index API and yield VendorData. diff --git a/vulntotal/tests/test_oss_index.py b/vulntotal/tests/test_oss_index.py index 6ba02e634..11da3befd 100644 --- a/vulntotal/tests/test_oss_index.py +++ b/vulntotal/tests/test_oss_index.py @@ -8,8 +8,11 @@ # import json +import os from pathlib import Path +from unittest import mock +import requests from commoncode import testcase from packageurl import PackageURL @@ -17,9 +20,15 @@ from vulntotal.datasources import oss_index -class TestDeps(testcase.FileBasedTesting): +class TestSonatypeGuide(testcase.FileBasedTesting): test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "oss_index") + def setUp(self): + super().setUp() + patcher = mock.patch("vulntotal.datasources.oss_index.load_dotenv") + self.mock_load_dotenv = patcher.start() + self.addCleanup(patcher.stop) + def test_parse_advisory(self): advisory_file = self.get_test_loc("advisory.json") with open(advisory_file) as f: @@ -32,3 +41,132 @@ def test_parse_advisory(self): ] expected_file = self.get_test_loc("parse_advisory-expected.json", must_exist=False) util_tests.check_results_against_json(results, expected_file) + + @mock.patch.dict(os.environ, {}, clear=True) + @mock.patch("requests.post") + def test_fetch_json_response_unauthenticated(self, mock_post): + mock_response = mock.MagicMock() + mock_response.json.return_value = [{"coordinates": "pkg:npm/foo@1.0"}] + mock_post.return_value = mock_response + + ds = oss_index.SonatypeGuideDataSource() + result = ds.fetch_json_response(["pkg:npm/foo@1.0"]) + + mock_post.assert_called_once_with( + "https://api.guide.sonatype.com/api/v3/component-report", + auth=None, + json={"coordinates": ["pkg:npm/foo@1.0"]}, + ) + assert result == [{"coordinates": "pkg:npm/foo@1.0"}] + + @mock.patch.dict(os.environ, {"SONATYPE_GUIDE_TOKEN": "test-pat-123"}, clear=True) + @mock.patch("requests.post") + def test_fetch_json_response_with_pat_only(self, mock_post): + mock_response = mock.MagicMock() + mock_response.json.return_value = [{"coordinates": "pkg:npm/foo@1.0"}] + mock_post.return_value = mock_response + + ds = oss_index.SonatypeGuideDataSource() + result = ds.fetch_json_response(["pkg:npm/foo@1.0"]) + + mock_post.assert_called_once_with( + "https://api.guide.sonatype.com/api/v3/authorized/component-report", + auth=("token", "test-pat-123"), + json={"coordinates": ["pkg:npm/foo@1.0"]}, + ) + assert result == [{"coordinates": "pkg:npm/foo@1.0"}] + + @mock.patch.dict( + os.environ, + {"SONATYPE_GUIDE_USERNAME": "developer@example.com", "SONATYPE_GUIDE_TOKEN": "test-pat-123"}, + clear=True, + ) + @mock.patch("requests.post") + def test_fetch_json_response_with_username_and_pat(self, mock_post): + mock_response = mock.MagicMock() + mock_response.json.return_value = [{"coordinates": "pkg:npm/foo@1.0"}] + mock_post.return_value = mock_response + + ds = oss_index.SonatypeGuideDataSource() + result = ds.fetch_json_response(["pkg:npm/foo@1.0"]) + + mock_post.assert_called_once_with( + "https://api.guide.sonatype.com/api/v3/authorized/component-report", + auth=("developer@example.com", "test-pat-123"), + json={"coordinates": ["pkg:npm/foo@1.0"]}, + ) + assert result == [{"coordinates": "pkg:npm/foo@1.0"}] + + @mock.patch.dict( + os.environ, + {"OSS_USERNAME": "legacy@example.com", "OSS_TOKEN": "legacy-token"}, + clear=True, + ) + @mock.patch("requests.post") + def test_fetch_json_response_legacy_env_fallback(self, mock_post): + mock_response = mock.MagicMock() + mock_response.json.return_value = [{"coordinates": "pkg:npm/foo@1.0"}] + mock_post.return_value = mock_response + + ds = oss_index.OSSDataSource() + result = ds.fetch_json_response(["pkg:npm/foo@1.0"]) + + mock_post.assert_called_once_with( + "https://api.guide.sonatype.com/api/v3/authorized/component-report", + auth=("legacy@example.com", "legacy-token"), + json={"coordinates": ["pkg:npm/foo@1.0"]}, + ) + assert result == [{"coordinates": "pkg:npm/foo@1.0"}] + + @mock.patch.dict(os.environ, {}, clear=True) + @mock.patch("requests.post") + def test_fetch_json_response_401_unauthenticated(self, mock_post): + mock_response = mock.MagicMock() + mock_response.status_code = 401 + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( + response=mock_response + ) + mock_post.return_value = mock_response + + ds = oss_index.SonatypeGuideDataSource() + with self.assertLogs("vulntotal.datasources.oss_index", level="ERROR") as cm: + assert ds.fetch_json_response(["pkg:npm/foo@1.0"]) is None + assert any( + "Sonatype Guide requires a token: set SONATYPE_GUIDE_TOKEN in your environment or .env" + in msg + for msg in cm.output + ) + + @mock.patch.dict(os.environ, {"SONATYPE_GUIDE_TOKEN": "bad-token"}, clear=True) + @mock.patch("requests.post") + def test_fetch_json_response_401_authenticated(self, mock_post): + mock_response = mock.MagicMock() + mock_response.status_code = 401 + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( + response=mock_response + ) + mock_post.return_value = mock_response + + ds = oss_index.SonatypeGuideDataSource() + with self.assertLogs("vulntotal.datasources.oss_index", level="ERROR") as cm: + assert ds.fetch_json_response(["pkg:npm/foo@1.0"]) is None + assert any("Invalid credentials" in msg for msg in cm.output) + + @mock.patch.dict(os.environ, {}, clear=True) + @mock.patch("requests.post") + def test_fetch_json_response_http_errors(self, mock_post): + for status_code in (401, 429, 500): + mock_response = mock.MagicMock() + mock_response.status_code = status_code + error = requests.exceptions.HTTPError(response=mock_response) + mock_response.raise_for_status.side_effect = error + mock_post.return_value = mock_response + mock_post.side_effect = None + + ds = oss_index.SonatypeGuideDataSource() + assert ds.fetch_json_response(["pkg:npm/foo@1.0"]) is None + + def test_datasource_advisory_unsupported_purl(self): + ds = oss_index.SonatypeGuideDataSource() + unsupported = PackageURL.from_string("pkg:unsupported/foo@1.0") + assert ds.datasource_advisory(unsupported) is None diff --git a/vulntotal/vulntotal_cli.py b/vulntotal/vulntotal_cli.py index d60ceeb85..77c029f6c 100755 --- a/vulntotal/vulntotal_cli.py +++ b/vulntotal/vulntotal_cli.py @@ -578,7 +578,7 @@ def get_range_from_discrete_version_string(schema, versions): "deps": get_range_from_discrete_version_string, "github": build_range_from_github_advisory_constraint, "gitlab": from_gitlab_native, - "oss_index": None, + "sonatype_guide": None, "osv": get_range_from_discrete_version_string, "snyk": build_range_from_snyk_advisory_string, "safetydb": build_range_from_snyk_advisory_string,