Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion vulntotal/datasources/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"github": github.GithubDataSource,
"safetydb": safetydb.SafetydbDataSource,
"gitlab": gitlab.GitlabDataSource,
"oss_index": oss_index.OSSDataSource,
"sonatype_guide": oss_index.SonatypeGuideDataSource,
"osv": osv.OSVDataSource,
"snyk": snyk.SnykDataSource,
"vulnerablecode": vulnerablecode.VulnerableCodeDataSource,
Expand Down
60 changes: 50 additions & 10 deletions vulntotal/datasources/oss_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from typing import Iterable

import requests
from dotenv import load_dotenv
from packageurl import PackageURL

from vulntotal.validator import DataSource
Expand All @@ -20,26 +21,56 @@
logger = logging.getLogger(__name__)


class OSSDataSource(DataSource):
class SonatypeGuideDataSource(DataSource):
"""
Datasource for Sonatype Guide (formerly OSS Index).

Authentication:
Configure your Sonatype Guide Personal Access Token (PAT) in the
environment or ``.env`` file::

SONATYPE_GUIDE_TOKEN="your-sonatype-guide-pat"

When authenticating with a PAT, the username is optional and automatically
defaults to ``"token"``, so users do not need to invent or provide a username.
An explicit username can optionally be configured via ``SONATYPE_GUIDE_USERNAME``::

SONATYPE_GUIDE_USERNAME="user@example.com"
SONATYPE_GUIDE_TOKEN="your-sonatype-guide-pat"

Legacy ``OSS_TOKEN`` and ``OSS_USERNAME`` environment variables are also
supported for backward compatibility.
"""

spdx_license_expression = "TODO"
license_url = "TODO"
api_unauthenticated = "https://ossindex.sonatype.org/api/v3/component-report"
api_authenticated = "https://ossindex.sonatype.org/api/v3/authorized/component-report"
api_unauthenticated = "https://api.guide.sonatype.com/api/v3/component-report"
api_authenticated = "https://api.guide.sonatype.com/api/v3/authorized/component-report"

def fetch_json_response(self, coordinates):
"""Fetch JSON response from OSS Index API for a given list of coordinates.
"""Fetch JSON response from Sonatype Guide API for a given list of coordinates.

Authentication uses ``SONATYPE_GUIDE_TOKEN`` (or legacy ``OSS_TOKEN``).
When only ``SONATYPE_GUIDE_TOKEN`` is set, username defaults to ``"token"``,
so users only need to provide their PAT without inventing a username.

Parameters:
coordinates: A list of strings representing the package coordinates.

Returns:
A dictionary containing the JSON response from the OSS Index API, or None if the response is unsuccessful or an error occurs while fetching data.
A dictionary containing the JSON response from the Sonatype Guide API,
or None if the response is unsuccessful or an error occurs while fetching data.
"""
username = os.environ.get("OSS_USERNAME", None)
token = os.environ.get("OSS_TOKEN", None)
load_dotenv()
token = os.environ.get("SONATYPE_GUIDE_TOKEN") or os.environ.get("OSS_TOKEN")
username = (
os.environ.get("SONATYPE_GUIDE_USERNAME")
or os.environ.get("OSS_USERNAME")
or ("token" if token else None)
)
auth = None
url = self.api_unauthenticated
if username and token:
if token:
auth = (username, token)
url = self.api_authenticated
response = requests.post(url, auth=auth, json={"coordinates": coordinates})
Expand All @@ -48,12 +79,17 @@ def fetch_json_response(self, coordinates):
return response.json()
except requests.exceptions.HTTPError as e:
if e.response.status_code == 401:
logger.error("Invalid credentials")
if auth:
logger.error("Invalid credentials")
else:
logger.error(
"Sonatype Guide requires a token: set SONATYPE_GUIDE_TOKEN in your environment or .env"
)
elif e.response.status_code == 429:
msg = (
"Too many requests"
if auth
else "Too many requests: add OSS_USERNAME and OSS_TOKEN in .env file"
else "Too many requests: add SONATYPE_GUIDE_TOKEN (or legacy OSS_TOKEN) in .env file"
)
logger.error(msg)
else:
Expand Down Expand Up @@ -89,6 +125,10 @@ def supported_ecosystem(cls):
}


# Maintain backward compatibility for existing imports and usages
OSSDataSource = SonatypeGuideDataSource


def parse_advisory(component, purl) -> Iterable[VendorData]:
"""
Parse component from OSS Index API and yield VendorData.
Expand Down
140 changes: 139 additions & 1 deletion vulntotal/tests/test_oss_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,27 @@
#

import json
import os
from pathlib import Path
from unittest import mock

import requests
from commoncode import testcase
from packageurl import PackageURL

from vulnerabilities.tests import util_tests
from vulntotal.datasources import oss_index


class TestDeps(testcase.FileBasedTesting):
class TestSonatypeGuide(testcase.FileBasedTesting):
test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "oss_index")

def setUp(self):
super().setUp()
patcher = mock.patch("vulntotal.datasources.oss_index.load_dotenv")
self.mock_load_dotenv = patcher.start()
self.addCleanup(patcher.stop)

def test_parse_advisory(self):
advisory_file = self.get_test_loc("advisory.json")
with open(advisory_file) as f:
Expand All @@ -32,3 +41,132 @@ def test_parse_advisory(self):
]
expected_file = self.get_test_loc("parse_advisory-expected.json", must_exist=False)
util_tests.check_results_against_json(results, expected_file)

@mock.patch.dict(os.environ, {}, clear=True)
@mock.patch("requests.post")
def test_fetch_json_response_unauthenticated(self, mock_post):
mock_response = mock.MagicMock()
mock_response.json.return_value = [{"coordinates": "pkg:npm/foo@1.0"}]
mock_post.return_value = mock_response

ds = oss_index.SonatypeGuideDataSource()
result = ds.fetch_json_response(["pkg:npm/foo@1.0"])

mock_post.assert_called_once_with(
"https://api.guide.sonatype.com/api/v3/component-report",
auth=None,
json={"coordinates": ["pkg:npm/foo@1.0"]},
)
assert result == [{"coordinates": "pkg:npm/foo@1.0"}]

@mock.patch.dict(os.environ, {"SONATYPE_GUIDE_TOKEN": "test-pat-123"}, clear=True)
@mock.patch("requests.post")
def test_fetch_json_response_with_pat_only(self, mock_post):
mock_response = mock.MagicMock()
mock_response.json.return_value = [{"coordinates": "pkg:npm/foo@1.0"}]
mock_post.return_value = mock_response

ds = oss_index.SonatypeGuideDataSource()
result = ds.fetch_json_response(["pkg:npm/foo@1.0"])

mock_post.assert_called_once_with(
"https://api.guide.sonatype.com/api/v3/authorized/component-report",
auth=("token", "test-pat-123"),
json={"coordinates": ["pkg:npm/foo@1.0"]},
)
assert result == [{"coordinates": "pkg:npm/foo@1.0"}]

@mock.patch.dict(
os.environ,
{"SONATYPE_GUIDE_USERNAME": "developer@example.com", "SONATYPE_GUIDE_TOKEN": "test-pat-123"},
clear=True,
)
@mock.patch("requests.post")
def test_fetch_json_response_with_username_and_pat(self, mock_post):
mock_response = mock.MagicMock()
mock_response.json.return_value = [{"coordinates": "pkg:npm/foo@1.0"}]
mock_post.return_value = mock_response

ds = oss_index.SonatypeGuideDataSource()
result = ds.fetch_json_response(["pkg:npm/foo@1.0"])

mock_post.assert_called_once_with(
"https://api.guide.sonatype.com/api/v3/authorized/component-report",
auth=("developer@example.com", "test-pat-123"),
json={"coordinates": ["pkg:npm/foo@1.0"]},
)
assert result == [{"coordinates": "pkg:npm/foo@1.0"}]

@mock.patch.dict(
os.environ,
{"OSS_USERNAME": "legacy@example.com", "OSS_TOKEN": "legacy-token"},
clear=True,
)
@mock.patch("requests.post")
def test_fetch_json_response_legacy_env_fallback(self, mock_post):
mock_response = mock.MagicMock()
mock_response.json.return_value = [{"coordinates": "pkg:npm/foo@1.0"}]
mock_post.return_value = mock_response

ds = oss_index.OSSDataSource()
result = ds.fetch_json_response(["pkg:npm/foo@1.0"])

mock_post.assert_called_once_with(
"https://api.guide.sonatype.com/api/v3/authorized/component-report",
auth=("legacy@example.com", "legacy-token"),
json={"coordinates": ["pkg:npm/foo@1.0"]},
)
assert result == [{"coordinates": "pkg:npm/foo@1.0"}]

@mock.patch.dict(os.environ, {}, clear=True)
@mock.patch("requests.post")
def test_fetch_json_response_401_unauthenticated(self, mock_post):
mock_response = mock.MagicMock()
mock_response.status_code = 401
mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError(
response=mock_response
)
mock_post.return_value = mock_response

ds = oss_index.SonatypeGuideDataSource()
with self.assertLogs("vulntotal.datasources.oss_index", level="ERROR") as cm:
assert ds.fetch_json_response(["pkg:npm/foo@1.0"]) is None
assert any(
"Sonatype Guide requires a token: set SONATYPE_GUIDE_TOKEN in your environment or .env"
in msg
for msg in cm.output
)

@mock.patch.dict(os.environ, {"SONATYPE_GUIDE_TOKEN": "bad-token"}, clear=True)
@mock.patch("requests.post")
def test_fetch_json_response_401_authenticated(self, mock_post):
mock_response = mock.MagicMock()
mock_response.status_code = 401
mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError(
response=mock_response
)
mock_post.return_value = mock_response

ds = oss_index.SonatypeGuideDataSource()
with self.assertLogs("vulntotal.datasources.oss_index", level="ERROR") as cm:
assert ds.fetch_json_response(["pkg:npm/foo@1.0"]) is None
assert any("Invalid credentials" in msg for msg in cm.output)

@mock.patch.dict(os.environ, {}, clear=True)
@mock.patch("requests.post")
def test_fetch_json_response_http_errors(self, mock_post):
for status_code in (401, 429, 500):
mock_response = mock.MagicMock()
mock_response.status_code = status_code
error = requests.exceptions.HTTPError(response=mock_response)
mock_response.raise_for_status.side_effect = error
mock_post.return_value = mock_response
mock_post.side_effect = None

ds = oss_index.SonatypeGuideDataSource()
assert ds.fetch_json_response(["pkg:npm/foo@1.0"]) is None

def test_datasource_advisory_unsupported_purl(self):
ds = oss_index.SonatypeGuideDataSource()
unsupported = PackageURL.from_string("pkg:unsupported/foo@1.0")
assert ds.datasource_advisory(unsupported) is None
2 changes: 1 addition & 1 deletion vulntotal/vulntotal_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -578,7 +578,7 @@ def get_range_from_discrete_version_string(schema, versions):
"deps": get_range_from_discrete_version_string,
"github": build_range_from_github_advisory_constraint,
"gitlab": from_gitlab_native,
"oss_index": None,
"sonatype_guide": None,
"osv": get_range_from_discrete_version_string,
"snyk": build_range_from_snyk_advisory_string,
"safetydb": build_range_from_snyk_advisory_string,
Expand Down