diff --git a/.github/workflows/update-download-badges.yml b/.github/workflows/update-download-badges.yml index 6b57637..c27004a 100644 --- a/.github/workflows/update-download-badges.yml +++ b/.github/workflows/update-download-badges.yml @@ -10,6 +10,12 @@ name: Update profile totals # PyPI - packages that belong to a repo (verified by a link back to # the repo or by the repo's packaging file). Rounded shields.io # figures by default; set PEPY_API_KEY for exact counts. +# npm - packages that belong to a repo (same verification, via the +# registry metadata or the repo's package.json). Exact all-time +# counts from the registry downloads API, summed over +# consecutive 18-month windows from the package's first publish +# day (npm has no data before 2015-01-10); falls back to the +# rounded shields.io /npm/dt badge if the API is down. # jsDelivr - shields.io /jsdelivr/gh/hy badge for every repo; hits over # the last 12 months (there is no all-time jsDelivr figure). # diff --git a/profile/README.md b/profile/README.md index 32f69f2..e674e3a 100644 --- a/profile/README.md +++ b/profile/README.md @@ -7,6 +7,8 @@ [![Static Badge](https://img.shields.io/badge/Facebook-Visit-white?logo=facebook&labelColor=blue)](https://www.facebook.com/Infinitode.LTD) [![Static Badge](https://img.shields.io/badge/YouTube-Subscribe-white?logo=youtube&labelColor=red)](https://www.youtube.com/@infinitodeltd) [![Static Badge](https://img.shields.io/badge/Huggingface-Visit-white?logo=huggingface&labelColor=white)](https://huggingface.co/InfinitodeLTD) +[![Static Badge](https://img.shields.io/badge/PyPI-View%20Packages-white?logo=pypi&labelColor=white)](https://pypi.org/user/Infinitode/) +[![Static Badge](https://img.shields.io/badge/npm-View%20Packages-white?logo=npm&labelColor=white)](https://www.npmjs.com/org/infinitodeltd) ![](https://komarev.com/ghpvc/?username=infinitode&color=5d17eb) ![Total Downloads](https://img.shields.io/badge/Total%20Downloads-124.7k-5d17eb) @@ -442,6 +444,13 @@ Welcome to Infinitode, a cutting-edge small technology company dedicated to deve ![jsDelivr hits (GitHub)](https://img.shields.io/jsdelivr/gh/hy/infinitode/cloakjs) ![GitHub stars](https://img.shields.io/github/stars/infinitode/cloakjs) ![GitHub last commit](https://img.shields.io/github/last-commit/infinitode/cloakjs) +[**TailClass**](https://github.com/Infinitode/TailClass) + +![npm version](https://img.shields.io/npm/v/tailclass) ![npm downloads](https://img.shields.io/npm/dt/tailclass) ![npm monthly downloads](https://img.shields.io/npm/dm/tailclass) ![GitHub stars](https://img.shields.io/github/stars/infinitode/tailclass) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/infinitode/tailclass) ![GitHub last commit](https://img.shields.io/github/last-commit/infinitode/tailclass) + +> [!NOTE] +> Our Python packages are published on PyPI under [**Infinitode**](https://pypi.org/user/Infinitode/), and our npm packages (including `TailClass`) are published on npm under [**infinitodeltd**](https://www.npmjs.com/org/infinitodeltd). + ## Our mission @@ -460,6 +469,7 @@ We're proud to showcase some of our current projects: * **PyWebScrapr**: An open-source Python web scraping tool supporting image and text scraping for effortless data collection. * **FuncProfiler**: An open-source Python library for identifying bottlenecks in code, featuring function profiling, data exports, logging, and line-by-line profiling for more granular control. * **CloakJS**: A single-file, dependency-free JavaScript obfuscator for the browser and Node.js, with identifier renaming, string encryption, and control-flow options. [View on GitHub](https://github.com/Infinitode/CloakJS). +* **TailClass**: An open-source, zero-dependency Tailwind CSS auto-classer for Node.js, with a CLI, smart semantic component naming, custom mapping dictionaries, and centralized stylesheet generation. [View on npm](https://www.npmjs.com/package/tailclass). > [!TIP] > For more projects, see: https://github.com/Infinitode?tab=repositories @@ -490,6 +500,8 @@ Follow us on social media to stay up-to-date with our latest developments, relea * X/Twitter: @infinitodeltd * YouTube: @infinitodeltd * Facebook: Infinitode.LTD +* PyPI: https://pypi.org/user/Infinitode/ +* npm: https://www.npmjs.com/org/infinitodeltd * Website: https://infinitode.netlify.app * Documentation: https://infinitode-docs.gitbook.io/documentation * FAQs: https://infinitode-docs.gitbook.io/faqs diff --git a/scripts/test_update_download_badges.py b/scripts/test_update_download_badges.py index 9f521ba..835d94b 100644 --- a/scripts/test_update_download_badges.py +++ b/scripts/test_update_download_badges.py @@ -19,6 +19,7 @@ import tempfile import unittest import urllib.error +from datetime import date, timedelta _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) @@ -70,6 +71,16 @@ def pypi_meta(home=None, description=""): } +def npm_meta(repository=None, homepage=None, created=None): + """npm registry metadata shaped like the real ``GET /{package}`` response.""" + latest = {"repository": {"type": "git", "url": repository} if repository else None, + "homepage": homepage, "bugs": None} + meta = {"dist-tags": {"latest": "1.0.2"}, "versions": {"1.0.2": latest}} + if created: + meta["time"] = {"created": created, "modified": created} + return meta + + def contents_file(name, text): """What GET /repos/{o}/{r}/contents/{file} returns for a file.""" return {"content": base64.b64encode(text.encode("utf-8")).decode("ascii"), "name": name} @@ -141,6 +152,7 @@ def standard_world(): repo("Infinitode", "Hued", stars=2, forks=1), repo("Infinitode", "qrforge"), repo("Infinitode", "CloakJS"), + repo("Infinitode", "TailClass"), repo("someone-else", "hued-fork", stars=50), # not ours, even if listed ] ), @@ -151,6 +163,18 @@ def standard_world(): "pypi.org/pypi/localshare/json": pypi_meta(home="https://github.com/sunu/localshare"), "pypi.org/pypi/alloy/json": http_error(404), "pypi.org/pypi/cloakjs/json": http_error(404), + "pypi.org/pypi/tailclass/json": http_error(404), + # npm: tailclass links back to our repo through a git+ URL; + # hued is somebody else's package; cloakjs and friends are not on npm. + "registry.npmjs.org/tailclass": npm_meta( + repository="git+https://github.com/Infinitode/TailClass.git", + created="2026-11-08T10:09:47.000Z", + ), + "registry.npmjs.org/hued": npm_meta(repository="https://github.com/someone-else/hued"), + "registry.npmjs.org/cloakjs": http_error(404), + "registry.npmjs.org/qrforge": http_error(404), + "registry.npmjs.org/localshare": http_error(404), + "registry.npmjs.org/alloy": http_error(404), "/repos/Infinitode/qrforge/contents/setup.py": contents_file( "setup.py", "setup(name='qrforge', version='1.0.1')" ), @@ -171,6 +195,7 @@ def standard_text_world(web): web.text.update( { "img.shields.io/pepy/dt/": lambda url, headers: shields_svg("downloads", "1.0k"), + "img.shields.io/npm/dt/": lambda url, headers: shields_svg("downloads", "350"), "img.shields.io/jsdelivr/gh/hy/infinitode/cloakjs": shields_svg("hits", "250"), "img.shields.io/jsdelivr/gh/hy/": shields_svg("hits", "0"), } @@ -238,6 +263,34 @@ def test_packaging_names(self): self.assertEqual(udb.packaging_names("pyproject.toml", '[project]\nname = "hued"\n'), {"hued"}) self.assertEqual(udb.packaging_names("setup.cfg", "[metadata]\nname = valx\n"), {"valx"}) + def test_npm_name_lowercases_the_repo_name(self): + self.assertEqual(udb.npm_name("TailClass"), "tailclass") + self.assertEqual(udb.npm_name("LocalSiteMap"), "localsitemap") + self.assertEqual(udb.npm_name("Scikit-learn-Model-Updater"), "scikit-learn-model-updater") + self.assertEqual(udb.npm_name("Not A Package"), "not-a-package") + + def test_npm_links_back_accepts_git_urls(self): + meta = npm_meta(repository="git+https://github.com/Infinitode/TailClass.git") + self.assertTrue(udb.npm_links_back(meta, "Infinitode", "TailClass")) + meta = npm_meta(repository="git+ssh://git@github.com/Infinitode/TailClass.git") + self.assertTrue(udb.npm_links_back(meta, "infinitode", "tailclass")) + self.assertTrue(udb.npm_links_back(npm_meta(homepage="https://github.com/Infinitode/TailClass"), "Infinitode", "TailClass")) + # someone else's package, and a longer repo name, must not match + self.assertFalse(udb.npm_links_back(npm_meta(repository="https://github.com/someone-else/hued"), "Infinitode", "Hued")) + self.assertFalse(udb.npm_links_back(npm_meta(homepage="https://github.com/infinitode/tailclass-tools"), "infinitode", "tailclass")) + + def test_npm_packaging_names_reads_package_json(self): + self.assertEqual(udb.npm_packaging_names("package.json", '{"name": "tailclass"}'), {"tailclass"}) + self.assertEqual(udb.npm_packaging_names("package.json", '{"version": "1.0.0"}'), set()) + self.assertEqual(udb.npm_packaging_names("package.json", "not json"), set()) + + def test_npm_first_day_reads_time_created(self): + meta = npm_meta(created="2026-11-08T10:09:47.000Z") + self.assertEqual(udb.npm_first_day(meta), date(2026, 11, 8)) + self.assertIsNone(udb.npm_first_day(npm_meta()), "no time block") + self.assertIsNone(udb.npm_first_day({"time": {"created": "not a date"}})) + self.assertIsNone(udb.npm_first_day({"time": {"created": ""}})) + class TestReadmeRewrite(unittest.TestCase): def setUp(self): @@ -294,7 +347,14 @@ def test_lists_only_public_repos_of_the_two_owners(self): names = sorted(r["full_name"] for r in udb.discover_repos()) self.assertEqual( names, - ["Infinitode/CloakJS", "Infinitode/Hued", "Infinitode/qrforge", "johanbeytell1/Alloy", "johanbeytell1/LocalShare"], + [ + "Infinitode/CloakJS", + "Infinitode/Hued", + "Infinitode/TailClass", + "Infinitode/qrforge", + "johanbeytell1/Alloy", + "johanbeytell1/LocalShare", + ], ) def test_listing_follows_pages(self): @@ -331,6 +391,199 @@ def test_pypi_server_error_is_a_note_not_a_crash(self): self.assertNotIn("Infinitode/Hued", packages) self.assertTrue(any("Infinitode/Hued" in n for n in notes)) + def test_npm_package_must_belong_to_the_repo(self): + pairs = { + "Infinitode/TailClass": "tailclass", # registry metadata links back to the repo + "Infinitode/Hued": None, # npm "hued" is someone else's package + "Infinitode/CloakJS": None, # not on npm at all + } + for full_name, expected in pairs.items(): + owner, name = full_name.split("/") + found = udb.npm_package_for(repo(owner, name), fetch_json=self.web.fetch_json) + self.assertEqual(found, expected, full_name) + + def test_npm_package_can_be_declared_by_package_json(self): + web = FakeWeb() + web.json["registry.npmjs.org/widget"] = npm_meta(homepage="https://example.com") + web.json["/repos/Infinitode/Widget/contents"] = [ + { + "type": "file", + "name": "package.json", + "url": "https://api.github.com/repos/Infinitode/Widget/contents/package.json", + } + ] + web.json["/repos/Infinitode/Widget/contents/package.json"] = contents_file( + "package.json", '{"name": "widget"}' + ) + found = udb.npm_package_for(repo("Infinitode", "Widget"), fetch_json=web.fetch_json) + self.assertEqual(found, "widget") + + web.json["/repos/Infinitode/Widget/contents/package.json"] = contents_file( + "package.json", '{"name": "something-else"}' + ) + self.assertIsNone(udb.npm_package_for(repo("Infinitode", "Widget"), fetch_json=web.fetch_json)) + + def test_npm_metadata_pointing_at_another_repo_beats_a_name_match(self): + """An unrelated package that merely shares our repo's name is not ours. + + npm's ``localshare`` is someone else's 2019 package while our + ``johanbeytell1/LocalShare`` repo's package.json declares that name. + """ + web = FakeWeb() + web.json["registry.npmjs.org/localshare"] = npm_meta( + repository="git+https://github.com/localshare/localshare.git" + ) + web.json["/repos/johanbeytell1/LocalShare/contents"] = [ + { + "type": "file", + "name": "package.json", + "url": "https://api.github.com/repos/johanbeytell1/LocalShare/contents/package.json", + } + ] + web.json["/repos/johanbeytell1/LocalShare/contents/package.json"] = contents_file( + "package.json", '{"name": "localshare"}' + ) + found = udb.npm_package_for( + repo("johanbeytell1", "LocalShare"), fetch_json=web.fetch_json + ) + self.assertIsNone(found) + + def test_discover_npm_maps_repos_to_packages(self): + packages, first_days, notes = udb.discover_npm( + udb.discover_repos(), fetch_json=self.web.fetch_json + ) + self.assertEqual(packages, {"Infinitode/TailClass": "tailclass"}) + self.assertEqual(first_days, {"tailclass": date(2026, 11, 8)}, "lifetime start is kept") + self.assertEqual(notes, []) + + def test_npm_server_error_is_a_note_not_a_crash(self): + self.web.json["registry.npmjs.org/tailclass"] = http_error(500) + packages, _, notes = udb.discover_npm(udb.discover_repos(), fetch_json=self.web.fetch_json) + self.assertNotIn("Infinitode/TailClass", packages) + self.assertTrue(any("Infinitode/TailClass" in n for n in notes)) + + +# --- npm downloads ------------------------------------------------------------ + + +class TestNpmDownloads(StubMixin, unittest.TestCase): + def setUp(self): + self.web = FakeWeb() + self.install(self.web) + + @staticmethod + def _window(url): + span = url.split("/point/", 1)[1].split("/", 1)[0] + start, end = span.split(":") + return date.fromisoformat(start), date.fromisoformat(end) + + def test_all_time_total_is_summed_over_18_month_windows(self): + calls = [] + + def respond(url, headers): + calls.append(url) + return {"downloads": 10} + + self.web.json["api.npmjs.org/downloads/point/"] = respond + total = udb.npm_range_total("tailclass", today=date(2019, 1, 1)) + self.assertGreater(len(calls), 1, "the API caps a query at 18 months") + self.assertEqual(total, 10 * len(calls)) + windows = [self._window(url) for url in calls] + self.assertEqual(windows[0][0], udb.NPM_FIRST_DAY) + self.assertEqual(windows[-1][1], date(2019, 1, 1)) + for start, end in windows: + self.assertLessEqual((end - start).days + 1, udb.NPM_WINDOW_DAYS) + for (_, previous_end), (next_start, _) in zip(windows, windows[1:]): + self.assertEqual(next_start, previous_end + timedelta(days=1), "no gaps or overlaps") + + def test_a_short_history_needs_a_single_window(self): + calls = [] + + def respond(url, headers): + calls.append(url) + return {"downloads": 7} + + self.web.json["api.npmjs.org/downloads/point/"] = respond + total = udb.npm_range_total("tailclass", today=date(2015, 1, 11)) + self.assertEqual(len(calls), 1) + self.assertEqual(total, 7) + + def test_walk_starts_at_the_first_publish_day(self): + calls = [] + + def respond(url, headers): + calls.append(url) + return {"downloads": 4} + + self.web.json["api.npmjs.org/downloads/point/"] = respond + total = udb.npm_range_total( + "tailclass", today=date(2026, 11, 11), first_day=date(2026, 11, 8) + ) + self.assertEqual(len(calls), 1, "a four-day-old package needs a single query") + self.assertEqual(self._window(calls[0]), (date(2026, 11, 8), date(2026, 11, 11))) + self.assertEqual(total, 4) + + def test_first_day_before_npm_data_is_clamped_to_the_floor(self): + calls = [] + + def respond(url, headers): + calls.append(url) + return {"downloads": 1} + + self.web.json["api.npmjs.org/downloads/point/"] = respond + udb.npm_range_total("python-era", today=date(2015, 2, 1), first_day=date(2010, 1, 1)) + self.assertEqual(self._window(calls[0])[0], udb.NPM_FIRST_DAY) + + def test_a_package_published_today_queries_only_today(self): + calls = [] + + def respond(url, headers): + calls.append(url) + return {"downloads": 99} + + self.web.json["api.npmjs.org/downloads/point/"] = respond + totals, failures = udb.npm_totals( + ["tailclass"], today=date(2026, 11, 11), first_days={"tailclass": date(2026, 11, 11)} + ) + self.assertEqual(len(calls), 1) + self.assertEqual(self._window(calls[0]), (date(2026, 11, 11), date(2026, 11, 11))) + self.assertEqual(totals, {"tailclass": 99}) + self.assertEqual(failures, {}) + + def test_a_first_day_after_today_queries_nothing(self): + calls = [] + + def respond(url, headers): + calls.append(url) + return {"downloads": 99} + + self.web.json["api.npmjs.org/downloads/point/"] = respond + totals, failures = udb.npm_totals( + ["tailclass"], today=date(2026, 11, 11), first_days={"tailclass": date(2026, 12, 1)} + ) + self.assertEqual(calls, []) + self.assertEqual(totals, {"tailclass": 0}) + self.assertEqual(failures, {}) + + def test_a_window_without_a_count_raises_instead_of_undercounting(self): + self.web.json["api.npmjs.org/downloads/point/"] = {"unexpected": 1} + with self.assertRaises(ValueError): + udb.npm_range_total("tailclass", today=date(2015, 1, 11)) + + def test_totals_fall_back_to_shields_when_the_api_fails(self): + self.web.json["api.npmjs.org/downloads/point/"] = urllib.error.URLError("offline") + self.web.text["img.shields.io/npm/dt/"] = shields_svg("downloads", "1.2k") + totals, failures = udb.npm_totals(["tailclass"], today=date(2015, 1, 11)) + self.assertEqual(totals, {"tailclass": 1200}) + self.assertEqual(failures, {}) + + def test_totals_report_a_failure_when_every_source_fails(self): + self.web.json["api.npmjs.org/downloads/point/"] = urllib.error.URLError("offline") + self.web.text["img.shields.io/npm/dt/"] = urllib.error.URLError("offline") + totals, failures = udb.npm_totals(["tailclass"], today=date(2015, 1, 11)) + self.assertEqual(totals, {}) + self.assertIn("tailclass", failures) + # --- collection --------------------------------------------------------------- @@ -339,18 +592,30 @@ class TestCollect(StubMixin, unittest.TestCase): def setUp(self): self.web = standard_text_world(standard_world()) self.web.json["api.pepy.tech"] = lambda url, headers: {"total_downloads": 1_000} + self.web.json["api.npmjs.org/downloads/point/"] = {"downloads": 100} self.install(self.web) + def _collect(self): + # A fixed "today" keeps the npm window count (and so the total) stable. + # tailclass was published 2026-11-08, so its lifetime is a single window. + return udb.collect( + api_key="k", + fetch_json=self.web.fetch_json, + fetch_text=self.web.fetch_text, + today=date(2026, 11, 11), + ) + def test_stars_and_forks_are_summed_over_counted_repos(self): - result = udb.collect(api_key="k", fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text) + result = self._collect() self.assertEqual(result["stars"], 1 + 1 + 2) # LocalShare, Alloy, Hued; Secret and the foreign fork excluded self.assertEqual(result["forks"], 1) - def test_downloads_come_from_pypi_and_jsdelivr(self): - result = udb.collect(api_key="k", fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text) + def test_downloads_come_from_pypi_npm_and_jsdelivr(self): + result = self._collect() self.assertEqual(result["pypi"], {"hued": 1_000, "qrforge": 1_000}) + self.assertEqual(result["npm"], {"tailclass": 100}) self.assertEqual(result["jsdelivr"]["infinitode/cloakjs"], 250) - self.assertEqual(result["downloads"], 1_000 + 1_000 + 250) + self.assertEqual(result["downloads"], 1_000 + 1_000 + 100 + 250) self.assertNotIn("localshare", result["pypi"]) @@ -396,6 +661,8 @@ def _run(self, web, argv=(), env=None): def _healthy_web(self): web = standard_text_world(standard_world()) web.json["api.pepy.tech"] = lambda url, headers: {"total_downloads": 1_000} + # Force the npm fallback so the total does not depend on today's date. + web.json["api.npmjs.org/downloads/point/"] = http_error(500) return web def test_listing_failure_changes_nothing(self): @@ -425,8 +692,9 @@ def test_tokenless_run_writes_downloads_stars_and_forks(self): def test_happy_path_with_pepy_key_is_exact(self): code, output = self._run(self._healthy_web(), env={"PEPY_API_KEY": "k"}) self.assertEqual(code, 0, output) - # PyPI: hued + qrforge at 1,000 each (pepy); jsDelivr: CloakJS 250, the rest 0. - self.assertIn(f"Total%20Downloads-{udb.format_count(2_250)}-", self._read(self.readme)) + # PyPI: hued + qrforge at 1,000 each (pepy); npm: tailclass 350 from the + # shields fallback; jsDelivr: CloakJS 250, the rest 0. + self.assertIn(f"Total%20Downloads-{udb.format_count(2_600)}-", self._read(self.readme)) def test_dry_run_changes_nothing(self): code, output = self._run(self._healthy_web(), argv=["--dry-run"], env={"PEPY_API_KEY": "k"}) diff --git a/scripts/update-download-badges.py b/scripts/update-download-badges.py index 5b3fb1c..aff1fcb 100644 --- a/scripts/update-download-badges.py +++ b/scripts/update-download-badges.py @@ -17,6 +17,16 @@ Same-named packages by other people fail both checks, so they are skipped. 1. pepy v2 API when PEPY_API_KEY is set (exact counts), otherwise 2. shields.io ``/pepy/dt`` badge (rounded figures). + npm (all-time). Same ownership rule, lowercased (npm names are lowercase): + an npm package belongs to a repo when its name matches the repo's and either + its registry metadata links back to the repo, or -- only when that metadata + names no GitHub repository at all -- the repo's package.json declares the + name. Metadata pointing at a different repo is always rejected. + 1. registry downloads API, summed over 18-month windows from the package's + first publish day (a single query may span at most 18 months, and npm + has no data before 2015-01-10, so that is the floor), + 2. shields.io ``/npm/dt`` badge when that is unavailable. Note shields + builds that badge from a capped range, so the fallback can undercount. jsDelivr (last 12 months). shields.io ``/jsdelivr/gh/hy`` for every counted repo. jsDelivr publishes period stats only; there is no all-time figure. @@ -48,6 +58,7 @@ import sys import urllib.error import urllib.request +from datetime import date, timedelta # --- what gets counted ------------------------------------------------------- @@ -61,15 +72,24 @@ GITHUB_HEADERS = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"} MAX_PAGES = 20 PACKAGING_FILES = ("pyproject.toml", "setup.py", "setup.cfg") +NPM_PACKAGING_FILES = ("package.json",) PYPI_JSON = "https://pypi.org/pypi/{pkg}/json" PEPY_V2 = "https://api.pepy.tech/api/v2/projects/{pkg}" SHIELDS_PYPI = "https://img.shields.io/pepy/dt/{pkg}" +SHIELDS_NPM = "https://img.shields.io/npm/dt/{package}" SHIELDS_JSDELIVR = "https://img.shields.io/jsdelivr/gh/{period}/{repo}" +NPM_REGISTRY = "https://registry.npmjs.org/{package}" +NPM_DOWNLOADS_POINT = "https://api.npmjs.org/downloads/point/{start}:{end}/{package}" +# The point API refuses ranges longer than 18 months, and has no data before +# this date, so all-time npm totals have to be summed over windows. +NPM_FIRST_DAY = date(2015, 1, 10) +NPM_WINDOW_DAYS = 540 # safely under the 18-month (~547 day) limit + # A new total this far below the published one counts as partial data and is # refused. 5% absorbs display rounding while still catching a missing source -# (dropping one of eleven sources moves the total by ~9%). +# (losing one source, or one chunk of an npm total, moves the total by more). DROP_TOLERANCE = 0.05 # Keys that hold an all-time count, highest confidence first. @@ -92,6 +112,7 @@ _PY_NAME_RE = re.compile(r"""\bname\s*=\s*["']([A-Za-z0-9][A-Za-z0-9._-]*)["']""") _CFG_NAME_RE = re.compile(r"^name\s*=\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*$", re.M) +_GITHUB_URL_RE = re.compile(r"github\.com[/:]([^/\s]+)/([^/\s#?]+)", re.I) # --- network (the only I/O in this module) ----------------------------------- @@ -224,6 +245,70 @@ def packaging_names(filename, text): return set(_PY_NAME_RE.findall(text)) +def npm_name(repo_name): + """The npm package name a repo's name implies. + + npm names are lowercase and may only hold ``a-z 0-9 . _ -``, so the + PascalCase repo names map straight across (TailClass -> tailclass). + """ + return re.sub(r"[^a-z0-9._-]+", "-", repo_name.strip().lower()).strip("-._") + + +def normalize_npm_name(name): + """npm package names are lowercase; compare them that way.""" + return name.strip().lower() + + +def npm_github_repos(meta): + """Every ``owner/repo`` the npm package's metadata points at on GitHub.""" + version = (meta.get("dist-tags") or {}).get("latest") or "" + latest = (meta.get("versions") or {}).get(version) or meta + urls = [] + for key in ("repository", "homepage", "bugs"): + for source in (latest, meta): + value = source.get(key) + if isinstance(value, dict): + value = value.get("url") + if isinstance(value, str) and value: + urls.append(value) + # git+https://github.com/o/r.git and git+ssh://git@github.com:o/r both count + text = re.sub(r"\.git\b", "", " ".join(urls), flags=re.I) + return { + (match.group(1).lower(), match.group(2).lower()) + for match in _GITHUB_URL_RE.finditer(text) + } + + +def npm_links_back(meta, owner, repo): + """True if the npm package's metadata links to github.com/owner/repo.""" + return (owner.lower(), repo.lower()) in npm_github_repos(meta) + + +def npm_first_day(meta): + """The day the npm package was first published, or ``None`` if unknown. + + Registry metadata carries a ``time.created`` timestamp. Downloads cannot + predate it, so it is the earliest window worth asking the API for. + """ + created = (meta.get("time") or {}).get("created") + if not isinstance(created, str) or len(created) < 10: + return None + try: + return date.fromisoformat(created[:10]) + except ValueError: + return None + + +def npm_packaging_names(filename, text): + """Package names a package.json declares.""" + try: + meta = json.loads(text) + except ValueError: + return set() + name = meta.get("name") if isinstance(meta, dict) else None + return {name} if isinstance(name, str) and name.strip() else set() + + def render_block(values): """The badge block. *values* is a list of ``(label, badge_text)`` pairs.""" lines = [START_MARK] @@ -277,9 +362,16 @@ def discover_repos(fetch_json=None, token=None): return repos -def declared_names(owner, repo, fetch_json=None, token=None): - """Package names declared by the repo's own packaging files.""" +def declared_names( + owner, repo, fetch_json=None, token=None, files=PACKAGING_FILES, parse=None +): + """Package names declared by the repo's own packaging files. + + *files* and *parse* select the ecosystem: Python's packaging files by + default, or ``NPM_PACKAGING_FILES`` with ``npm_packaging_names``. + """ fetch_json = fetch_json or globals()["fetch_json"] + parse = parse or packaging_names try: listing = fetch_json(f"{GITHUB_API}/repos/{owner}/{repo}/contents", github_headers(token)) except urllib.error.HTTPError as error: @@ -288,10 +380,10 @@ def declared_names(owner, repo, fetch_json=None, token=None): raise names = set() for item in listing if isinstance(listing, list) else []: - if item.get("type") == "file" and item.get("name") in PACKAGING_FILES: + if item.get("type") == "file" and item.get("name") in files: blob = fetch_json(item["url"], github_headers(token)) text = base64.b64decode(blob.get("content", "")).decode("utf-8", "replace") - names |= packaging_names(item["name"], text) + names |= parse(item["name"], text) return names @@ -326,6 +418,71 @@ def discover_pypi(repos, fetch_json=None, token=None): return packages, notes +def npm_lookup(repo, fetch_json=None, token=None): + """``(package, first_day)`` for the npm package that belongs to *repo*. + + Both are ``None`` when the repo has no package of its own. + """ + fetch_json = fetch_json or globals()["fetch_json"] + owner, name = repo["owner"]["login"], repo["name"] + package = npm_name(name) + if not package: + return None, None + try: + meta = fetch_json(NPM_REGISTRY.format(package=package)) + except urllib.error.HTTPError as error: + if error.code == 404: + return None, None + raise + if not isinstance(meta, dict): + raise ValueError(f"unexpected npm metadata for {package}") + if not npm_links_back(meta, owner, name): + # Provenance wins over a name match: a package that names some *other* + # GitHub repo in its metadata is not ours, even if our package.json + # happens to declare the same name (npm/local-share style collisions). + if npm_github_repos(meta): + return None, None + declared = { + normalize_npm_name(n) + for n in declared_names( + owner, + name, + fetch_json, + token, + files=NPM_PACKAGING_FILES, + parse=npm_packaging_names, + ) + } + if package not in declared: + return None, None + return package, npm_first_day(meta) + + +def npm_package_for(repo, fetch_json=None, token=None): + """The npm package that belongs to *repo*, or ``None``.""" + return npm_lookup(repo, fetch_json=fetch_json, token=token)[0] + + +def discover_npm(repos, fetch_json=None, token=None): + """Map ``full_name -> npm package``, plus each package's first day. + + Returns ``(packages, first_days, notes)``, where *first_days* maps a package + name to the day it was first published (used to walk only its lifetime). + """ + packages, first_days, notes = {}, {}, [] + for repo in repos: + try: + package, first_day = npm_lookup(repo, fetch_json=fetch_json, token=token) + except (urllib.error.URLError, ValueError) as error: + notes.append(f"{repo['full_name']}: npm lookup skipped ({error})") + continue + if package: + packages[repo["full_name"]] = package + if first_day: + first_days[package] = first_day + return packages, first_days, notes + + # --- source lookups ---------------------------------------------------------- @@ -362,6 +519,64 @@ def pypi_totals(packages, api_key=None, fetch_json=None, fetch_text=None): return totals, failures, sorted(set(notes)) +def npm_range_total(package, fetch_json=None, today=None, first_day=None): + """All-time npm downloads for *package*, summed over 18-month windows. + + The walk starts at the package's first publish day (nothing can predate it) + and never earlier than 2015-01-10. The point API refuses any single range + longer than 18 months, so the lifetime is covered in windows. A failed or + malformed window raises instead of returning a small partial sum. + """ + fetch_json = fetch_json or globals()["fetch_json"] + last_day = today or date.today() + start = max(NPM_FIRST_DAY, first_day) if first_day else NPM_FIRST_DAY + total = 0 + while start <= last_day: + end = min(start + timedelta(days=NPM_WINDOW_DAYS - 1), last_day) + payload = fetch_json( + NPM_DOWNLOADS_POINT.format(package=package, start=start.isoformat(), end=end.isoformat()) + ) + count = find_count(payload) + if count is None: + raise ValueError(f"no download count in response for {package}") + total += count + start = end + timedelta(days=1) + return total + + +def npm_totals(packages, fetch_json=None, fetch_text=None, today=None, first_days=None): + """All-time npm downloads per package, exact where the API answers. + + *first_days* maps a package to its first publish day so that only its own + lifetime is queried; packages missing from it are walked from npm's floor. + """ + fetch_json = fetch_json or globals()["fetch_json"] + fetch_text = fetch_text or globals()["fetch_text"] + first_days = first_days or {} + totals, failures = {}, {} + for package in packages: + errors = [] + count = None + try: + count = npm_range_total( + package, fetch_json=fetch_json, today=today, first_day=first_days.get(package) + ) + except (urllib.error.URLError, ValueError) as error: + errors.append(f"npm api -> {error}") + if count is None: # fallback: shields.io builds this from a capped range + try: + count = parse_shields_badge(fetch_text(SHIELDS_NPM.format(package=package))) + except (urllib.error.URLError, ValueError) as error: + errors.append(f"shields.io -> {error}") + if count is None and not any("shields.io" in e for e in errors): + errors.append("shields.io -> no numeric value in badge") + if count is not None: + totals[package] = count + else: + failures[package] = "; ".join(errors) or "unknown error" + return totals, failures + + def jsdelivr_totals(repos, period=JSDELIVR_PERIOD, fetch_text=None): """jsDelivr hits per repo (``owner/repo``) for *period* (default: 12 months).""" fetch_text = fetch_text or globals()["fetch_text"] @@ -383,7 +598,7 @@ def jsdelivr_totals(repos, period=JSDELIVR_PERIOD, fetch_text=None): # --- collection -------------------------------------------------------------- -def collect(api_key=None, github_token=None, fetch_json=None, fetch_text=None): +def collect(api_key=None, github_token=None, fetch_json=None, fetch_text=None, today=None): """Gather every download, star and fork figure.""" fetch_json = fetch_json or globals()["fetch_json"] fetch_text = fetch_text or globals()["fetch_text"] @@ -401,6 +616,18 @@ def collect(api_key=None, github_token=None, fetch_json=None, fetch_text=None): ) notes += pypi_notes + npm_packages, npm_first_days, npm_notes = discover_npm( + repos, fetch_json=fetch_json, token=github_token + ) + notes += npm_notes + npm, npm_failures = npm_totals( + sorted(set(npm_packages.values())), + fetch_json=fetch_json, + fetch_text=fetch_text, + today=today, + first_days=npm_first_days, + ) + jsdelivr, jsdelivr_failures = jsdelivr_totals( [repo["full_name"].lower() for repo in repos], fetch_text=fetch_text ) @@ -410,12 +637,14 @@ def collect(api_key=None, github_token=None, fetch_json=None, fetch_text=None): "stars": stars, "forks": forks, "packages": packages, + "npm_packages": npm_packages, "pypi": pypi, + "npm": npm, "jsdelivr": jsdelivr, - "failures": {**pypi_failures, **jsdelivr_failures}, + "failures": {**pypi_failures, **npm_failures, **jsdelivr_failures}, "notes": sorted(set(notes)), - "download_sources": len(pypi) + len(jsdelivr), - "downloads": sum(pypi.values()) + sum(jsdelivr.values()), + "download_sources": len(pypi) + len(npm) + len(jsdelivr), + "downloads": sum(pypi.values()) + sum(npm.values()) + sum(jsdelivr.values()), } @@ -447,8 +676,12 @@ def main(argv=None): print(f" repos {len(result['repos'])} public repos counted: {', '.join(sorted(result['repos']))}") for repo, package in sorted(result["packages"].items()): print(f" package {repo:<40} -> pypi/{package}") + for repo, package in sorted(result["npm_packages"].items()): + print(f" package {repo:<40} -> npm/{package}") for name, count in sorted(result["pypi"].items()): print(f" pypi {name:<29} {count:>14,}") + for name, count in sorted(result["npm"].items()): + print(f" npm {name:<29} {count:>14,}") for name, count in sorted(result["jsdelivr"].items()): print(f" jsdelivr {name:<29} {count:>14,}") for name, reason in sorted(result["failures"].items()):