From 62efb0424223707d9d3b25bf4ca5559e852ed065 Mon Sep 17 00:00:00 2001 From: johanbeytell1 <107624055+johanbeytell1@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:33:53 +0000 Subject: [PATCH 1/2] docs: list TailClass and link our PyPI and npm registries Add TailClass to the projects list and the package status section, with npm version/downloads/star badges. Add top badges and Stay Updated links for the PyPI maintainer profile (Infinitode) and the npm org (infinitodeltd). Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- profile/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/profile/README.md b/profile/README.md index 32f69f2..e674e3a 100644 --- a/profile/README.md +++ b/profile/README.md @@ -7,6 +7,8 @@ [![Static Badge](https://img.shields.io/badge/Facebook-Visit-white?logo=facebook&labelColor=blue)](https://www.facebook.com/Infinitode.LTD) [![Static Badge](https://img.shields.io/badge/YouTube-Subscribe-white?logo=youtube&labelColor=red)](https://www.youtube.com/@infinitodeltd) [![Static Badge](https://img.shields.io/badge/Huggingface-Visit-white?logo=huggingface&labelColor=white)](https://huggingface.co/InfinitodeLTD) +[![Static Badge](https://img.shields.io/badge/PyPI-View%20Packages-white?logo=pypi&labelColor=white)](https://pypi.org/user/Infinitode/) +[![Static Badge](https://img.shields.io/badge/npm-View%20Packages-white?logo=npm&labelColor=white)](https://www.npmjs.com/org/infinitodeltd) ![](https://komarev.com/ghpvc/?username=infinitode&color=5d17eb) ![Total Downloads](https://img.shields.io/badge/Total%20Downloads-124.7k-5d17eb) @@ -442,6 +444,13 @@ Welcome to Infinitode, a cutting-edge small technology company dedicated to deve ![jsDelivr hits (GitHub)](https://img.shields.io/jsdelivr/gh/hy/infinitode/cloakjs) ![GitHub stars](https://img.shields.io/github/stars/infinitode/cloakjs) ![GitHub last commit](https://img.shields.io/github/last-commit/infinitode/cloakjs) +[**TailClass**](https://github.com/Infinitode/TailClass) + +![npm version](https://img.shields.io/npm/v/tailclass) ![npm downloads](https://img.shields.io/npm/dt/tailclass) ![npm monthly downloads](https://img.shields.io/npm/dm/tailclass) ![GitHub stars](https://img.shields.io/github/stars/infinitode/tailclass) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/infinitode/tailclass) ![GitHub last commit](https://img.shields.io/github/last-commit/infinitode/tailclass) + +> [!NOTE] +> Our Python packages are published on PyPI under [**Infinitode**](https://pypi.org/user/Infinitode/), and our npm packages (including `TailClass`) are published on npm under [**infinitodeltd**](https://www.npmjs.com/org/infinitodeltd). + ## Our mission @@ -460,6 +469,7 @@ We're proud to showcase some of our current projects: * **PyWebScrapr**: An open-source Python web scraping tool supporting image and text scraping for effortless data collection. * **FuncProfiler**: An open-source Python library for identifying bottlenecks in code, featuring function profiling, data exports, logging, and line-by-line profiling for more granular control. * **CloakJS**: A single-file, dependency-free JavaScript obfuscator for the browser and Node.js, with identifier renaming, string encryption, and control-flow options. [View on GitHub](https://github.com/Infinitode/CloakJS). +* **TailClass**: An open-source, zero-dependency Tailwind CSS auto-classer for Node.js, with a CLI, smart semantic component naming, custom mapping dictionaries, and centralized stylesheet generation. [View on npm](https://www.npmjs.com/package/tailclass). > [!TIP] > For more projects, see: https://github.com/Infinitode?tab=repositories @@ -490,6 +500,8 @@ Follow us on social media to stay up-to-date with our latest developments, relea * X/Twitter: @infinitodeltd * YouTube: @infinitodeltd * Facebook: Infinitode.LTD +* PyPI: https://pypi.org/user/Infinitode/ +* npm: https://www.npmjs.com/org/infinitodeltd * Website: https://infinitode.netlify.app * Documentation: https://infinitode-docs.gitbook.io/documentation * FAQs: https://infinitode-docs.gitbook.io/faqs From ba00522d4d3b257784402c832d78710907a0a053 Mon Sep 17 00:00:00 2001 From: johanbeytell1 <107624055+johanbeytell1@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:33:53 +0000 Subject: [PATCH 2/2] feat: count npm downloads in the profile totals Discover npm packages from the public repos the same way PyPI packages are discovered: the repo name lowercased, accepted only when the registry metadata links back to that exact repo or, when the metadata names no GitHub repo at all, the repo's package.json declares the name. Metadata pointing at a different repo always loses (npm 'localshare' style collisions). All-time downloads come from the registry point API, summed over consecutive 540-day windows from the package's first publish day (18 months per query is the API limit, 2015-01-10 is the oldest data). A failed or malformed window raises rather than contributing a partial sum, so the drop guard still works. The shields.io /npm/dt badge is the fallback, and npm now counts toward download_sources and the published total. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/update-download-badges.yml | 6 + scripts/test_update_download_badges.py | 282 ++++++++++++++++++- scripts/update-download-badges.py | 251 ++++++++++++++++- 3 files changed, 523 insertions(+), 16 deletions(-) diff --git a/.github/workflows/update-download-badges.yml b/.github/workflows/update-download-badges.yml index 6b57637..c27004a 100644 --- a/.github/workflows/update-download-badges.yml +++ b/.github/workflows/update-download-badges.yml @@ -10,6 +10,12 @@ name: Update profile totals # PyPI - packages that belong to a repo (verified by a link back to # the repo or by the repo's packaging file). Rounded shields.io # figures by default; set PEPY_API_KEY for exact counts. +# npm - packages that belong to a repo (same verification, via the +# registry metadata or the repo's package.json). Exact all-time +# counts from the registry downloads API, summed over +# consecutive 18-month windows from the package's first publish +# day (npm has no data before 2015-01-10); falls back to the +# rounded shields.io /npm/dt badge if the API is down. # jsDelivr - shields.io /jsdelivr/gh/hy badge for every repo; hits over # the last 12 months (there is no all-time jsDelivr figure). # diff --git a/scripts/test_update_download_badges.py b/scripts/test_update_download_badges.py index 9f521ba..835d94b 100644 --- a/scripts/test_update_download_badges.py +++ b/scripts/test_update_download_badges.py @@ -19,6 +19,7 @@ import tempfile import unittest import urllib.error +from datetime import date, timedelta _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) @@ -70,6 +71,16 @@ def pypi_meta(home=None, description=""): } +def npm_meta(repository=None, homepage=None, created=None): + """npm registry metadata shaped like the real ``GET /{package}`` response.""" + latest = {"repository": {"type": "git", "url": repository} if repository else None, + "homepage": homepage, "bugs": None} + meta = {"dist-tags": {"latest": "1.0.2"}, "versions": {"1.0.2": latest}} + if created: + meta["time"] = {"created": created, "modified": created} + return meta + + def contents_file(name, text): """What GET /repos/{o}/{r}/contents/{file} returns for a file.""" return {"content": base64.b64encode(text.encode("utf-8")).decode("ascii"), "name": name} @@ -141,6 +152,7 @@ def standard_world(): repo("Infinitode", "Hued", stars=2, forks=1), repo("Infinitode", "qrforge"), repo("Infinitode", "CloakJS"), + repo("Infinitode", "TailClass"), repo("someone-else", "hued-fork", stars=50), # not ours, even if listed ] ), @@ -151,6 +163,18 @@ def standard_world(): "pypi.org/pypi/localshare/json": pypi_meta(home="https://github.com/sunu/localshare"), "pypi.org/pypi/alloy/json": http_error(404), "pypi.org/pypi/cloakjs/json": http_error(404), + "pypi.org/pypi/tailclass/json": http_error(404), + # npm: tailclass links back to our repo through a git+ URL; + # hued is somebody else's package; cloakjs and friends are not on npm. + "registry.npmjs.org/tailclass": npm_meta( + repository="git+https://github.com/Infinitode/TailClass.git", + created="2026-11-08T10:09:47.000Z", + ), + "registry.npmjs.org/hued": npm_meta(repository="https://github.com/someone-else/hued"), + "registry.npmjs.org/cloakjs": http_error(404), + "registry.npmjs.org/qrforge": http_error(404), + "registry.npmjs.org/localshare": http_error(404), + "registry.npmjs.org/alloy": http_error(404), "/repos/Infinitode/qrforge/contents/setup.py": contents_file( "setup.py", "setup(name='qrforge', version='1.0.1')" ), @@ -171,6 +195,7 @@ def standard_text_world(web): web.text.update( { "img.shields.io/pepy/dt/": lambda url, headers: shields_svg("downloads", "1.0k"), + "img.shields.io/npm/dt/": lambda url, headers: shields_svg("downloads", "350"), "img.shields.io/jsdelivr/gh/hy/infinitode/cloakjs": shields_svg("hits", "250"), "img.shields.io/jsdelivr/gh/hy/": shields_svg("hits", "0"), } @@ -238,6 +263,34 @@ def test_packaging_names(self): self.assertEqual(udb.packaging_names("pyproject.toml", '[project]\nname = "hued"\n'), {"hued"}) self.assertEqual(udb.packaging_names("setup.cfg", "[metadata]\nname = valx\n"), {"valx"}) + def test_npm_name_lowercases_the_repo_name(self): + self.assertEqual(udb.npm_name("TailClass"), "tailclass") + self.assertEqual(udb.npm_name("LocalSiteMap"), "localsitemap") + self.assertEqual(udb.npm_name("Scikit-learn-Model-Updater"), "scikit-learn-model-updater") + self.assertEqual(udb.npm_name("Not A Package"), "not-a-package") + + def test_npm_links_back_accepts_git_urls(self): + meta = npm_meta(repository="git+https://github.com/Infinitode/TailClass.git") + self.assertTrue(udb.npm_links_back(meta, "Infinitode", "TailClass")) + meta = npm_meta(repository="git+ssh://git@github.com/Infinitode/TailClass.git") + self.assertTrue(udb.npm_links_back(meta, "infinitode", "tailclass")) + self.assertTrue(udb.npm_links_back(npm_meta(homepage="https://github.com/Infinitode/TailClass"), "Infinitode", "TailClass")) + # someone else's package, and a longer repo name, must not match + self.assertFalse(udb.npm_links_back(npm_meta(repository="https://github.com/someone-else/hued"), "Infinitode", "Hued")) + self.assertFalse(udb.npm_links_back(npm_meta(homepage="https://github.com/infinitode/tailclass-tools"), "infinitode", "tailclass")) + + def test_npm_packaging_names_reads_package_json(self): + self.assertEqual(udb.npm_packaging_names("package.json", '{"name": "tailclass"}'), {"tailclass"}) + self.assertEqual(udb.npm_packaging_names("package.json", '{"version": "1.0.0"}'), set()) + self.assertEqual(udb.npm_packaging_names("package.json", "not json"), set()) + + def test_npm_first_day_reads_time_created(self): + meta = npm_meta(created="2026-11-08T10:09:47.000Z") + self.assertEqual(udb.npm_first_day(meta), date(2026, 11, 8)) + self.assertIsNone(udb.npm_first_day(npm_meta()), "no time block") + self.assertIsNone(udb.npm_first_day({"time": {"created": "not a date"}})) + self.assertIsNone(udb.npm_first_day({"time": {"created": ""}})) + class TestReadmeRewrite(unittest.TestCase): def setUp(self): @@ -294,7 +347,14 @@ def test_lists_only_public_repos_of_the_two_owners(self): names = sorted(r["full_name"] for r in udb.discover_repos()) self.assertEqual( names, - ["Infinitode/CloakJS", "Infinitode/Hued", "Infinitode/qrforge", "johanbeytell1/Alloy", "johanbeytell1/LocalShare"], + [ + "Infinitode/CloakJS", + "Infinitode/Hued", + "Infinitode/TailClass", + "Infinitode/qrforge", + "johanbeytell1/Alloy", + "johanbeytell1/LocalShare", + ], ) def test_listing_follows_pages(self): @@ -331,6 +391,199 @@ def test_pypi_server_error_is_a_note_not_a_crash(self): self.assertNotIn("Infinitode/Hued", packages) self.assertTrue(any("Infinitode/Hued" in n for n in notes)) + def test_npm_package_must_belong_to_the_repo(self): + pairs = { + "Infinitode/TailClass": "tailclass", # registry metadata links back to the repo + "Infinitode/Hued": None, # npm "hued" is someone else's package + "Infinitode/CloakJS": None, # not on npm at all + } + for full_name, expected in pairs.items(): + owner, name = full_name.split("/") + found = udb.npm_package_for(repo(owner, name), fetch_json=self.web.fetch_json) + self.assertEqual(found, expected, full_name) + + def test_npm_package_can_be_declared_by_package_json(self): + web = FakeWeb() + web.json["registry.npmjs.org/widget"] = npm_meta(homepage="https://example.com") + web.json["/repos/Infinitode/Widget/contents"] = [ + { + "type": "file", + "name": "package.json", + "url": "https://api.github.com/repos/Infinitode/Widget/contents/package.json", + } + ] + web.json["/repos/Infinitode/Widget/contents/package.json"] = contents_file( + "package.json", '{"name": "widget"}' + ) + found = udb.npm_package_for(repo("Infinitode", "Widget"), fetch_json=web.fetch_json) + self.assertEqual(found, "widget") + + web.json["/repos/Infinitode/Widget/contents/package.json"] = contents_file( + "package.json", '{"name": "something-else"}' + ) + self.assertIsNone(udb.npm_package_for(repo("Infinitode", "Widget"), fetch_json=web.fetch_json)) + + def test_npm_metadata_pointing_at_another_repo_beats_a_name_match(self): + """An unrelated package that merely shares our repo's name is not ours. + + npm's ``localshare`` is someone else's 2019 package while our + ``johanbeytell1/LocalShare`` repo's package.json declares that name. + """ + web = FakeWeb() + web.json["registry.npmjs.org/localshare"] = npm_meta( + repository="git+https://github.com/localshare/localshare.git" + ) + web.json["/repos/johanbeytell1/LocalShare/contents"] = [ + { + "type": "file", + "name": "package.json", + "url": "https://api.github.com/repos/johanbeytell1/LocalShare/contents/package.json", + } + ] + web.json["/repos/johanbeytell1/LocalShare/contents/package.json"] = contents_file( + "package.json", '{"name": "localshare"}' + ) + found = udb.npm_package_for( + repo("johanbeytell1", "LocalShare"), fetch_json=web.fetch_json + ) + self.assertIsNone(found) + + def test_discover_npm_maps_repos_to_packages(self): + packages, first_days, notes = udb.discover_npm( + udb.discover_repos(), fetch_json=self.web.fetch_json + ) + self.assertEqual(packages, {"Infinitode/TailClass": "tailclass"}) + self.assertEqual(first_days, {"tailclass": date(2026, 11, 8)}, "lifetime start is kept") + self.assertEqual(notes, []) + + def test_npm_server_error_is_a_note_not_a_crash(self): + self.web.json["registry.npmjs.org/tailclass"] = http_error(500) + packages, _, notes = udb.discover_npm(udb.discover_repos(), fetch_json=self.web.fetch_json) + self.assertNotIn("Infinitode/TailClass", packages) + self.assertTrue(any("Infinitode/TailClass" in n for n in notes)) + + +# --- npm downloads ------------------------------------------------------------ + + +class TestNpmDownloads(StubMixin, unittest.TestCase): + def setUp(self): + self.web = FakeWeb() + self.install(self.web) + + @staticmethod + def _window(url): + span = url.split("/point/", 1)[1].split("/", 1)[0] + start, end = span.split(":") + return date.fromisoformat(start), date.fromisoformat(end) + + def test_all_time_total_is_summed_over_18_month_windows(self): + calls = [] + + def respond(url, headers): + calls.append(url) + return {"downloads": 10} + + self.web.json["api.npmjs.org/downloads/point/"] = respond + total = udb.npm_range_total("tailclass", today=date(2019, 1, 1)) + self.assertGreater(len(calls), 1, "the API caps a query at 18 months") + self.assertEqual(total, 10 * len(calls)) + windows = [self._window(url) for url in calls] + self.assertEqual(windows[0][0], udb.NPM_FIRST_DAY) + self.assertEqual(windows[-1][1], date(2019, 1, 1)) + for start, end in windows: + self.assertLessEqual((end - start).days + 1, udb.NPM_WINDOW_DAYS) + for (_, previous_end), (next_start, _) in zip(windows, windows[1:]): + self.assertEqual(next_start, previous_end + timedelta(days=1), "no gaps or overlaps") + + def test_a_short_history_needs_a_single_window(self): + calls = [] + + def respond(url, headers): + calls.append(url) + return {"downloads": 7} + + self.web.json["api.npmjs.org/downloads/point/"] = respond + total = udb.npm_range_total("tailclass", today=date(2015, 1, 11)) + self.assertEqual(len(calls), 1) + self.assertEqual(total, 7) + + def test_walk_starts_at_the_first_publish_day(self): + calls = [] + + def respond(url, headers): + calls.append(url) + return {"downloads": 4} + + self.web.json["api.npmjs.org/downloads/point/"] = respond + total = udb.npm_range_total( + "tailclass", today=date(2026, 11, 11), first_day=date(2026, 11, 8) + ) + self.assertEqual(len(calls), 1, "a four-day-old package needs a single query") + self.assertEqual(self._window(calls[0]), (date(2026, 11, 8), date(2026, 11, 11))) + self.assertEqual(total, 4) + + def test_first_day_before_npm_data_is_clamped_to_the_floor(self): + calls = [] + + def respond(url, headers): + calls.append(url) + return {"downloads": 1} + + self.web.json["api.npmjs.org/downloads/point/"] = respond + udb.npm_range_total("python-era", today=date(2015, 2, 1), first_day=date(2010, 1, 1)) + self.assertEqual(self._window(calls[0])[0], udb.NPM_FIRST_DAY) + + def test_a_package_published_today_queries_only_today(self): + calls = [] + + def respond(url, headers): + calls.append(url) + return {"downloads": 99} + + self.web.json["api.npmjs.org/downloads/point/"] = respond + totals, failures = udb.npm_totals( + ["tailclass"], today=date(2026, 11, 11), first_days={"tailclass": date(2026, 11, 11)} + ) + self.assertEqual(len(calls), 1) + self.assertEqual(self._window(calls[0]), (date(2026, 11, 11), date(2026, 11, 11))) + self.assertEqual(totals, {"tailclass": 99}) + self.assertEqual(failures, {}) + + def test_a_first_day_after_today_queries_nothing(self): + calls = [] + + def respond(url, headers): + calls.append(url) + return {"downloads": 99} + + self.web.json["api.npmjs.org/downloads/point/"] = respond + totals, failures = udb.npm_totals( + ["tailclass"], today=date(2026, 11, 11), first_days={"tailclass": date(2026, 12, 1)} + ) + self.assertEqual(calls, []) + self.assertEqual(totals, {"tailclass": 0}) + self.assertEqual(failures, {}) + + def test_a_window_without_a_count_raises_instead_of_undercounting(self): + self.web.json["api.npmjs.org/downloads/point/"] = {"unexpected": 1} + with self.assertRaises(ValueError): + udb.npm_range_total("tailclass", today=date(2015, 1, 11)) + + def test_totals_fall_back_to_shields_when_the_api_fails(self): + self.web.json["api.npmjs.org/downloads/point/"] = urllib.error.URLError("offline") + self.web.text["img.shields.io/npm/dt/"] = shields_svg("downloads", "1.2k") + totals, failures = udb.npm_totals(["tailclass"], today=date(2015, 1, 11)) + self.assertEqual(totals, {"tailclass": 1200}) + self.assertEqual(failures, {}) + + def test_totals_report_a_failure_when_every_source_fails(self): + self.web.json["api.npmjs.org/downloads/point/"] = urllib.error.URLError("offline") + self.web.text["img.shields.io/npm/dt/"] = urllib.error.URLError("offline") + totals, failures = udb.npm_totals(["tailclass"], today=date(2015, 1, 11)) + self.assertEqual(totals, {}) + self.assertIn("tailclass", failures) + # --- collection --------------------------------------------------------------- @@ -339,18 +592,30 @@ class TestCollect(StubMixin, unittest.TestCase): def setUp(self): self.web = standard_text_world(standard_world()) self.web.json["api.pepy.tech"] = lambda url, headers: {"total_downloads": 1_000} + self.web.json["api.npmjs.org/downloads/point/"] = {"downloads": 100} self.install(self.web) + def _collect(self): + # A fixed "today" keeps the npm window count (and so the total) stable. + # tailclass was published 2026-11-08, so its lifetime is a single window. + return udb.collect( + api_key="k", + fetch_json=self.web.fetch_json, + fetch_text=self.web.fetch_text, + today=date(2026, 11, 11), + ) + def test_stars_and_forks_are_summed_over_counted_repos(self): - result = udb.collect(api_key="k", fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text) + result = self._collect() self.assertEqual(result["stars"], 1 + 1 + 2) # LocalShare, Alloy, Hued; Secret and the foreign fork excluded self.assertEqual(result["forks"], 1) - def test_downloads_come_from_pypi_and_jsdelivr(self): - result = udb.collect(api_key="k", fetch_json=self.web.fetch_json, fetch_text=self.web.fetch_text) + def test_downloads_come_from_pypi_npm_and_jsdelivr(self): + result = self._collect() self.assertEqual(result["pypi"], {"hued": 1_000, "qrforge": 1_000}) + self.assertEqual(result["npm"], {"tailclass": 100}) self.assertEqual(result["jsdelivr"]["infinitode/cloakjs"], 250) - self.assertEqual(result["downloads"], 1_000 + 1_000 + 250) + self.assertEqual(result["downloads"], 1_000 + 1_000 + 100 + 250) self.assertNotIn("localshare", result["pypi"]) @@ -396,6 +661,8 @@ def _run(self, web, argv=(), env=None): def _healthy_web(self): web = standard_text_world(standard_world()) web.json["api.pepy.tech"] = lambda url, headers: {"total_downloads": 1_000} + # Force the npm fallback so the total does not depend on today's date. + web.json["api.npmjs.org/downloads/point/"] = http_error(500) return web def test_listing_failure_changes_nothing(self): @@ -425,8 +692,9 @@ def test_tokenless_run_writes_downloads_stars_and_forks(self): def test_happy_path_with_pepy_key_is_exact(self): code, output = self._run(self._healthy_web(), env={"PEPY_API_KEY": "k"}) self.assertEqual(code, 0, output) - # PyPI: hued + qrforge at 1,000 each (pepy); jsDelivr: CloakJS 250, the rest 0. - self.assertIn(f"Total%20Downloads-{udb.format_count(2_250)}-", self._read(self.readme)) + # PyPI: hued + qrforge at 1,000 each (pepy); npm: tailclass 350 from the + # shields fallback; jsDelivr: CloakJS 250, the rest 0. + self.assertIn(f"Total%20Downloads-{udb.format_count(2_600)}-", self._read(self.readme)) def test_dry_run_changes_nothing(self): code, output = self._run(self._healthy_web(), argv=["--dry-run"], env={"PEPY_API_KEY": "k"}) diff --git a/scripts/update-download-badges.py b/scripts/update-download-badges.py index 5b3fb1c..aff1fcb 100644 --- a/scripts/update-download-badges.py +++ b/scripts/update-download-badges.py @@ -17,6 +17,16 @@ Same-named packages by other people fail both checks, so they are skipped. 1. pepy v2 API when PEPY_API_KEY is set (exact counts), otherwise 2. shields.io ``/pepy/dt`` badge (rounded figures). + npm (all-time). Same ownership rule, lowercased (npm names are lowercase): + an npm package belongs to a repo when its name matches the repo's and either + its registry metadata links back to the repo, or -- only when that metadata + names no GitHub repository at all -- the repo's package.json declares the + name. Metadata pointing at a different repo is always rejected. + 1. registry downloads API, summed over 18-month windows from the package's + first publish day (a single query may span at most 18 months, and npm + has no data before 2015-01-10, so that is the floor), + 2. shields.io ``/npm/dt`` badge when that is unavailable. Note shields + builds that badge from a capped range, so the fallback can undercount. jsDelivr (last 12 months). shields.io ``/jsdelivr/gh/hy`` for every counted repo. jsDelivr publishes period stats only; there is no all-time figure. @@ -48,6 +58,7 @@ import sys import urllib.error import urllib.request +from datetime import date, timedelta # --- what gets counted ------------------------------------------------------- @@ -61,15 +72,24 @@ GITHUB_HEADERS = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"} MAX_PAGES = 20 PACKAGING_FILES = ("pyproject.toml", "setup.py", "setup.cfg") +NPM_PACKAGING_FILES = ("package.json",) PYPI_JSON = "https://pypi.org/pypi/{pkg}/json" PEPY_V2 = "https://api.pepy.tech/api/v2/projects/{pkg}" SHIELDS_PYPI = "https://img.shields.io/pepy/dt/{pkg}" +SHIELDS_NPM = "https://img.shields.io/npm/dt/{package}" SHIELDS_JSDELIVR = "https://img.shields.io/jsdelivr/gh/{period}/{repo}" +NPM_REGISTRY = "https://registry.npmjs.org/{package}" +NPM_DOWNLOADS_POINT = "https://api.npmjs.org/downloads/point/{start}:{end}/{package}" +# The point API refuses ranges longer than 18 months, and has no data before +# this date, so all-time npm totals have to be summed over windows. +NPM_FIRST_DAY = date(2015, 1, 10) +NPM_WINDOW_DAYS = 540 # safely under the 18-month (~547 day) limit + # A new total this far below the published one counts as partial data and is # refused. 5% absorbs display rounding while still catching a missing source -# (dropping one of eleven sources moves the total by ~9%). +# (losing one source, or one chunk of an npm total, moves the total by more). DROP_TOLERANCE = 0.05 # Keys that hold an all-time count, highest confidence first. @@ -92,6 +112,7 @@ _PY_NAME_RE = re.compile(r"""\bname\s*=\s*["']([A-Za-z0-9][A-Za-z0-9._-]*)["']""") _CFG_NAME_RE = re.compile(r"^name\s*=\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*$", re.M) +_GITHUB_URL_RE = re.compile(r"github\.com[/:]([^/\s]+)/([^/\s#?]+)", re.I) # --- network (the only I/O in this module) ----------------------------------- @@ -224,6 +245,70 @@ def packaging_names(filename, text): return set(_PY_NAME_RE.findall(text)) +def npm_name(repo_name): + """The npm package name a repo's name implies. + + npm names are lowercase and may only hold ``a-z 0-9 . _ -``, so the + PascalCase repo names map straight across (TailClass -> tailclass). + """ + return re.sub(r"[^a-z0-9._-]+", "-", repo_name.strip().lower()).strip("-._") + + +def normalize_npm_name(name): + """npm package names are lowercase; compare them that way.""" + return name.strip().lower() + + +def npm_github_repos(meta): + """Every ``owner/repo`` the npm package's metadata points at on GitHub.""" + version = (meta.get("dist-tags") or {}).get("latest") or "" + latest = (meta.get("versions") or {}).get(version) or meta + urls = [] + for key in ("repository", "homepage", "bugs"): + for source in (latest, meta): + value = source.get(key) + if isinstance(value, dict): + value = value.get("url") + if isinstance(value, str) and value: + urls.append(value) + # git+https://github.com/o/r.git and git+ssh://git@github.com:o/r both count + text = re.sub(r"\.git\b", "", " ".join(urls), flags=re.I) + return { + (match.group(1).lower(), match.group(2).lower()) + for match in _GITHUB_URL_RE.finditer(text) + } + + +def npm_links_back(meta, owner, repo): + """True if the npm package's metadata links to github.com/owner/repo.""" + return (owner.lower(), repo.lower()) in npm_github_repos(meta) + + +def npm_first_day(meta): + """The day the npm package was first published, or ``None`` if unknown. + + Registry metadata carries a ``time.created`` timestamp. Downloads cannot + predate it, so it is the earliest window worth asking the API for. + """ + created = (meta.get("time") or {}).get("created") + if not isinstance(created, str) or len(created) < 10: + return None + try: + return date.fromisoformat(created[:10]) + except ValueError: + return None + + +def npm_packaging_names(filename, text): + """Package names a package.json declares.""" + try: + meta = json.loads(text) + except ValueError: + return set() + name = meta.get("name") if isinstance(meta, dict) else None + return {name} if isinstance(name, str) and name.strip() else set() + + def render_block(values): """The badge block. *values* is a list of ``(label, badge_text)`` pairs.""" lines = [START_MARK] @@ -277,9 +362,16 @@ def discover_repos(fetch_json=None, token=None): return repos -def declared_names(owner, repo, fetch_json=None, token=None): - """Package names declared by the repo's own packaging files.""" +def declared_names( + owner, repo, fetch_json=None, token=None, files=PACKAGING_FILES, parse=None +): + """Package names declared by the repo's own packaging files. + + *files* and *parse* select the ecosystem: Python's packaging files by + default, or ``NPM_PACKAGING_FILES`` with ``npm_packaging_names``. + """ fetch_json = fetch_json or globals()["fetch_json"] + parse = parse or packaging_names try: listing = fetch_json(f"{GITHUB_API}/repos/{owner}/{repo}/contents", github_headers(token)) except urllib.error.HTTPError as error: @@ -288,10 +380,10 @@ def declared_names(owner, repo, fetch_json=None, token=None): raise names = set() for item in listing if isinstance(listing, list) else []: - if item.get("type") == "file" and item.get("name") in PACKAGING_FILES: + if item.get("type") == "file" and item.get("name") in files: blob = fetch_json(item["url"], github_headers(token)) text = base64.b64decode(blob.get("content", "")).decode("utf-8", "replace") - names |= packaging_names(item["name"], text) + names |= parse(item["name"], text) return names @@ -326,6 +418,71 @@ def discover_pypi(repos, fetch_json=None, token=None): return packages, notes +def npm_lookup(repo, fetch_json=None, token=None): + """``(package, first_day)`` for the npm package that belongs to *repo*. + + Both are ``None`` when the repo has no package of its own. + """ + fetch_json = fetch_json or globals()["fetch_json"] + owner, name = repo["owner"]["login"], repo["name"] + package = npm_name(name) + if not package: + return None, None + try: + meta = fetch_json(NPM_REGISTRY.format(package=package)) + except urllib.error.HTTPError as error: + if error.code == 404: + return None, None + raise + if not isinstance(meta, dict): + raise ValueError(f"unexpected npm metadata for {package}") + if not npm_links_back(meta, owner, name): + # Provenance wins over a name match: a package that names some *other* + # GitHub repo in its metadata is not ours, even if our package.json + # happens to declare the same name (npm/local-share style collisions). + if npm_github_repos(meta): + return None, None + declared = { + normalize_npm_name(n) + for n in declared_names( + owner, + name, + fetch_json, + token, + files=NPM_PACKAGING_FILES, + parse=npm_packaging_names, + ) + } + if package not in declared: + return None, None + return package, npm_first_day(meta) + + +def npm_package_for(repo, fetch_json=None, token=None): + """The npm package that belongs to *repo*, or ``None``.""" + return npm_lookup(repo, fetch_json=fetch_json, token=token)[0] + + +def discover_npm(repos, fetch_json=None, token=None): + """Map ``full_name -> npm package``, plus each package's first day. + + Returns ``(packages, first_days, notes)``, where *first_days* maps a package + name to the day it was first published (used to walk only its lifetime). + """ + packages, first_days, notes = {}, {}, [] + for repo in repos: + try: + package, first_day = npm_lookup(repo, fetch_json=fetch_json, token=token) + except (urllib.error.URLError, ValueError) as error: + notes.append(f"{repo['full_name']}: npm lookup skipped ({error})") + continue + if package: + packages[repo["full_name"]] = package + if first_day: + first_days[package] = first_day + return packages, first_days, notes + + # --- source lookups ---------------------------------------------------------- @@ -362,6 +519,64 @@ def pypi_totals(packages, api_key=None, fetch_json=None, fetch_text=None): return totals, failures, sorted(set(notes)) +def npm_range_total(package, fetch_json=None, today=None, first_day=None): + """All-time npm downloads for *package*, summed over 18-month windows. + + The walk starts at the package's first publish day (nothing can predate it) + and never earlier than 2015-01-10. The point API refuses any single range + longer than 18 months, so the lifetime is covered in windows. A failed or + malformed window raises instead of returning a small partial sum. + """ + fetch_json = fetch_json or globals()["fetch_json"] + last_day = today or date.today() + start = max(NPM_FIRST_DAY, first_day) if first_day else NPM_FIRST_DAY + total = 0 + while start <= last_day: + end = min(start + timedelta(days=NPM_WINDOW_DAYS - 1), last_day) + payload = fetch_json( + NPM_DOWNLOADS_POINT.format(package=package, start=start.isoformat(), end=end.isoformat()) + ) + count = find_count(payload) + if count is None: + raise ValueError(f"no download count in response for {package}") + total += count + start = end + timedelta(days=1) + return total + + +def npm_totals(packages, fetch_json=None, fetch_text=None, today=None, first_days=None): + """All-time npm downloads per package, exact where the API answers. + + *first_days* maps a package to its first publish day so that only its own + lifetime is queried; packages missing from it are walked from npm's floor. + """ + fetch_json = fetch_json or globals()["fetch_json"] + fetch_text = fetch_text or globals()["fetch_text"] + first_days = first_days or {} + totals, failures = {}, {} + for package in packages: + errors = [] + count = None + try: + count = npm_range_total( + package, fetch_json=fetch_json, today=today, first_day=first_days.get(package) + ) + except (urllib.error.URLError, ValueError) as error: + errors.append(f"npm api -> {error}") + if count is None: # fallback: shields.io builds this from a capped range + try: + count = parse_shields_badge(fetch_text(SHIELDS_NPM.format(package=package))) + except (urllib.error.URLError, ValueError) as error: + errors.append(f"shields.io -> {error}") + if count is None and not any("shields.io" in e for e in errors): + errors.append("shields.io -> no numeric value in badge") + if count is not None: + totals[package] = count + else: + failures[package] = "; ".join(errors) or "unknown error" + return totals, failures + + def jsdelivr_totals(repos, period=JSDELIVR_PERIOD, fetch_text=None): """jsDelivr hits per repo (``owner/repo``) for *period* (default: 12 months).""" fetch_text = fetch_text or globals()["fetch_text"] @@ -383,7 +598,7 @@ def jsdelivr_totals(repos, period=JSDELIVR_PERIOD, fetch_text=None): # --- collection -------------------------------------------------------------- -def collect(api_key=None, github_token=None, fetch_json=None, fetch_text=None): +def collect(api_key=None, github_token=None, fetch_json=None, fetch_text=None, today=None): """Gather every download, star and fork figure.""" fetch_json = fetch_json or globals()["fetch_json"] fetch_text = fetch_text or globals()["fetch_text"] @@ -401,6 +616,18 @@ def collect(api_key=None, github_token=None, fetch_json=None, fetch_text=None): ) notes += pypi_notes + npm_packages, npm_first_days, npm_notes = discover_npm( + repos, fetch_json=fetch_json, token=github_token + ) + notes += npm_notes + npm, npm_failures = npm_totals( + sorted(set(npm_packages.values())), + fetch_json=fetch_json, + fetch_text=fetch_text, + today=today, + first_days=npm_first_days, + ) + jsdelivr, jsdelivr_failures = jsdelivr_totals( [repo["full_name"].lower() for repo in repos], fetch_text=fetch_text ) @@ -410,12 +637,14 @@ def collect(api_key=None, github_token=None, fetch_json=None, fetch_text=None): "stars": stars, "forks": forks, "packages": packages, + "npm_packages": npm_packages, "pypi": pypi, + "npm": npm, "jsdelivr": jsdelivr, - "failures": {**pypi_failures, **jsdelivr_failures}, + "failures": {**pypi_failures, **npm_failures, **jsdelivr_failures}, "notes": sorted(set(notes)), - "download_sources": len(pypi) + len(jsdelivr), - "downloads": sum(pypi.values()) + sum(jsdelivr.values()), + "download_sources": len(pypi) + len(npm) + len(jsdelivr), + "downloads": sum(pypi.values()) + sum(npm.values()) + sum(jsdelivr.values()), } @@ -447,8 +676,12 @@ def main(argv=None): print(f" repos {len(result['repos'])} public repos counted: {', '.join(sorted(result['repos']))}") for repo, package in sorted(result["packages"].items()): print(f" package {repo:<40} -> pypi/{package}") + for repo, package in sorted(result["npm_packages"].items()): + print(f" package {repo:<40} -> npm/{package}") for name, count in sorted(result["pypi"].items()): print(f" pypi {name:<29} {count:>14,}") + for name, count in sorted(result["npm"].items()): + print(f" npm {name:<29} {count:>14,}") for name, count in sorted(result["jsdelivr"].items()): print(f" jsdelivr {name:<29} {count:>14,}") for name, reason in sorted(result["failures"].items()):