From d47bdfa813cbca087a2ac63d954b33c49e6a0438 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 18 Nov 2026 13:56:22 +0100 Subject: [PATCH 1/4] Select refresh file kinds only from files clients downloaded A file a refresh mirrored could widen the kinds later refreshes fetch, for example a py2.py3 wheel adding py2-none-any. MostUsedPackages now marks the files clients downloaded, and the variants come from those only. Completeness is still checked against every cached file. Fixes #750 --- internal/database/most_used.go | 12 +++++-- internal/database/most_used_test.go | 55 ++++++++++++++++++++++++++--- internal/mirror/refresh.go | 5 +++ internal/mirror/refresh_test.go | 40 +++++++++++++++++++++ 4 files changed, 106 insertions(+), 6 deletions(-) diff --git a/internal/database/most_used.go b/internal/database/most_used.go index a365cb71..4cc11178 100644 --- a/internal/database/most_used.go +++ b/internal/database/most_used.go @@ -20,11 +20,15 @@ type UsedFile struct { Version string Filename string UpstreamURL string + // Downloaded is false for a file a mirror run fetched that no client + // has downloaded since. + Downloaded bool } // MostUsedPackages returns, for each ecosystem, the top packages with the most // downloads among the cached files downloaded since cutoff, ordered by -// ecosystem and then by hits. Each lists every file cached for it. Files +// ecosystem and then by hits. Each lists every file cached for it and marks +// the ones clients downloaded. Files // from local sources, which have no upstream, are left out, and so are // files a mirror run fetched that no client has downloaded since. func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) { @@ -59,9 +63,12 @@ func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) VersionPURL string `db:"version_purl"` Filename string `db:"filename"` UpstreamURL string `db:"upstream_url"` + Downloaded int `db:"downloaded"` } err := db.Select(&files, db.Rebind(` - SELECT a.version_purl, a.filename, a.upstream_url + SELECT a.version_purl, a.filename, a.upstream_url, + CASE WHEN a.last_accessed_at IS NOT NULL OR a.mirrored_at IS NULL OR a.mirrored_at < a.fetched_at + THEN 1 ELSE 0 END AS downloaded FROM artifacts a JOIN versions v ON v.purl = a.version_purl WHERE v.package_purl = ? AND a.storage_path IS NOT NULL AND a.source = ? @@ -73,6 +80,7 @@ func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) for _, f := range files { packages[i].Files = append(packages[i].Files, UsedFile{ Version: VersionFromPURL(f.VersionPURL), Filename: f.Filename, UpstreamURL: f.UpstreamURL, + Downloaded: f.Downloaded == 1, }) } } diff --git a/internal/database/most_used_test.go b/internal/database/most_used_test.go index 4b8693e4..31f39f1b 100644 --- a/internal/database/most_used_test.go +++ b/internal/database/most_used_test.go @@ -50,16 +50,16 @@ func TestMostUsedPackages(t *testing.T) { } want := []UsedPackage{ {Ecosystem: "cargo", Name: "serde", PURL: "pkg:cargo/serde", Hits: 1, Files: []UsedFile{ - {Version: "1.0.200", Filename: "serde-1.0.200.crate", UpstreamURL: "https://example.test/serde-1.0.200.crate"}, + {Version: "1.0.200", Filename: "serde-1.0.200.crate", UpstreamURL: "https://example.test/serde-1.0.200.crate", Downloaded: true}, }}, // react has more hits than lodash. Every cached file of lodash is // listed, but only recent downloads count. {Ecosystem: "npm", Name: "react", PURL: "pkg:npm/react", Hits: 10, Files: []UsedFile{ - {Version: "18.3.1", Filename: "react-18.3.1.tgz", UpstreamURL: "https://example.test/react-18.3.1.tgz"}, + {Version: "18.3.1", Filename: "react-18.3.1.tgz", UpstreamURL: "https://example.test/react-18.3.1.tgz", Downloaded: true}, }}, {Ecosystem: "npm", Name: "lodash", PURL: "pkg:npm/lodash", Hits: 5, Files: []UsedFile{ - {Version: "4.17.20", Filename: "lodash-4.17.20.tgz", UpstreamURL: "https://example.test/lodash-4.17.20.tgz"}, - {Version: "4.17.21", Filename: "lodash-4.17.21.tgz", UpstreamURL: "https://example.test/lodash-4.17.21.tgz"}, + {Version: "4.17.20", Filename: "lodash-4.17.20.tgz", UpstreamURL: "https://example.test/lodash-4.17.20.tgz", Downloaded: true}, + {Version: "4.17.21", Filename: "lodash-4.17.21.tgz", UpstreamURL: "https://example.test/lodash-4.17.21.tgz", Downloaded: true}, }}, } if !reflect.DeepEqual(got, want) { @@ -116,3 +116,50 @@ func TestMostUsedPackagesSkipsMirroredArtifactsNotDownloaded(t *testing.T) { } }) } + +func TestMostUsedPackagesMarksTheFilesClientsDownloaded(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + seedUsedArtifact(t, db, "npm", "lodash", "4.17.20", "lodash-4.17.20.tgz", 5, now) + seedUsedArtifact(t, db, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", 0, now) + seedUsedArtifact(t, db, "npm", "lodash", "4.17.22", "lodash-4.17.22.tgz", 0, now) + seedUsedArtifact(t, db, "npm", "lodash", "4.17.23", "lodash-4.17.23.tgz", 0, now) + seedUsedArtifact(t, db, "npm", "lodash", "4.17.24", "lodash-4.17.24.tgz", 0, now) + // A refresh mirrored 4.17.21 and 4.17.22, and a client downloaded + // 4.17.22 since. A client fetched 4.17.23 without a recorded hit, + // and fetched 4.17.24 again after a refresh had mirrored it. + for _, u := range []struct { + filename string + mirrored any + }{ + {"lodash-4.17.21.tgz", now.Add(time.Minute).UTC()}, + {"lodash-4.17.22.tgz", now.Add(time.Minute).UTC()}, + {"lodash-4.17.23.tgz", nil}, + {"lodash-4.17.24.tgz", now.Add(-time.Minute).UTC()}, + } { + if _, err := db.Exec(db.Rebind("UPDATE artifacts SET last_accessed_at = NULL, mirrored_at = ? WHERE filename = ?"), + u.mirrored, u.filename); err != nil { + t.Fatal(err) + } + } + if err := db.RecordArtifactHit("pkg:npm/lodash@4.17.22", "lodash-4.17.22.tgz"); err != nil { + t.Fatal(err) + } + + got, err := db.MostUsedPackages(now.Add(-time.Hour), 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("MostUsedPackages = %+v, want lodash", got) + } + downloaded := map[string]bool{} + for _, f := range got[0].Files { + downloaded[f.Version] = f.Downloaded + } + want := map[string]bool{"4.17.20": true, "4.17.21": false, "4.17.22": true, "4.17.23": true, "4.17.24": true} + if !reflect.DeepEqual(downloaded, want) { + t.Errorf("downloaded = %v, want %v", downloaded, want) + } + }) +} diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index 513f2398..903d9e72 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -172,9 +172,14 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] return items, nil } +// cachedVariants returns the kinds of files clients downloaded. Files a +// refresh mirrored are left out, so they don't widen later refreshes. func cachedVariants(eco fileEcosystem, files []database.UsedFile) map[string]bool { variants := map[string]bool{} for _, f := range files { + if !f.Downloaded { + continue + } for _, v := range eco.cachedVariants(f.Filename) { variants[v] = true } diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index 3bf7c5cb..6478a367 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -298,6 +298,46 @@ func TestRefreshMirrorsTheFileVariantsClientsFetched(t *testing.T) { } } +func TestRefreshSelectsVariantsOnlyFromFilesClientsFetched(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "wheel") + ctx := context.Background() + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "pypi", "demo", "1.0.0", "demo-1.0.0-py3-none-any.whl", "https://files.example/demo-1.0.0-py3-none-any.whl", "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + files := func(version string, names ...string) registries.Version { + v := registries.Version{Number: version, PublishedAt: time.Now().Add(-30 * 24 * time.Hour)} + for _, n := range names { + v.Artifacts = append(v.Artifacts, registries.Artifact{Filename: n, URL: "https://files.example/" + n}) + } + return v + } + releases := []registries.Version{ + files("1.0.0", "demo-1.0.0-py3-none-any.whl"), + files("1.1.0", "demo-1.1.0-py2.py3-none-any.whl"), + } + m.Versions = stubReleases{"pypi/demo": releases} + if _, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}); err != nil { + t.Fatalf("Refresh: %v", err) + } + if got := fetchedMatching(fetcher.fetched(), "1.1.0"); len(got) != 1 { + t.Fatalf("first refresh fetched %v, want the 1.1.0 wheel", got) + } + + // The mirrored py2.py3 wheel adds py2-none-any to its variants, but no + // client fetched that kind. + m.Versions = stubReleases{"pypi/demo": append(releases, + files("1.2.0", "demo-1.2.0-py3-none-any.whl", "demo-1.2.0-py2-none-any.whl"))} + fetcher.reset() + if _, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}); err != nil { + t.Fatalf("Refresh: %v", err) + } + if got, want := fetcher.fetched(), []string{"https://files.example/demo-1.2.0-py3-none-any.whl"}; !slices.Equal(got, want) { + t.Errorf("second refresh fetched %v, want %v", got, want) + } +} + func TestFileVariants(t *testing.T) { tests := []struct { ecosystem, cacheFilename, want string From b18a9a0b4d95a256993217e97f11c784337e526c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 18 Nov 2026 14:51:50 +0100 Subject: [PATCH 2/4] Keep mirror cache reads out of client downloads A refresh that retries a release read its cached files through the cache-hit path, which set last_accessed_at, so they looked downloaded by a client. A mirror run's read no longer records a hit. Refresh also writes buffered hits before reading them, and skips a package left with no downloaded file kinds, which eviction between the two queries can cause. --- internal/database/most_used.go | 8 ++-- internal/handler/handler.go | 15 ++++--- internal/mirror/refresh.go | 10 +++++ internal/mirror/refresh_test.go | 79 +++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 9 deletions(-) diff --git a/internal/database/most_used.go b/internal/database/most_used.go index 4cc11178..ddf38e6c 100644 --- a/internal/database/most_used.go +++ b/internal/database/most_used.go @@ -27,10 +27,10 @@ type UsedFile struct { // MostUsedPackages returns, for each ecosystem, the top packages with the most // downloads among the cached files downloaded since cutoff, ordered by -// ecosystem and then by hits. Each lists every file cached for it and marks -// the ones clients downloaded. Files -// from local sources, which have no upstream, are left out, and so are -// files a mirror run fetched that no client has downloaded since. +// ecosystem and then by hits. Files a mirror run fetched that no client has +// downloaded since don't count toward the ranking. Each package lists every +// file cached for it, those included, and marks the ones clients downloaded. +// Files from local sources, which have no upstream, are left out. func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) { var rows []struct { Ecosystem string `db:"ecosystem"` diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9c883053..ff7e427a 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -565,7 +565,7 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s if servingHead(ctx) { result.rangeable = readsSeekable(p.Storage) - p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) + p.recordCacheHit(ctx, artifact.Ecosystem, versionPURL, filename) return result, nil } @@ -573,7 +573,7 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL) if err == nil { result.RedirectURL = signed - p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) + p.recordCacheHit(ctx, artifact.Ecosystem, versionPURL, filename) return result, nil } if !errors.Is(err, storage.ErrSignedURLUnsupported) { @@ -609,12 +609,17 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err) return nil, nil } - p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) + p.recordCacheHit(ctx, artifact.Ecosystem, versionPURL, filename) return result, nil } -func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) { - _ = p.DB.RecordArtifactHit(versionPURL, filename) +// recordCacheHit counts a cache hit. A mirror run's read leaves the +// artifact's hits and last access alone, so it doesn't look like a client +// download. +func (p *Proxy) recordCacheHit(ctx context.Context, ecosystem, versionPURL, filename string) { + if !mirrorFetch(ctx) { + _ = p.DB.RecordArtifactHit(versionPURL, filename) + } metrics.RecordArtifactCacheHit(ecosystem) } diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index 903d9e72..b7356f4b 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -59,6 +59,11 @@ func (m *Mirror) releaseLister() ReleaseLister { //nolint:ireturn // any lister func (m *Mirror) Refresh(ctx context.Context, r Refresh) (*Progress, error) { ctx = handler.MirrorFetch(ctx) cutoff := time.Now().Add(-r.Since) + // Without the buffered hits, a recent download is missed until the next + // run, which is all it costs, so a failed write doesn't stop this one. + if err := m.db.FlushHits(); err != nil { + m.logger.Warn("mirror refresh: failed to write buffered hits", "error", err) + } packages, err := m.db.MostUsedPackages(cutoff, r.Top) if err != nil { return nil, fmt.Errorf("listing the most used packages: %w", err) @@ -160,6 +165,11 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] var incomplete func(registries.Version) bool if files { variants = cachedVariants(eco, pkg.Files) + if len(variants) == 0 { + // Eviction removed every file a client downloaded after the + // ranking query ran. + return nil, nil + } incomplete = func(r registries.Version) bool { return eco.missing(PackageVersion{Ecosystem: ecosystem, Name: name, Version: r.Number, variants: variants}, r, pkg.Files) } diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index 6478a367..4945e2f5 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -12,6 +12,8 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/registries" "github.com/git-pkgs/registries/fetch" ) @@ -338,6 +340,83 @@ func TestRefreshSelectsVariantsOnlyFromFilesClientsFetched(t *testing.T) { } } +func TestRefreshRetryDoesNotCountCachedFilesAsDownloaded(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "wheel") + ctx := context.Background() + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "pypi", "demo", "1.0.0", "demo-1.0.0-py3-none-any.whl", "https://files.example/demo-1.0.0-py3-none-any.whl", "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + files := func(version string, names ...string) registries.Version { + v := registries.Version{Number: version, PublishedAt: time.Now().Add(-30 * 24 * time.Hour)} + for _, n := range names { + v.Artifacts = append(v.Artifacts, registries.Artifact{Filename: n, URL: "https://files.example/" + n}) + } + return v + } + releases := []registries.Version{ + files("1.0.0", "demo-1.0.0-py3-none-any.whl"), + files("1.1.0", "demo-1.1.0-py2.py3-none-any.whl", "demo-1.1.0-1-py3-none-any.whl"), + } + m.Versions = stubReleases{"pypi/demo": releases} + refresh := func() { + t.Helper() + if _, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}); err != nil { + t.Fatalf("Refresh: %v", err) + } + } + // The first refresh caches the py2.py3 wheel and fails on the other, + // so the second reads the py2.py3 wheel from the cache again. + fetcher.failing("1.1.0-1-py3") + refresh() + fetcher.failing("") + refresh() + + m.Versions = stubReleases{"pypi/demo": append(releases, + files("1.2.0", "demo-1.2.0-py3-none-any.whl", "demo-1.2.0-py2-none-any.whl"))} + fetcher.reset() + refresh() + if got, want := fetcher.fetched(), []string{"https://files.example/demo-1.2.0-py3-none-any.whl"}; !slices.Equal(got, want) { + t.Errorf("refresh after the retry fetched %v, want %v", got, want) + } +} + +func TestRefreshSeesBufferedDownloads(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + m.db.BatchHits(time.Hour, nil) + result, err := m.proxy.GetOrFetchArtifact(handler.MirrorFetch(context.Background()), "npm", "lodash", "4.17.20", "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + // A client downloads the mirrored version; the hit is still buffered. + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21")} + fetcher.reset() + + if _, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour}); err != nil { + t.Fatalf("Refresh: %v", err) + } + if got := fetchedMatching(fetcher.fetched(), "lodash-4.17.21.tgz"); len(got) != 1 { + t.Errorf("fetched %v, want lodash 4.17.21", fetcher.fetched()) + } +} + +func TestRefreshSkipsAPackageWithNoDownloadedFileKinds(t *testing.T) { + m, _ := setupRefreshMirror(t, "wheel") + m.Versions = stubReleases{"pypi/demo": oldReleases("1.0.0", "1.1.0")} + // Eviction removed the file a client downloaded after the ranking query + // ran, leaving only one a refresh mirrored. + items, err := m.refreshItems(context.Background(), database.UsedPackage{ + Ecosystem: "pypi", Name: "demo", PURL: "pkg:pypi/demo", + Files: []database.UsedFile{{Version: "1.0.0", Filename: "demo-1.0.0-py3-none-any.whl"}}, + }) + if err != nil || len(items) != 0 { + t.Errorf("refreshItems = %+v, %v; want none", items, err) + } +} + func TestFileVariants(t *testing.T) { tests := []struct { ecosystem, cacheFilename, want string From 936f226e06c4e92313064f1a8a3926714dd7aeaf Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 18 Nov 2026 16:42:01 +0100 Subject: [PATCH 3/4] Keep files a refresh reads from retention Since mirror reads no longer record a hit, a cached file a refresh retry read could expire right after. The read now sets mirrored_at, except on a file a client fetched and hasn't downloaded since, which that would mark as fetched by a mirror run. Refresh also checks for downloaded file kinds before listing releases, the retry test checks what the retry fetches, and the flush comment no longer promises the next run recovers a missed download. --- internal/database/retention.go | 13 ++++++++++ internal/database/retention_test.go | 39 +++++++++++++++++++++++++++++ internal/handler/handler.go | 6 +++-- internal/mirror/refresh.go | 23 +++++++++-------- internal/mirror/refresh_test.go | 30 +++++++++++++++++++--- 5 files changed, 96 insertions(+), 15 deletions(-) diff --git a/internal/database/retention.go b/internal/database/retention.go index f9e74e3f..b3b7d72b 100644 --- a/internal/database/retention.go +++ b/internal/database/retention.go @@ -141,6 +141,19 @@ func (db *DB) MarkArtifactMirrored(versionPURL, filename string) error { return err } +// KeepMirroredArtifact records that a mirror run read a cached artifact +// again, which keeps it from retention without counting as a client +// download. An artifact a client fetched and hasn't downloaded since is left +// alone: marking it mirrored would make it look fetched by a mirror run. +func (db *DB) KeepMirroredArtifact(versionPURL, filename string) error { + _, err := db.Exec(db.Rebind(` + UPDATE artifacts SET mirrored_at = ? + WHERE version_purl = ? AND filename = ? AND storage_path IS NOT NULL + AND (last_accessed_at IS NOT NULL OR mirrored_at >= fetched_at) + `), time.Now().UTC(), versionPURL, filename) + return err +} + // GetArtifactEcosystem returns the packages.ecosystem value of the package an // artifact version belongs to, or "" when it has no package record. func (db *DB) GetArtifactEcosystem(versionPURL string) (string, error) { diff --git a/internal/database/retention_test.go b/internal/database/retention_test.go index 6ea1a10a..0b719585 100644 --- a/internal/database/retention_test.go +++ b/internal/database/retention_test.go @@ -544,3 +544,42 @@ func TestFlushHitsRunsOneAtATime(t *testing.T) { t.Fatalf("FlushHits: %v", err) } } + +func TestKeepMirroredArtifact(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + cutoff := now.Add(-24 * time.Hour) + old := now.Add(-48 * time.Hour) + seedRetentionArtifact(t, db, "npm", "mirrored", old, time.Time{}) + seedRetentionArtifact(t, db, "npm", "served", old, old) + seedRetentionArtifact(t, db, "npm", "client-fetched", old, time.Time{}) + if _, err := db.Exec(db.Rebind("UPDATE artifacts SET mirrored_at = ? WHERE filename = ?"), old.UTC(), "mirrored.tgz"); err != nil { + t.Fatal(err) + } + for _, name := range []string{"mirrored", "served", "client-fetched"} { + if err := db.KeepMirroredArtifact("pkg:npm/"+name+"@1.0.0", name+".tgz"); err != nil { + t.Fatalf("KeepMirroredArtifact(%s): %v", name, err) + } + } + + maxID, err := db.MaxArtifactID() + if err != nil { + t.Fatal(err) + } + candidates, err := db.GetRetentionCandidates(0, maxID, cutoff) + if err != nil { + t.Fatal(err) + } + got := candidateNames(candidates) + for _, kept := range []string{"mirrored.tgz", "served.tgz"} { + if _, ok := got[kept]; ok { + t.Errorf("%s is a retention candidate after a mirror run read it", kept) + } + } + // Marking this one mirrored would make it look fetched by a mirror + // run rather than a client. + if c, ok := got["client-fetched.tgz"]; !ok || c.MirroredAt.Valid { + t.Errorf("client-fetched.tgz = %+v, %v; want it left unmarked", c, ok) + } + }) +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index ff7e427a..bd9fd2c0 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -615,9 +615,11 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s // recordCacheHit counts a cache hit. A mirror run's read leaves the // artifact's hits and last access alone, so it doesn't look like a client -// download. +// download, and keeps it from retention instead. func (p *Proxy) recordCacheHit(ctx context.Context, ecosystem, versionPURL, filename string) { - if !mirrorFetch(ctx) { + if mirrorFetch(ctx) { + _ = p.DB.KeepMirroredArtifact(versionPURL, filename) + } else { _ = p.DB.RecordArtifactHit(versionPURL, filename) } metrics.RecordArtifactCacheHit(ecosystem) diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index b7356f4b..9715d2a3 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -59,8 +59,9 @@ func (m *Mirror) releaseLister() ReleaseLister { //nolint:ireturn // any lister func (m *Mirror) Refresh(ctx context.Context, r Refresh) (*Progress, error) { ctx = handler.MirrorFetch(ctx) cutoff := time.Now().Add(-r.Since) - // Without the buffered hits, a recent download is missed until the next - // run, which is all it costs, so a failed write doesn't stop this one. + // Without the buffered hits, this run misses the downloads still in + // them, and a later run only sees those still within Since. That is + // logged rather than stopping the run. if err := m.db.FlushHits(); err != nil { m.logger.Warn("mirror refresh: failed to write buffered hits", "error", err) } @@ -139,6 +140,16 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] if !refreshable(ecosystem) { return nil, nil } + eco, files := fileEcosystems[ecosystem] + var variants map[string]bool + if files { + variants = cachedVariants(eco, pkg.Files) + if len(variants) == 0 { + // Eviction removed every file a client downloaded after the + // ranking query ran. + return nil, nil + } + } releases, err := m.releaseLister().ListReleases(ctx, ecosystem, name) if err != nil { return nil, fmt.Errorf("listing releases: %w", err) @@ -160,16 +171,8 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] return m.proxy == nil || m.proxy.ReleaseAllowed(pkg.Ecosystem, pkg.Name, v.Number, v.PublishedAt) } - eco, files := fileEcosystems[ecosystem] - var variants map[string]bool var incomplete func(registries.Version) bool if files { - variants = cachedVariants(eco, pkg.Files) - if len(variants) == 0 { - // Eviction removed every file a client downloaded after the - // ranking query ran. - return nil, nil - } incomplete = func(r registries.Version) bool { return eco.missing(PackageVersion{Ecosystem: ecosystem, Name: name, Version: r.Number, variants: variants}, r, pkg.Files) } diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index 4945e2f5..c2587171 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -370,8 +370,31 @@ func TestRefreshRetryDoesNotCountCachedFilesAsDownloaded(t *testing.T) { // so the second reads the py2.py3 wheel from the cache again. fetcher.failing("1.1.0-1-py3") refresh() + old := time.Now().Add(-48 * time.Hour).UTC() + if _, err := m.db.Exec(m.db.Rebind("UPDATE artifacts SET fetched_at = ?, mirrored_at = ? WHERE filename = ?"), + old, old, "demo-1.1.0-py2.py3-none-any.whl"); err != nil { + t.Fatal(err) + } fetcher.failing("") + fetcher.reset() refresh() + if got, want := fetcher.fetched(), []string{"https://files.example/demo-1.1.0-1-py3-none-any.whl"}; !slices.Equal(got, want) { + t.Fatalf("retry fetched %v, want %v", got, want) + } + // The retry read the py2.py3 wheel, which keeps it from retention. + maxID, err := m.db.MaxArtifactID() + if err != nil { + t.Fatal(err) + } + candidates, err := m.db.GetRetentionCandidates(0, maxID, time.Now().Add(-24*time.Hour)) + if err != nil { + t.Fatal(err) + } + for _, c := range candidates { + if c.Filename == "demo-1.1.0-py2.py3-none-any.whl" { + t.Errorf("the wheel the retry read is a retention candidate: %+v", c) + } + } m.Versions = stubReleases{"pypi/demo": append(releases, files("1.2.0", "demo-1.2.0-py3-none-any.whl", "demo-1.2.0-py2-none-any.whl"))} @@ -405,9 +428,10 @@ func TestRefreshSeesBufferedDownloads(t *testing.T) { func TestRefreshSkipsAPackageWithNoDownloadedFileKinds(t *testing.T) { m, _ := setupRefreshMirror(t, "wheel") - m.Versions = stubReleases{"pypi/demo": oldReleases("1.0.0", "1.1.0")} - // Eviction removed the file a client downloaded after the ranking query - // ran, leaving only one a refresh mirrored. + // Listing the releases would fail, but there is nothing to list them + // for: eviction removed the file a client downloaded after the ranking + // query ran, leaving only one a refresh mirrored. + m.Versions = stubReleases{} items, err := m.refreshItems(context.Background(), database.UsedPackage{ Ecosystem: "pypi", Name: "demo", PURL: "pkg:pypi/demo", Files: []database.UsedFile{{Version: "1.0.0", Filename: "demo-1.0.0-py3-none-any.whl"}}, From c06861bca6424fd1fafaef03b549ece85619783c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 18 Nov 2026 16:58:50 +0100 Subject: [PATCH 4/4] Record a download for a client joining a shared fetch A client that joined a fetch a refresh started was served without a hit, so the file looked fetched only by the mirror run and its kind was left out of later refreshes. A joining client now records a hit; a joining mirror run still doesn't. --- internal/handler/coalesce_semantics_test.go | 71 +++++++++++++++++++++ internal/handler/handler.go | 5 ++ 2 files changed, 76 insertions(+) diff --git a/internal/handler/coalesce_semantics_test.go b/internal/handler/coalesce_semantics_test.go index 42026f72..910449aa 100644 --- a/internal/handler/coalesce_semantics_test.go +++ b/internal/handler/coalesce_semantics_test.go @@ -613,3 +613,74 @@ func TestCoalesce_PanicInSharedFetchDoesNotStrandWaiters(t *testing.T) { t.Errorf("in-flight entries after a panic = %d, want 0", n) } } + +// TestCoalesce_ClientJoiningAMirrorFetchCountsAsADownload checks that a client +// served by a fetch a mirror run started is recorded as having downloaded the +// artifact. Otherwise the artifact looks fetched only by the mirror run. +func TestCoalesce_ClientJoiningAMirrorFetchCountsAsADownload(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: "bytes", delay: fetchHoldTime, entered: make(chan struct{})} + proxy.Fetcher = fetcher + get := func(ctx context.Context) error { + res, err := proxy.GetOrFetchArtifactFromURL(ctx, "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", + "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz") + drain(res) + return err + } + + mirrored := make(chan error, 1) + go func() { mirrored <- get(MirrorFetch(context.Background())) }() + <-fetcher.entered + if err := get(context.Background()); err != nil { + t.Fatalf("client: %v", err) + } + if err := <-mirrored; err != nil { + t.Fatalf("mirror run: %v", err) + } + if n := fetcher.calls.Load(); n != 1 { + t.Fatalf("upstream fetched %d times, want 1 shared fetch", n) + } + + a, err := db.GetArtifact("pkg:npm/pkg@1.0.0", "pkg-1.0.0.tgz") + if err != nil || a == nil { + t.Fatalf("GetArtifact = %+v, %v", a, err) + } + if !a.LastAccessedAt.Valid || a.HitCount != 1 { + t.Errorf("artifact hits = %d, last accessed %v; want the client's download recorded", a.HitCount, a.LastAccessedAt) + } +} + +// TestCoalesce_MirrorJoiningAClientFetchIsNotADownload checks the other way +// round: a mirror run joining a client's fetch records no download. +func TestCoalesce_MirrorJoiningAClientFetchIsNotADownload(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: "bytes", delay: fetchHoldTime, entered: make(chan struct{})} + proxy.Fetcher = fetcher + get := func(ctx context.Context) error { + res, err := proxy.GetOrFetchArtifactFromURL(ctx, "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", + "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz") + drain(res) + return err + } + + client := make(chan error, 1) + go func() { client <- get(context.Background()) }() + <-fetcher.entered + if err := get(MirrorFetch(context.Background())); err != nil { + t.Fatalf("mirror run: %v", err) + } + if err := <-client; err != nil { + t.Fatalf("client: %v", err) + } + if n := fetcher.calls.Load(); n != 1 { + t.Fatalf("upstream fetched %d times, want 1 shared fetch", n) + } + + a, err := db.GetArtifact("pkg:npm/pkg@1.0.0", "pkg-1.0.0.tgz") + if err != nil || a == nil { + t.Fatalf("GetArtifact = %+v, %v", a, err) + } + if a.HitCount != 0 { + t.Errorf("artifact hits = %d, want none from the mirror run", a.HitCount) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index bd9fd2c0..6df7e105 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -989,6 +989,11 @@ func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (a if f.unrecorded != nil { return p.serveUnrecorded(ctx, f.unrecorded, f.val.artifact, f.val.storagePath) } + // A client served by a fetch it joined downloaded the artifact, even + // when a mirror run started the fetch. + if !mirrorFetch(ctx) { + _ = p.DB.RecordArtifactHit(f.val.artifact.PURL, f.val.artifact.Filename) + } return p.openStoredArtifact(ctx, f.val.artifact, f.val.storagePath) }