diff --git a/internal/database/most_used.go b/internal/database/most_used.go index a365cb71..ddf38e6c 100644 --- a/internal/database/most_used.go +++ b/internal/database/most_used.go @@ -20,13 +20,17 @@ type UsedFile struct { Version string Filename string UpstreamURL string + // Downloaded is false for a file a mirror run fetched that no client + // has downloaded since. + Downloaded bool } // MostUsedPackages returns, for each ecosystem, the top packages with the most // downloads among the cached files downloaded since cutoff, ordered by -// ecosystem and then by hits. Each lists every file cached for it. Files -// from local sources, which have no upstream, are left out, and so are -// files a mirror run fetched that no client has downloaded since. +// ecosystem and then by hits. Files a mirror run fetched that no client has +// downloaded since don't count toward the ranking. Each package lists every +// file cached for it, those included, and marks the ones clients downloaded. +// Files from local sources, which have no upstream, are left out. func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) { var rows []struct { Ecosystem string `db:"ecosystem"` @@ -59,9 +63,12 @@ func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) VersionPURL string `db:"version_purl"` Filename string `db:"filename"` UpstreamURL string `db:"upstream_url"` + Downloaded int `db:"downloaded"` } err := db.Select(&files, db.Rebind(` - SELECT a.version_purl, a.filename, a.upstream_url + SELECT a.version_purl, a.filename, a.upstream_url, + CASE WHEN a.last_accessed_at IS NOT NULL OR a.mirrored_at IS NULL OR a.mirrored_at < a.fetched_at + THEN 1 ELSE 0 END AS downloaded FROM artifacts a JOIN versions v ON v.purl = a.version_purl WHERE v.package_purl = ? AND a.storage_path IS NOT NULL AND a.source = ? @@ -73,6 +80,7 @@ func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) for _, f := range files { packages[i].Files = append(packages[i].Files, UsedFile{ Version: VersionFromPURL(f.VersionPURL), Filename: f.Filename, UpstreamURL: f.UpstreamURL, + Downloaded: f.Downloaded == 1, }) } } diff --git a/internal/database/most_used_test.go b/internal/database/most_used_test.go index 4b8693e4..31f39f1b 100644 --- a/internal/database/most_used_test.go +++ b/internal/database/most_used_test.go @@ -50,16 +50,16 @@ func TestMostUsedPackages(t *testing.T) { } want := []UsedPackage{ {Ecosystem: "cargo", Name: "serde", PURL: "pkg:cargo/serde", Hits: 1, Files: []UsedFile{ - {Version: "1.0.200", Filename: "serde-1.0.200.crate", UpstreamURL: "https://example.test/serde-1.0.200.crate"}, + {Version: "1.0.200", Filename: "serde-1.0.200.crate", UpstreamURL: "https://example.test/serde-1.0.200.crate", Downloaded: true}, }}, // react has more hits than lodash. Every cached file of lodash is // listed, but only recent downloads count. {Ecosystem: "npm", Name: "react", PURL: "pkg:npm/react", Hits: 10, Files: []UsedFile{ - {Version: "18.3.1", Filename: "react-18.3.1.tgz", UpstreamURL: "https://example.test/react-18.3.1.tgz"}, + {Version: "18.3.1", Filename: "react-18.3.1.tgz", UpstreamURL: "https://example.test/react-18.3.1.tgz", Downloaded: true}, }}, {Ecosystem: "npm", Name: "lodash", PURL: "pkg:npm/lodash", Hits: 5, Files: []UsedFile{ - {Version: "4.17.20", Filename: "lodash-4.17.20.tgz", UpstreamURL: "https://example.test/lodash-4.17.20.tgz"}, - {Version: "4.17.21", Filename: "lodash-4.17.21.tgz", UpstreamURL: "https://example.test/lodash-4.17.21.tgz"}, + {Version: "4.17.20", Filename: "lodash-4.17.20.tgz", UpstreamURL: "https://example.test/lodash-4.17.20.tgz", Downloaded: true}, + {Version: "4.17.21", Filename: "lodash-4.17.21.tgz", UpstreamURL: "https://example.test/lodash-4.17.21.tgz", Downloaded: true}, }}, } if !reflect.DeepEqual(got, want) { @@ -116,3 +116,50 @@ func TestMostUsedPackagesSkipsMirroredArtifactsNotDownloaded(t *testing.T) { } }) } + +func TestMostUsedPackagesMarksTheFilesClientsDownloaded(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + seedUsedArtifact(t, db, "npm", "lodash", "4.17.20", "lodash-4.17.20.tgz", 5, now) + seedUsedArtifact(t, db, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", 0, now) + seedUsedArtifact(t, db, "npm", "lodash", "4.17.22", "lodash-4.17.22.tgz", 0, now) + seedUsedArtifact(t, db, "npm", "lodash", "4.17.23", "lodash-4.17.23.tgz", 0, now) + seedUsedArtifact(t, db, "npm", "lodash", "4.17.24", "lodash-4.17.24.tgz", 0, now) + // A refresh mirrored 4.17.21 and 4.17.22, and a client downloaded + // 4.17.22 since. A client fetched 4.17.23 without a recorded hit, + // and fetched 4.17.24 again after a refresh had mirrored it. + for _, u := range []struct { + filename string + mirrored any + }{ + {"lodash-4.17.21.tgz", now.Add(time.Minute).UTC()}, + {"lodash-4.17.22.tgz", now.Add(time.Minute).UTC()}, + {"lodash-4.17.23.tgz", nil}, + {"lodash-4.17.24.tgz", now.Add(-time.Minute).UTC()}, + } { + if _, err := db.Exec(db.Rebind("UPDATE artifacts SET last_accessed_at = NULL, mirrored_at = ? WHERE filename = ?"), + u.mirrored, u.filename); err != nil { + t.Fatal(err) + } + } + if err := db.RecordArtifactHit("pkg:npm/lodash@4.17.22", "lodash-4.17.22.tgz"); err != nil { + t.Fatal(err) + } + + got, err := db.MostUsedPackages(now.Add(-time.Hour), 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("MostUsedPackages = %+v, want lodash", got) + } + downloaded := map[string]bool{} + for _, f := range got[0].Files { + downloaded[f.Version] = f.Downloaded + } + want := map[string]bool{"4.17.20": true, "4.17.21": false, "4.17.22": true, "4.17.23": true, "4.17.24": true} + if !reflect.DeepEqual(downloaded, want) { + t.Errorf("downloaded = %v, want %v", downloaded, want) + } + }) +} diff --git a/internal/database/retention.go b/internal/database/retention.go index f9e74e3f..b3b7d72b 100644 --- a/internal/database/retention.go +++ b/internal/database/retention.go @@ -141,6 +141,19 @@ func (db *DB) MarkArtifactMirrored(versionPURL, filename string) error { return err } +// KeepMirroredArtifact records that a mirror run read a cached artifact +// again, which keeps it from retention without counting as a client +// download. An artifact a client fetched and hasn't downloaded since is left +// alone: marking it mirrored would make it look fetched by a mirror run. +func (db *DB) KeepMirroredArtifact(versionPURL, filename string) error { + _, err := db.Exec(db.Rebind(` + UPDATE artifacts SET mirrored_at = ? + WHERE version_purl = ? AND filename = ? AND storage_path IS NOT NULL + AND (last_accessed_at IS NOT NULL OR mirrored_at >= fetched_at) + `), time.Now().UTC(), versionPURL, filename) + return err +} + // GetArtifactEcosystem returns the packages.ecosystem value of the package an // artifact version belongs to, or "" when it has no package record. func (db *DB) GetArtifactEcosystem(versionPURL string) (string, error) { diff --git a/internal/database/retention_test.go b/internal/database/retention_test.go index 6ea1a10a..0b719585 100644 --- a/internal/database/retention_test.go +++ b/internal/database/retention_test.go @@ -544,3 +544,42 @@ func TestFlushHitsRunsOneAtATime(t *testing.T) { t.Fatalf("FlushHits: %v", err) } } + +func TestKeepMirroredArtifact(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + cutoff := now.Add(-24 * time.Hour) + old := now.Add(-48 * time.Hour) + seedRetentionArtifact(t, db, "npm", "mirrored", old, time.Time{}) + seedRetentionArtifact(t, db, "npm", "served", old, old) + seedRetentionArtifact(t, db, "npm", "client-fetched", old, time.Time{}) + if _, err := db.Exec(db.Rebind("UPDATE artifacts SET mirrored_at = ? WHERE filename = ?"), old.UTC(), "mirrored.tgz"); err != nil { + t.Fatal(err) + } + for _, name := range []string{"mirrored", "served", "client-fetched"} { + if err := db.KeepMirroredArtifact("pkg:npm/"+name+"@1.0.0", name+".tgz"); err != nil { + t.Fatalf("KeepMirroredArtifact(%s): %v", name, err) + } + } + + maxID, err := db.MaxArtifactID() + if err != nil { + t.Fatal(err) + } + candidates, err := db.GetRetentionCandidates(0, maxID, cutoff) + if err != nil { + t.Fatal(err) + } + got := candidateNames(candidates) + for _, kept := range []string{"mirrored.tgz", "served.tgz"} { + if _, ok := got[kept]; ok { + t.Errorf("%s is a retention candidate after a mirror run read it", kept) + } + } + // Marking this one mirrored would make it look fetched by a mirror + // run rather than a client. + if c, ok := got["client-fetched.tgz"]; !ok || c.MirroredAt.Valid { + t.Errorf("client-fetched.tgz = %+v, %v; want it left unmarked", c, ok) + } + }) +} diff --git a/internal/handler/coalesce_semantics_test.go b/internal/handler/coalesce_semantics_test.go index 42026f72..910449aa 100644 --- a/internal/handler/coalesce_semantics_test.go +++ b/internal/handler/coalesce_semantics_test.go @@ -613,3 +613,74 @@ func TestCoalesce_PanicInSharedFetchDoesNotStrandWaiters(t *testing.T) { t.Errorf("in-flight entries after a panic = %d, want 0", n) } } + +// TestCoalesce_ClientJoiningAMirrorFetchCountsAsADownload checks that a client +// served by a fetch a mirror run started is recorded as having downloaded the +// artifact. Otherwise the artifact looks fetched only by the mirror run. +func TestCoalesce_ClientJoiningAMirrorFetchCountsAsADownload(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: "bytes", delay: fetchHoldTime, entered: make(chan struct{})} + proxy.Fetcher = fetcher + get := func(ctx context.Context) error { + res, err := proxy.GetOrFetchArtifactFromURL(ctx, "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", + "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz") + drain(res) + return err + } + + mirrored := make(chan error, 1) + go func() { mirrored <- get(MirrorFetch(context.Background())) }() + <-fetcher.entered + if err := get(context.Background()); err != nil { + t.Fatalf("client: %v", err) + } + if err := <-mirrored; err != nil { + t.Fatalf("mirror run: %v", err) + } + if n := fetcher.calls.Load(); n != 1 { + t.Fatalf("upstream fetched %d times, want 1 shared fetch", n) + } + + a, err := db.GetArtifact("pkg:npm/pkg@1.0.0", "pkg-1.0.0.tgz") + if err != nil || a == nil { + t.Fatalf("GetArtifact = %+v, %v", a, err) + } + if !a.LastAccessedAt.Valid || a.HitCount != 1 { + t.Errorf("artifact hits = %d, last accessed %v; want the client's download recorded", a.HitCount, a.LastAccessedAt) + } +} + +// TestCoalesce_MirrorJoiningAClientFetchIsNotADownload checks the other way +// round: a mirror run joining a client's fetch records no download. +func TestCoalesce_MirrorJoiningAClientFetchIsNotADownload(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: "bytes", delay: fetchHoldTime, entered: make(chan struct{})} + proxy.Fetcher = fetcher + get := func(ctx context.Context) error { + res, err := proxy.GetOrFetchArtifactFromURL(ctx, "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", + "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz") + drain(res) + return err + } + + client := make(chan error, 1) + go func() { client <- get(context.Background()) }() + <-fetcher.entered + if err := get(MirrorFetch(context.Background())); err != nil { + t.Fatalf("mirror run: %v", err) + } + if err := <-client; err != nil { + t.Fatalf("client: %v", err) + } + if n := fetcher.calls.Load(); n != 1 { + t.Fatalf("upstream fetched %d times, want 1 shared fetch", n) + } + + a, err := db.GetArtifact("pkg:npm/pkg@1.0.0", "pkg-1.0.0.tgz") + if err != nil || a == nil { + t.Fatalf("GetArtifact = %+v, %v", a, err) + } + if a.HitCount != 0 { + t.Errorf("artifact hits = %d, want none from the mirror run", a.HitCount) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9c883053..6df7e105 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -565,7 +565,7 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s if servingHead(ctx) { result.rangeable = readsSeekable(p.Storage) - p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) + p.recordCacheHit(ctx, artifact.Ecosystem, versionPURL, filename) return result, nil } @@ -573,7 +573,7 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL) if err == nil { result.RedirectURL = signed - p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) + p.recordCacheHit(ctx, artifact.Ecosystem, versionPURL, filename) return result, nil } if !errors.Is(err, storage.ErrSignedURLUnsupported) { @@ -609,12 +609,19 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err) return nil, nil } - p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) + p.recordCacheHit(ctx, artifact.Ecosystem, versionPURL, filename) return result, nil } -func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) { - _ = p.DB.RecordArtifactHit(versionPURL, filename) +// recordCacheHit counts a cache hit. A mirror run's read leaves the +// artifact's hits and last access alone, so it doesn't look like a client +// download, and keeps it from retention instead. +func (p *Proxy) recordCacheHit(ctx context.Context, ecosystem, versionPURL, filename string) { + if mirrorFetch(ctx) { + _ = p.DB.KeepMirroredArtifact(versionPURL, filename) + } else { + _ = p.DB.RecordArtifactHit(versionPURL, filename) + } metrics.RecordArtifactCacheHit(ecosystem) } @@ -982,6 +989,11 @@ func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (a if f.unrecorded != nil { return p.serveUnrecorded(ctx, f.unrecorded, f.val.artifact, f.val.storagePath) } + // A client served by a fetch it joined downloaded the artifact, even + // when a mirror run started the fetch. + if !mirrorFetch(ctx) { + _ = p.DB.RecordArtifactHit(f.val.artifact.PURL, f.val.artifact.Filename) + } return p.openStoredArtifact(ctx, f.val.artifact, f.val.storagePath) } diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index 513f2398..9715d2a3 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -59,6 +59,12 @@ func (m *Mirror) releaseLister() ReleaseLister { //nolint:ireturn // any lister func (m *Mirror) Refresh(ctx context.Context, r Refresh) (*Progress, error) { ctx = handler.MirrorFetch(ctx) cutoff := time.Now().Add(-r.Since) + // Without the buffered hits, this run misses the downloads still in + // them, and a later run only sees those still within Since. That is + // logged rather than stopping the run. + if err := m.db.FlushHits(); err != nil { + m.logger.Warn("mirror refresh: failed to write buffered hits", "error", err) + } packages, err := m.db.MostUsedPackages(cutoff, r.Top) if err != nil { return nil, fmt.Errorf("listing the most used packages: %w", err) @@ -134,6 +140,16 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] if !refreshable(ecosystem) { return nil, nil } + eco, files := fileEcosystems[ecosystem] + var variants map[string]bool + if files { + variants = cachedVariants(eco, pkg.Files) + if len(variants) == 0 { + // Eviction removed every file a client downloaded after the + // ranking query ran. + return nil, nil + } + } releases, err := m.releaseLister().ListReleases(ctx, ecosystem, name) if err != nil { return nil, fmt.Errorf("listing releases: %w", err) @@ -155,11 +171,8 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] return m.proxy == nil || m.proxy.ReleaseAllowed(pkg.Ecosystem, pkg.Name, v.Number, v.PublishedAt) } - eco, files := fileEcosystems[ecosystem] - var variants map[string]bool var incomplete func(registries.Version) bool if files { - variants = cachedVariants(eco, pkg.Files) incomplete = func(r registries.Version) bool { return eco.missing(PackageVersion{Ecosystem: ecosystem, Name: name, Version: r.Number, variants: variants}, r, pkg.Files) } @@ -172,9 +185,14 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] return items, nil } +// cachedVariants returns the kinds of files clients downloaded. Files a +// refresh mirrored are left out, so they don't widen later refreshes. func cachedVariants(eco fileEcosystem, files []database.UsedFile) map[string]bool { variants := map[string]bool{} for _, f := range files { + if !f.Downloaded { + continue + } for _, v := range eco.cachedVariants(f.Filename) { variants[v] = true } diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index 3bf7c5cb..c2587171 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -12,6 +12,8 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/registries" "github.com/git-pkgs/registries/fetch" ) @@ -298,6 +300,147 @@ func TestRefreshMirrorsTheFileVariantsClientsFetched(t *testing.T) { } } +func TestRefreshSelectsVariantsOnlyFromFilesClientsFetched(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "wheel") + ctx := context.Background() + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "pypi", "demo", "1.0.0", "demo-1.0.0-py3-none-any.whl", "https://files.example/demo-1.0.0-py3-none-any.whl", "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + files := func(version string, names ...string) registries.Version { + v := registries.Version{Number: version, PublishedAt: time.Now().Add(-30 * 24 * time.Hour)} + for _, n := range names { + v.Artifacts = append(v.Artifacts, registries.Artifact{Filename: n, URL: "https://files.example/" + n}) + } + return v + } + releases := []registries.Version{ + files("1.0.0", "demo-1.0.0-py3-none-any.whl"), + files("1.1.0", "demo-1.1.0-py2.py3-none-any.whl"), + } + m.Versions = stubReleases{"pypi/demo": releases} + if _, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}); err != nil { + t.Fatalf("Refresh: %v", err) + } + if got := fetchedMatching(fetcher.fetched(), "1.1.0"); len(got) != 1 { + t.Fatalf("first refresh fetched %v, want the 1.1.0 wheel", got) + } + + // The mirrored py2.py3 wheel adds py2-none-any to its variants, but no + // client fetched that kind. + m.Versions = stubReleases{"pypi/demo": append(releases, + files("1.2.0", "demo-1.2.0-py3-none-any.whl", "demo-1.2.0-py2-none-any.whl"))} + fetcher.reset() + if _, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}); err != nil { + t.Fatalf("Refresh: %v", err) + } + if got, want := fetcher.fetched(), []string{"https://files.example/demo-1.2.0-py3-none-any.whl"}; !slices.Equal(got, want) { + t.Errorf("second refresh fetched %v, want %v", got, want) + } +} + +func TestRefreshRetryDoesNotCountCachedFilesAsDownloaded(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "wheel") + ctx := context.Background() + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "pypi", "demo", "1.0.0", "demo-1.0.0-py3-none-any.whl", "https://files.example/demo-1.0.0-py3-none-any.whl", "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + files := func(version string, names ...string) registries.Version { + v := registries.Version{Number: version, PublishedAt: time.Now().Add(-30 * 24 * time.Hour)} + for _, n := range names { + v.Artifacts = append(v.Artifacts, registries.Artifact{Filename: n, URL: "https://files.example/" + n}) + } + return v + } + releases := []registries.Version{ + files("1.0.0", "demo-1.0.0-py3-none-any.whl"), + files("1.1.0", "demo-1.1.0-py2.py3-none-any.whl", "demo-1.1.0-1-py3-none-any.whl"), + } + m.Versions = stubReleases{"pypi/demo": releases} + refresh := func() { + t.Helper() + if _, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}); err != nil { + t.Fatalf("Refresh: %v", err) + } + } + // The first refresh caches the py2.py3 wheel and fails on the other, + // so the second reads the py2.py3 wheel from the cache again. + fetcher.failing("1.1.0-1-py3") + refresh() + old := time.Now().Add(-48 * time.Hour).UTC() + if _, err := m.db.Exec(m.db.Rebind("UPDATE artifacts SET fetched_at = ?, mirrored_at = ? WHERE filename = ?"), + old, old, "demo-1.1.0-py2.py3-none-any.whl"); err != nil { + t.Fatal(err) + } + fetcher.failing("") + fetcher.reset() + refresh() + if got, want := fetcher.fetched(), []string{"https://files.example/demo-1.1.0-1-py3-none-any.whl"}; !slices.Equal(got, want) { + t.Fatalf("retry fetched %v, want %v", got, want) + } + // The retry read the py2.py3 wheel, which keeps it from retention. + maxID, err := m.db.MaxArtifactID() + if err != nil { + t.Fatal(err) + } + candidates, err := m.db.GetRetentionCandidates(0, maxID, time.Now().Add(-24*time.Hour)) + if err != nil { + t.Fatal(err) + } + for _, c := range candidates { + if c.Filename == "demo-1.1.0-py2.py3-none-any.whl" { + t.Errorf("the wheel the retry read is a retention candidate: %+v", c) + } + } + + m.Versions = stubReleases{"pypi/demo": append(releases, + files("1.2.0", "demo-1.2.0-py3-none-any.whl", "demo-1.2.0-py2-none-any.whl"))} + fetcher.reset() + refresh() + if got, want := fetcher.fetched(), []string{"https://files.example/demo-1.2.0-py3-none-any.whl"}; !slices.Equal(got, want) { + t.Errorf("refresh after the retry fetched %v, want %v", got, want) + } +} + +func TestRefreshSeesBufferedDownloads(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + m.db.BatchHits(time.Hour, nil) + result, err := m.proxy.GetOrFetchArtifact(handler.MirrorFetch(context.Background()), "npm", "lodash", "4.17.20", "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + // A client downloads the mirrored version; the hit is still buffered. + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21")} + fetcher.reset() + + if _, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour}); err != nil { + t.Fatalf("Refresh: %v", err) + } + if got := fetchedMatching(fetcher.fetched(), "lodash-4.17.21.tgz"); len(got) != 1 { + t.Errorf("fetched %v, want lodash 4.17.21", fetcher.fetched()) + } +} + +func TestRefreshSkipsAPackageWithNoDownloadedFileKinds(t *testing.T) { + m, _ := setupRefreshMirror(t, "wheel") + // Listing the releases would fail, but there is nothing to list them + // for: eviction removed the file a client downloaded after the ranking + // query ran, leaving only one a refresh mirrored. + m.Versions = stubReleases{} + items, err := m.refreshItems(context.Background(), database.UsedPackage{ + Ecosystem: "pypi", Name: "demo", PURL: "pkg:pypi/demo", + Files: []database.UsedFile{{Version: "1.0.0", Filename: "demo-1.0.0-py3-none-any.whl"}}, + }) + if err != nil || len(items) != 0 { + t.Errorf("refreshItems = %+v, %v; want none", items, err) + } +} + func TestFileVariants(t *testing.T) { tests := []struct { ecosystem, cacheFilename, want string