Repository navigation
Select refresh file kinds only from files clients downloaded #752
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
d47bdfa
b18a9a0
936f226
c06861b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -613,3 +613,74 @@ func TestCoalesce_PanicInSharedFetchDoesNotStrandWaiters(t *testing.T) { | |
| t.Errorf("in-flight entries after a panic = %d, want 0", n) | ||
| } | ||
| } | ||
|
|
||
| // TestCoalesce_ClientJoiningAMirrorFetchCountsAsADownload checks that a client | ||
| // served by a fetch a mirror run started is recorded as having downloaded the | ||
| // artifact. Otherwise the artifact looks fetched only by the mirror run. | ||
| func TestCoalesce_ClientJoiningAMirrorFetchCountsAsADownload(t *testing.T) { | ||
| proxy, db, _, _ := setupTestProxy(t) | ||
| fetcher := &countingFetcher{content: "bytes", delay: fetchHoldTime, entered: make(chan struct{})} | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Wait for the second caller to join before releasing the fetch.
Use a fetcher that waits on a release channel. Release it after the keyed fetch has a waiter under Based on learnings, confirm that the caller reached the actual deduplication point rather than inferring it from goroutine timing. Also applies to: 657-657
|
||
| proxy.Fetcher = fetcher | ||
| get := func(ctx context.Context) error { | ||
| res, err := proxy.GetOrFetchArtifactFromURL(ctx, "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", | ||
| "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz") | ||
| drain(res) | ||
| return err | ||
| } | ||
|
|
||
| mirrored := make(chan error, 1) | ||
| go func() { mirrored <- get(MirrorFetch(context.Background())) }() | ||
| <-fetcher.entered | ||
| if err := get(context.Background()); err != nil { | ||
| t.Fatalf("client: %v", err) | ||
| } | ||
| if err := <-mirrored; err != nil { | ||
| t.Fatalf("mirror run: %v", err) | ||
| } | ||
| if n := fetcher.calls.Load(); n != 1 { | ||
| t.Fatalf("upstream fetched %d times, want 1 shared fetch", n) | ||
| } | ||
|
|
||
| a, err := db.GetArtifact("pkg:npm/pkg@1.0.0", "pkg-1.0.0.tgz") | ||
| if err != nil || a == nil { | ||
| t.Fatalf("GetArtifact = %+v, %v", a, err) | ||
| } | ||
| if !a.LastAccessedAt.Valid || a.HitCount != 1 { | ||
| t.Errorf("artifact hits = %d, last accessed %v; want the client's download recorded", a.HitCount, a.LastAccessedAt) | ||
| } | ||
| } | ||
|
|
||
| // TestCoalesce_MirrorJoiningAClientFetchIsNotADownload checks the other way | ||
| // round: a mirror run joining a client's fetch records no download. | ||
| func TestCoalesce_MirrorJoiningAClientFetchIsNotADownload(t *testing.T) { | ||
| proxy, db, _, _ := setupTestProxy(t) | ||
| fetcher := &countingFetcher{content: "bytes", delay: fetchHoldTime, entered: make(chan struct{})} | ||
| proxy.Fetcher = fetcher | ||
| get := func(ctx context.Context) error { | ||
| res, err := proxy.GetOrFetchArtifactFromURL(ctx, "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", | ||
| "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz") | ||
| drain(res) | ||
| return err | ||
| } | ||
|
|
||
| client := make(chan error, 1) | ||
| go func() { client <- get(context.Background()) }() | ||
| <-fetcher.entered | ||
| if err := get(MirrorFetch(context.Background())); err != nil { | ||
| t.Fatalf("mirror run: %v", err) | ||
| } | ||
| if err := <-client; err != nil { | ||
| t.Fatalf("client: %v", err) | ||
| } | ||
| if n := fetcher.calls.Load(); n != 1 { | ||
| t.Fatalf("upstream fetched %d times, want 1 shared fetch", n) | ||
| } | ||
|
|
||
| a, err := db.GetArtifact("pkg:npm/pkg@1.0.0", "pkg-1.0.0.tgz") | ||
| if err != nil || a == nil { | ||
| t.Fatalf("GetArtifact = %+v, %v", a, err) | ||
| } | ||
| if a.HitCount != 0 { | ||
| t.Errorf("artifact hits = %d, want none from the mirror run", a.HitCount) | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -565,15 +565,15 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s | |
|
|
||
| if servingHead(ctx) { | ||
| result.rangeable = readsSeekable(p.Storage) | ||
| p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) | ||
| p.recordCacheHit(ctx, artifact.Ecosystem, versionPURL, filename) | ||
| return result, nil | ||
| } | ||
|
|
||
| if p.DirectServe { | ||
| signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL) | ||
| if err == nil { | ||
| result.RedirectURL = signed | ||
| p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) | ||
| p.recordCacheHit(ctx, artifact.Ecosystem, versionPURL, filename) | ||
| return result, nil | ||
| } | ||
| if !errors.Is(err, storage.ErrSignedURLUnsupported) { | ||
|
|
@@ -609,12 +609,19 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s | |
| p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err) | ||
| return nil, nil | ||
| } | ||
| p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) | ||
| p.recordCacheHit(ctx, artifact.Ecosystem, versionPURL, filename) | ||
| return result, nil | ||
| } | ||
|
|
||
| func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) { | ||
| _ = p.DB.RecordArtifactHit(versionPURL, filename) | ||
| // recordCacheHit counts a cache hit. A mirror run's read leaves the | ||
| // artifact's hits and last access alone, so it doesn't look like a client | ||
| // download, and keeps it from retention instead. | ||
| func (p *Proxy) recordCacheHit(ctx context.Context, ecosystem, versionPURL, filename string) { | ||
| if mirrorFetch(ctx) { | ||
| _ = p.DB.KeepMirroredArtifact(versionPURL, filename) | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| } else { | ||
| _ = p.DB.RecordArtifactHit(versionPURL, filename) | ||
| } | ||
| metrics.RecordArtifactCacheHit(ecosystem) | ||
| } | ||
|
|
||
|
|
@@ -982,6 +989,11 @@ func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (a | |
| if f.unrecorded != nil { | ||
| return p.serveUnrecorded(ctx, f.unrecorded, f.val.artifact, f.val.storagePath) | ||
| } | ||
| // A client served by a fetch it joined downloaded the artifact, even | ||
| // when a mirror run started the fetch. | ||
| if !mirrorFetch(ctx) { | ||
| _ = p.DB.RecordArtifactHit(f.val.artifact.PURL, f.val.artifact.Filename) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Record the client hit after the stored artifact opens. If
|
||
| } | ||
| return p.openStoredArtifact(ctx, f.val.artifact, f.val.storagePath) | ||
| } | ||
|
|
||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Flush buffered download hits before the refresh reads them.
RefreshcallsMostUsedPackageswithout first callingFlushHits. Hits from client downloads stay buffered until the flush interval elapses. A file the client downloaded may therefore not count as downloaded in this run, so its kind is omitted from the run's variants. A package whose only recent download is still buffered can also drop out of the run. The next run after a flush picks up the change, so the effect is a delay, not permanent loss.The
FlushHitscontract requires that a caller about to readlast_accessed_atflushes first.Refreshshould callFlushHitsbeforeMostUsedPackages. The choice of error handling is open. The adjacentMostUsedPackageserror is returned, whilesweepRetentionlogs and skips. Choose one and document it.Write the mirror marker in the same transaction as the cache record.
storeArtifactcommits the cache record throughRecordCachedArtifact, then callsMarkArtifactMirroredas a separate write. Until that second write commits, a mirror-filled row hasmirrored_at IS NULL. TheDownloadedexpression atinternal/database/most_used.go:70treats that state as a client download. A refresh that runs in this window can therefore select variants from a file no client requested.If the marker write fails, the error is only logged, and
mirrored_atstays null. The file remains marked as downloaded until a later mirror pass marks it.For mirror fills, write the marker in the same transaction as the cache record, so the row is never visible without its marker. Leave the client-fill path unchanged.
Keep mirror cache reads out of the client access marker.
A refresh that re-runs a release reads each cached selected file through
GetOrFetchArtifactFromURLWithDigest. That path callscheckCache, which reachesreadStored.readStoredcallsrecordCacheHit, which callsRecordArtifactHitwithout checking for a mirror fetch. The write setslast_accessed_at, so theCASEinMostUsedPackagesmarks the mirror-cached file as client-downloaded. The next refresh then builds variants from that file. For a combinedpy2.py3wheel, those variants includepy2-none-any, so a later release with a separatepy2wheel can be fetched even though no client requested it.Skip the hit write for mirror fetches. Pass
ctxtorecordCacheHitand callRecordArtifactHitonly whenmirrorFetch(ctx)is false. This follows the contract stated onHasCachedArtifact, which avoids counting a hit so that a mirror run does not look like client traffic. The stored row and completeness checks are unchanged. Check every caller ofrecordCacheHitwhen you change its signature.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed the flush and the mirror-read hit in b18a9a0: Refresh writes buffered hits first and logs a failure, and a mirror run's cache read no longer records a hit, with tests. I'd leave the separate
mirrored_atwrite as it is here. The window is the same for the ranking query, which already uses this test, and closing it means reworkingstoreArtifact's transaction, which is beyond #750.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@simonchrz Thanks for fixing the flush and mirror-read hit. I understand the decision to leave the separate
mirrored_atwrite outside#750.I won’t mark the thread as addressed because the marker-write concern remains.
If you found this review helpful, would you consider giving us a shout-out on X?
Thank you for using CodeRabbit!
You are interacting with an AI system.