From 3d05d61a14474c3bdbaed34bb7221b433a29fd58 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Wed, 28 Oct 2026 11:32:03 -0400 Subject: [PATCH 1/2] docs: add title frontmatter for auxesia Auxesia does not start when a page has no YAML frontmatter with a title. No page in docs/ had one. Each page now gets a title from its first heading. The .auxesia.yaml file gives the site the name objgit. Assisted-by: Claude Opus 5.5 via Claude Code Signed-off-by: Xe Iaso --- docs/.auxesia.yaml | 2 + docs/architecture/README.md | 4 + docs/architecture/auth.md | 4 + docs/architecture/hooks.md | 4 + docs/architecture/lfs.md | 32 ++++--- docs/architecture/metrics.md | 4 + docs/architecture/s3fs.md | 4 + docs/architecture/snapshots.md | 8 +- docs/architecture/tigris-storer.md | 4 + docs/architecture/transports.md | 4 + docs/plans/bin-cue-benchmark.md | 84 ++++++++-------- docs/plans/bound-concurrent-pushes.md | 4 + docs/plans/erofs-lfs-resolution.md | 4 + docs/plans/git-default-branck-problems.md | 4 + docs/plans/git-hooks-kefka.md | 4 + docs/plans/git-hooks-stream.md | 4 + docs/plans/git-http-protocol.md | 4 + docs/plans/git-keep-pushed-packs-whole.md | 4 + docs/plans/kefka-uutils.md | 4 + docs/plans/lfs.md | 93 +++++++++--------- docs/plans/pack-compaction-implementation.md | 74 ++++++++++---- docs/plans/pack-compaction.md | 14 ++- docs/plans/pack-prefetch.md | 16 ++-- docs/plans/pack-streaming.md | 4 + docs/plans/pack-temp-file-cache.md | 4 + docs/plans/per-repo-bucket.md | 4 + docs/plans/per-repo-fs-dispatch.md | 4 + docs/plans/prometheus-metrics.md | 4 + docs/plans/push-memory.md | 4 + docs/plans/s3fs-groupcache-listobjectsv2.md | 12 ++- docs/plans/s3fs-lazy-read.md | 4 + docs/plans/s3fs-reduce-listobjectsv2.md | 4 + docs/plans/s3fs-unix-metadata.md | 8 +- docs/plans/ssh-protocol.md | 67 +++++++------ docs/plans/streaming-container-sink.md | 18 ++-- docs/plans/tekton-kubernetes-commands.md | 4 + docs/plans/tigris-bin-cue-packs.md | 4 + docs/plans/tigris-delta-reuse.md | 22 +++-- docs/plans/tigris-gogit-storer.md | 44 ++++++--- docs/plans/webhook-push-events.md | 4 + docs/reference/how-tigris-fs-unix-metadata.md | 36 +++---- docs/reference/tigris-backend.md | 24 +++-- .../plans/2026-08-28-packed-refs-cas.md | 22 ++++- .../plans/2026-09-23-erofs-snapshots.md | 4 + .../2026-08-27-cue-compression-design.md | 60 ++++++------ .../2026-08-28-packed-refs-cas-design.md | 96 ++++++++++--------- .../2026-09-23-erofs-snapshots-design.md | 4 + ...26-09-23-erofs-streaming-builder-design.md | 4 + docs/usage/format-benchmark.md | 56 ++++++----- docs/usage/hooks.md | 4 + docs/usage/kubernetes-hooks.md | 4 + docs/usage/kubernetes.md | 4 + docs/usage/lfs.md | 20 ++-- docs/usage/memory-benchmark.md | 4 + docs/usage/webhooks.md | 4 + 55 files changed, 616 insertions(+), 328 deletions(-) create mode 100644 docs/.auxesia.yaml diff --git a/docs/.auxesia.yaml b/docs/.auxesia.yaml new file mode 100644 index 0000000..d999dbe --- /dev/null +++ b/docs/.auxesia.yaml @@ -0,0 +1,2 @@ +title: objgit +subtitle: A git server that stores repositories in Tigris diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 55c71c6..7895a8d 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -1,3 +1,7 @@ +--- +title: "Architecture" +--- + # Architecture `objgitd` is one binary. It stores git repositories as objects in a Tigris diff --git a/docs/architecture/auth.md b/docs/architecture/auth.md index ccc8ac9..c329266 100644 --- a/docs/architecture/auth.md +++ b/docs/architecture/auth.md @@ -1,3 +1,7 @@ +--- +title: "The auth seam (`internal/auth`)" +--- + # The auth seam (`internal/auth`) Every transport authorizes through one interface. A real authentication and diff --git a/docs/architecture/hooks.md b/docs/architecture/hooks.md index 4e2c4c8..c8b0d7d 100644 --- a/docs/architecture/hooks.md +++ b/docs/architecture/hooks.md @@ -1,3 +1,7 @@ +--- +title: "Push hooks (`hooks.go`)" +--- + # Push hooks (`hooks.go`) When `-allow-hooks` is set, a successful `receive-pack` runs the repository diff --git a/docs/architecture/lfs.md b/docs/architecture/lfs.md index b4f11de..05a406c 100644 --- a/docs/architecture/lfs.md +++ b/docs/architecture/lfs.md @@ -1,3 +1,7 @@ +--- +title: "Git LFS (`internal/lfs`)" +--- + # Git LFS (`internal/lfs`) `objgitd` answers the Git LFS batch API with presigned Tigris URLs. The client @@ -16,12 +20,12 @@ route, so every LFS path answers 404. ## Layout in the bucket -| Key | Contents | -| --- | --- | -| `lfs/blobs/` | The object bytes, stored once for the whole bucket | -| `//lfs/staging/` | An upload in flight, before it is verified | +| Key | Contents | +| ---------------------------------- | ----------------------------------------------------- | +| `lfs/blobs/` | The object bytes, stored once for the whole bucket | +| `//lfs/staging/` | An upload in flight, before it is verified | | `//lfs/objects/` | A zero-byte marker: this repository holds that object | -| `//lfs/locks.json` | Every lock held in this repository | +| `//lfs/locks.json` | Every lock held in this repository | `` is the SHA-256 of the object, in lowercase hexadecimal. @@ -92,7 +96,7 @@ the query string. `X-Amz-Checksum-Sha256` is one of those, and a hoisted checksum is a checksum nobody sends and nothing enforces. `noHoistPresigner` turns hoisting off for uploads. `TestPresignPutSignsTheChecksumHeader` pins it. -**At verify time**, `Verify` heads the *staged* object with the checksum mode +**At verify time**, `Verify` heads the _staged_ object with the checksum mode enabled and compares both the length and the digest. This is the only check that runs on the daemon's own credentials. It is also the only check on size, because `Content-Length` is not signed on a presigned PUT and cannot be enforced there. @@ -113,13 +117,13 @@ client sends that header back. A download URL has to stand on its own, so only ## Batch outcomes -| Operation | Marker | Response for that object | -| --- | --- | --- | -| `download` | present | `actions.download`, a presigned GET | -| `download` | absent | `error`, code 404 | -| `upload` | present, size matches | no `actions` key, so the client skips it | -| `upload` | present, size differs | `error`, code 422 | -| `upload` | absent | `actions.upload` and `actions.verify` | +| Operation | Marker | Response for that object | +| ---------- | --------------------- | ---------------------------------------- | +| `download` | present | `actions.download`, a presigned GET | +| `download` | absent | `error`, code 404 | +| `upload` | present, size matches | no `actions` key, so the client skips it | +| `upload` | present, size differs | `error`, code 422 | +| `upload` | absent | `actions.upload` and `actions.verify` | Every object carries `authenticated: true`. Without it git-lfs runs credential discovery against the storage host and can attach an `Authorization` header, @@ -205,7 +209,7 @@ trusting a change to `presign.go` or `store.go`. - Nothing deletes a blob. The marker gives a collector its reference count, but no collector exists. -- `-lfs-max-size` caps the size a client *declares*, not the size it uploads. +- `-lfs-max-size` caps the size a client _declares_, not the size it uploads. Content-Length is not signed on a presigned PUT, so the real limit at upload time is what the bucket accepts. `Verify` catches the difference, but a client that never verifies leaves the staged object behind. Staged objects under diff --git a/docs/architecture/metrics.md b/docs/architecture/metrics.md index 1cb36a4..4d0d97b 100644 --- a/docs/architecture/metrics.md +++ b/docs/architecture/metrics.md @@ -1,3 +1,7 @@ +--- +title: "Metrics (`internal/metrics`)" +--- + # Metrics (`internal/metrics`) `internal/metrics` defines every Prometheus vector with `promauto`, against diff --git a/docs/architecture/s3fs.md b/docs/architecture/s3fs.md index 85ea208..3bfccdd 100644 --- a/docs/architecture/s3fs.md +++ b/docs/architecture/s3fs.md @@ -1,3 +1,7 @@ +--- +title: "`internal/s3fs` — a billy.Filesystem on Tigris" +--- + # `internal/s3fs` — a billy.Filesystem on Tigris This package is vendored from the s3fs of Austin Poor. It is adapted to diff --git a/docs/architecture/snapshots.md b/docs/architecture/snapshots.md index b3d97cb..33df3ad 100644 --- a/docs/architecture/snapshots.md +++ b/docs/architecture/snapshots.md @@ -1,3 +1,7 @@ +--- +title: "erofs snapshots (`internal/snapshot`)" +--- + # erofs snapshots (`internal/snapshot`) After a push, `objgitd` can build one EROFS image of the tree at the tip of @@ -14,8 +18,8 @@ The design is in ## Object layout -| Key | Contents | -| --------------------------------- | -------------------------------------------------------------------- | +| Key | Contents | +| --------------------------------- | --------------------------------------------------------------------- | | `snapshots/erofs/v2/.erofs` | One EROFS image of the git tree ``, with LFS pointers resolved. | The key uses the tree hash, not the commit hash. Many commits can point at one diff --git a/docs/architecture/tigris-storer.md b/docs/architecture/tigris-storer.md index 4bc951b..8a5aff5 100644 --- a/docs/architecture/tigris-storer.md +++ b/docs/architecture/tigris-storer.md @@ -1,3 +1,7 @@ +--- +title: "`internal/storage/tigris` — a go-git `storage.Storer` on one Tigris bucket" +--- + # `internal/storage/tigris` — a go-git `storage.Storer` on one Tigris bucket This package implements `storage.Storer` for Tigris. It speaks diff --git a/docs/architecture/transports.md b/docs/architecture/transports.md index e3199c3..175dbd2 100644 --- a/docs/architecture/transports.md +++ b/docs/architecture/transports.md @@ -1,3 +1,7 @@ +--- +title: "Transports" +--- + # Transports objgitd speaks three git transports. All three answer the protocol natively diff --git a/docs/plans/bin-cue-benchmark.md b/docs/plans/bin-cue-benchmark.md index db254f4..3dc5253 100644 --- a/docs/plans/bin-cue-benchmark.md +++ b/docs/plans/bin-cue-benchmark.md @@ -1,3 +1,7 @@ +--- +title: "Benchmark objgit before and after the bin/cue format change" +--- + # Benchmark objgit before and after the bin/cue format change Status: Implemented as `cmd/formatbench`. See @@ -28,10 +32,10 @@ plus `packs/.cue` containers through `internal/storage/tigris`. ## What "before" and "after" mean -| Column | Ref | What it runs | -| --- | --- | --- | -| before | `v1.0.2` | `internal/s3fs` + go-git dotgit. One bucket. Listing cache and pack cache on. | -| after | `main` (`ce5a7ca`, v1.5.0) | `internal/storage/tigris`. bin/cue containers, zstd, cue v3. | +| Column | Ref | What it runs | +| ------ | -------------------------- | ----------------------------------------------------------------------------- | +| before | `v1.0.2` | `internal/s3fs` + go-git dotgit. One bucket. Listing cache and pack cache on. | +| after | `main` (`ce5a7ca`, v1.5.0) | `internal/storage/tigris`. bin/cue containers, zstd, cue v3. | `v1.0.2` is the last release where repositories live in the single `-bucket` flag, which is also how HEAD works. That keeps the setup identical on both @@ -55,10 +59,10 @@ One thing about this pair goes in the post as a caveat, not as a hidden fact: Mirror-clone each repository once. Never write to the mirror. -| Name | URL | Shape | -| --- | --- | --- | -| objgit | https://github.com/tigrisdata/objgit | Small. Mostly text. | -| x | https://github.com/Xe/x | Medium. Large object count. | +| Name | URL | Shape | +| ----------- | ----------------------------------------- | ----------------------------------- | +| objgit | https://github.com/tigrisdata/objgit | Small. Mostly text. | +| x | https://github.com/Xe/x | Medium. Large object count. | | tigris-blog | https://github.com/tigrisdata/tigris-blog | Blob-heavy. Images do not compress. | Confirm the `tigris-blog` clone URL before the first run. Record the resolved @@ -77,14 +81,14 @@ at `cmd/objgitd/main.go:81`. HEAD feeds it from `Storer.observe` Per cell, that is per build and per repository and per workload: -| Field | Source | -| --- | --- | -| Wall seconds | Driver timer around the `git` subprocess. | -| S3 requests by verb | Delta of `objgit_s3_requests_total{operation}`. | -| S3 time | Delta of `objgit_s3_request_duration_seconds_sum`. | -| Keys written | `ListObjectsV2` over the repository prefix after the push. | -| Bytes stored | Sum of the sizes from that same listing. | -| Bytes on the wire | Clone only. The `Receiving objects` line from git stderr. | +| Field | Source | +| ------------------- | ---------------------------------------------------------- | +| Wall seconds | Driver timer around the `git` subprocess. | +| S3 requests by verb | Delta of `objgit_s3_requests_total{operation}`. | +| S3 time | Delta of `objgit_s3_request_duration_seconds_sum`. | +| Keys written | `ListObjectsV2` over the repository prefix after the push. | +| Bytes stored | Sum of the sizes from that same listing. | +| Bytes on the wire | Clone only. The `Receiving objects` line from git stderr. | Both eras key a repository under its `orgID/name` path. `v1.0.2` gets there through the go-git loader (`cmd/objgitd/git_protocol.go:61`). HEAD gets there @@ -132,7 +136,7 @@ Reuse these patterns from membench rather than inventing new ones: 2. Make a fresh empty directory for `-pack-cache-dir`. 3. Start the daemon and wait for it to answer. Flags for both builds: `-bucket $BUCKET -http-bind 127.0.0.1:8080 -metrics-bind 127.0.0.1:9090 - -ssh-bind= -allow-push -pack-cache-dir `. Leave every other flag at +-ssh-bind= -allow-push -pack-cache-dir `. Leave every other flag at its default, so each build runs the way it shipped. 4. Scrape `/metrics` and record the counters. 5. Run `git push --mirror http://127.0.0.1:8080/bench/.git` from the @@ -197,12 +201,12 @@ Two files in the run directory: Push: | repo | before wall | after wall | before PUTs | after PUTs | before keys | after keys | before bytes | after bytes | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| ---- | ----------- | ---------- | ----------- | ---------- | ----------- | ---------- | ------------ | ----------- | Clone: | repo | before wall | after wall | before GETs | after GETs | wire bytes | -| --- | --- | --- | --- | --- | --- | +| ---- | ----------- | ---------- | ----------- | ---------- | ---------- | End the section with this line, as written: @@ -210,19 +214,19 @@ End the section with this line, as written: ## Files -| Path | Change | -| --- | --- | -| `cmd/formatbench/main.go` | New. The driver: setup, cell loop, interleaving, timeouts. | -| `cmd/formatbench/metrics.go` | New. Scrape and diff `objgit_s3_requests_total` by label. | -| `cmd/formatbench/bucket.go` | New. List a prefix, sum key count and bytes. | -| `cmd/formatbench/git.go` | New. Timed git subprocesses and the wire-byte parser. | -| `cmd/formatbench/verify.go` | New. The correctness gate. | -| `cmd/formatbench/report.go` | New. `results.json` and `results.md`. | -| `cmd/formatbench/uuid.go` | New. Collision-free repository names. | -| `docs/plans/bin-cue-benchmark.md` | New. This plan, committed where `AGENTS.md` says plans live. | -| `docs/usage/format-benchmark.md` | New. How to run it. Follow `docs/usage/memory-benchmark.md` in shape. | -| `AGENTS.md` | One row in the "Where the code lives" table, next to `cmd/membench/`. | -| `var/bin-cue-git.md` | Replace the `TODO(Xe)` at line 66 with the finished tables. | +| Path | Change | +| --------------------------------- | --------------------------------------------------------------------- | +| `cmd/formatbench/main.go` | New. The driver: setup, cell loop, interleaving, timeouts. | +| `cmd/formatbench/metrics.go` | New. Scrape and diff `objgit_s3_requests_total` by label. | +| `cmd/formatbench/bucket.go` | New. List a prefix, sum key count and bytes. | +| `cmd/formatbench/git.go` | New. Timed git subprocesses and the wire-byte parser. | +| `cmd/formatbench/verify.go` | New. The correctness gate. | +| `cmd/formatbench/report.go` | New. `results.json` and `results.md`. | +| `cmd/formatbench/uuid.go` | New. Collision-free repository names. | +| `docs/plans/bin-cue-benchmark.md` | New. This plan, committed where `AGENTS.md` says plans live. | +| `docs/usage/format-benchmark.md` | New. How to run it. Follow `docs/usage/memory-benchmark.md` in shape. | +| `AGENTS.md` | One row in the "Where the code lives" table, next to `cmd/membench/`. | +| `var/bin-cue-git.md` | Replace the `TODO(Xe)` at line 66 with the finished tables. | ## First results @@ -233,17 +237,17 @@ old layout pays on a big one. Push: -| Build | Wall | S3 requests | PUT | GET | LIST | Keys | Bucket bytes | -| --- | --: | --: | --: | --: | --: | --: | --: | -| before | 12.6s | 231 | 46 | 47 | 138 | 42 | 829.27 KiB | -| after | 2.1s | 19 | 5 | 11 | 3 | 4 | 902.98 KiB | +| Build | Wall | S3 requests | PUT | GET | LIST | Keys | Bucket bytes | +| ------ | ----: | ----------: | --: | --: | ---: | ---: | -----------: | +| before | 12.6s | 231 | 46 | 47 | 138 | 42 | 829.27 KiB | +| after | 2.1s | 19 | 5 | 11 | 3 | 4 | 902.98 KiB | Clone: -| Build | Wall | S3 requests | GET | LIST | Wire bytes | -| --- | --: | --: | --: | --: | --: | -| before | 14.3s | 323 | 51 | 272 | 763.63 KiB | -| after | 13.4s | 53 | 49 | 4 | 767.96 KiB | +| Build | Wall | S3 requests | GET | LIST | Wire bytes | +| ------ | ----: | ----------: | --: | ---: | ---------: | +| before | 14.3s | 323 | 51 | 272 | 763.63 KiB | +| after | 13.4s | 53 | 49 | 4 | 767.96 KiB | Two things to read from this before the full run: diff --git a/docs/plans/bound-concurrent-pushes.md b/docs/plans/bound-concurrent-pushes.md index 3cdd833..431d497 100644 --- a/docs/plans/bound-concurrent-pushes.md +++ b/docs/plans/bound-concurrent-pushes.md @@ -1,3 +1,7 @@ +--- +title: "Plan: bound the number of concurrent pushes" +--- + # Plan: bound the number of concurrent pushes ## Context diff --git a/docs/plans/erofs-lfs-resolution.md b/docs/plans/erofs-lfs-resolution.md index 40ab6a0..a132b9f 100644 --- a/docs/plans/erofs-lfs-resolution.md +++ b/docs/plans/erofs-lfs-resolution.md @@ -1,3 +1,7 @@ +--- +title: "Resolve LFS pointers in EROFS snapshots" +--- + # Resolve LFS pointers in EROFS snapshots ## Goal diff --git a/docs/plans/git-default-branck-problems.md b/docs/plans/git-default-branck-problems.md index 7936e95..09af6cb 100644 --- a/docs/plans/git-default-branck-problems.md +++ b/docs/plans/git-default-branck-problems.md @@ -1,3 +1,7 @@ +--- +title: "Fix: clone leaves `warning: remote HEAD refers to nonexistent ref, unable to checkout`" +--- + # Fix: clone leaves `warning: remote HEAD refers to nonexistent ref, unable to checkout` ## Context diff --git a/docs/plans/git-hooks-kefka.md b/docs/plans/git-hooks-kefka.md index f6be3b6..0b18dbe 100644 --- a/docs/plans/git-hooks-kefka.md +++ b/docs/plans/git-hooks-kefka.md @@ -1,3 +1,7 @@ +--- +title: "Plan: post-receive hooks for objgitd (sandboxed via kefka)" +--- + # Plan: post-receive hooks for objgitd (sandboxed via kefka) ## Context diff --git a/docs/plans/git-hooks-stream.md b/docs/plans/git-hooks-stream.md index 729ce92..132b215 100644 --- a/docs/plans/git-hooks-stream.md +++ b/docs/plans/git-hooks-stream.md @@ -1,3 +1,7 @@ +--- +title: "Stream push-hook output to the client live" +--- + # Stream push-hook output to the client live ## Context diff --git a/docs/plans/git-http-protocol.md b/docs/plans/git-http-protocol.md index abec5e9..5b4cae2 100644 --- a/docs/plans/git-http-protocol.md +++ b/docs/plans/git-http-protocol.md @@ -1,3 +1,7 @@ +--- +title: "Plan: Git Smart HTTP protocol for objgitd" +--- + # Plan: Git Smart HTTP protocol for objgitd ## Context diff --git a/docs/plans/git-keep-pushed-packs-whole.md b/docs/plans/git-keep-pushed-packs-whole.md index 8e2e96a..aed23f0 100644 --- a/docs/plans/git-keep-pushed-packs-whole.md +++ b/docs/plans/git-keep-pushed-packs-whole.md @@ -1,3 +1,7 @@ +--- +title: "Keep pushed packs whole on git:// and SSH" +--- + # Keep pushed packs whole on git:// and SSH ## Context diff --git a/docs/plans/kefka-uutils.md b/docs/plans/kefka-uutils.md index eda564f..66216c7 100644 --- a/docs/plans/kefka-uutils.md +++ b/docs/plans/kefka-uutils.md @@ -1,3 +1,7 @@ +--- +title: "Update kefka and add uutils to the hook shell" +--- + # Update kefka and add uutils to the hook shell ## Context diff --git a/docs/plans/lfs.md b/docs/plans/lfs.md index 49e3d96..83a5361 100644 --- a/docs/plans/lfs.md +++ b/docs/plans/lfs.md @@ -1,3 +1,7 @@ +--- +title: "Plan: Git LFS support" +--- + # Plan: Git LFS support ## Context @@ -35,24 +39,24 @@ is marked CAUTION below, and each is verified against source, not assumed. ## Files -| Action | Path | Purpose | -| --- | --- | --- | -| Create | `internal/lfs/lfs.go` | Package doc, oid validation, key layout. | -| Create | `internal/lfs/batch.go` | Batch types and the batch resolver. | -| Create | `internal/lfs/store.go` | The bucket surface: `s3API` and the presigner. | -| Create | `internal/lfs/presign.go` | The custom `HTTPPresignerV4` that stops header hoisting. | -| Create | `internal/lfs/locks.go` | The lock document and its compare-and-swap writer. | -| Create | `internal/lfs/errors.go` | The `application/vnd.git-lfs+json` error body. | -| Create | `cmd/objgitd/lfs.go` | The HTTP handlers, on `*daemon`. | -| Edit | `cmd/objgitd/http.go` | New routes, registered only when LFS is on. | -| Edit | `cmd/objgitd/ssh.go` | Intercept `git-lfs-authenticate`. | -| Edit | `cmd/objgitd/git_protocol.go` | One new `daemon` field. | -| Edit | `internal/repofs/repofs.go` | Reject reserved organization names. | -| Edit | `cmd/objgitd/main.go` | Flags, the presign client, and the wiring. | -| Edit | `internal/metrics/metrics.go` | The `objgit_lfs_*` vectors and helpers. | -| Create | `internal/lfs/*_test.go`, `cmd/objgitd/lfs_test.go` | Tests. | -| Create | `docs/architecture/lfs.md`, `docs/usage/lfs.md`, `docs/plans/lfs.md` | Documentation. | -| Edit | `docs/architecture/README.md`, `AGENTS.md`, `README.md` | Index rows. | +| Action | Path | Purpose | +| ------ | -------------------------------------------------------------------- | -------------------------------------------------------- | +| Create | `internal/lfs/lfs.go` | Package doc, oid validation, key layout. | +| Create | `internal/lfs/batch.go` | Batch types and the batch resolver. | +| Create | `internal/lfs/store.go` | The bucket surface: `s3API` and the presigner. | +| Create | `internal/lfs/presign.go` | The custom `HTTPPresignerV4` that stops header hoisting. | +| Create | `internal/lfs/locks.go` | The lock document and its compare-and-swap writer. | +| Create | `internal/lfs/errors.go` | The `application/vnd.git-lfs+json` error body. | +| Create | `cmd/objgitd/lfs.go` | The HTTP handlers, on `*daemon`. | +| Edit | `cmd/objgitd/http.go` | New routes, registered only when LFS is on. | +| Edit | `cmd/objgitd/ssh.go` | Intercept `git-lfs-authenticate`. | +| Edit | `cmd/objgitd/git_protocol.go` | One new `daemon` field. | +| Edit | `internal/repofs/repofs.go` | Reject reserved organization names. | +| Edit | `cmd/objgitd/main.go` | Flags, the presign client, and the wiring. | +| Edit | `internal/metrics/metrics.go` | The `objgit_lfs_*` vectors and helpers. | +| Create | `internal/lfs/*_test.go`, `cmd/objgitd/lfs_test.go` | Tests. | +| Create | `docs/architecture/lfs.md`, `docs/usage/lfs.md`, `docs/plans/lfs.md` | Documentation. | +| Edit | `docs/architecture/README.md`, `AGENTS.md`, `README.md` | Index rows. | ## Design @@ -131,13 +135,13 @@ For each object the handler does one `HeadObject` on the per-repository marker. The calls run under an `errgroup` with `SetLimit`. A batch larger than `-lfs-max-batch` is 413. -| Operation | Marker present | Response for that object | -| --- | --- | --- | -| `download` | yes | `actions.download`, a presigned GET | -| `download` | no | `error`, code 404 | -| `upload` | yes, size matches | no `actions` key, so the client skips it | -| `upload` | yes, size differs | `error`, code 422 | -| `upload` | no | `actions.upload` and `actions.verify` | +| Operation | Marker present | Response for that object | +| ---------- | ----------------- | ---------------------------------------- | +| `download` | yes | `actions.download`, a presigned GET | +| `download` | no | `error`, code 404 | +| `upload` | yes, size matches | no `actions` key, so the client skips it | +| `upload` | yes, size differs | `error`, code 422 | +| `upload` | no | `actions.upload` and `actions.verify` | When the marker is absent but the global blob already exists, the upload action still goes out. The client re-uploads bytes the bucket already holds. The @@ -203,12 +207,12 @@ use the compare-and-swap the ref cache already uses write, then `If-Match: `, re-read and retry on refusal, under a bounded retry count. -| Route | Operation | Behavior | -| --- | --- | --- | -| `POST .../info/lfs/locks` | Write | 201 with the lock, or 409 with the holder. | -| `GET .../info/lfs/locks` | Read | List, with `path`, `id`, `cursor`, `limit`. | -| `POST .../info/lfs/locks/verify` | Read | Split into `ours` and `theirs`. | -| `POST .../info/lfs/locks/{id}/unlock` | Write | Delete one. `force` takes another owner's lock. | +| Route | Operation | Behavior | +| ------------------------------------- | --------- | ----------------------------------------------- | +| `POST .../info/lfs/locks` | Write | 201 with the lock, or 409 with the holder. | +| `GET .../info/lfs/locks` | Read | List, with `path`, `id`, `cursor`, `limit`. | +| `POST .../info/lfs/locks/verify` | Read | Split into `ours` and `theirs`. | +| `POST .../info/lfs/locks/{id}/unlock` | Write | Delete one. `force` takes another owner's lock. | Lock IDs are generated by the server as lowercase hex, and validated on read. A client-supplied `{id}` is never used to build a key. @@ -235,7 +239,11 @@ and `upload` to `auth.Write`, calls `d.authorize` with `Transport: "ssh"`, then writes one JSON object to the session, not to stderr, and exits 0: ```json -{"href":"https://git.example.com/acme/test.git/info/lfs","header":{},"expires_in":900} +{ + "href": "https://git.example.com/acme/test.git/info/lfs", + "header": {}, + "expires_in": 900 +} ``` CAUTION: When LFS is off, or `-external-url` is unset, the handler must write @@ -249,14 +257,14 @@ The `header` map is empty, because the HTTP side accepts anonymous requests. ### Flags -| Flag | Default | Purpose | -| --- | --- | --- | -| `-allow-lfs` | `false` | Register the LFS routes. Opt-in, like `-allow-hooks`. | -| `-allow-lfs-locks` | `true` | Register the four lock routes. | -| `-external-url` | `""` | The public base URL, for `git-lfs-authenticate`. | -| `-lfs-url-ttl` | `15m` | How long a presigned URL stays valid. | -| `-lfs-max-size` | `5368709120` | The largest object. One PUT caps at 5 GiB. | -| `-lfs-max-batch` | `500` | Objects per batch, over which the answer is 413. | +| Flag | Default | Purpose | +| ------------------ | ------------ | ----------------------------------------------------- | +| `-allow-lfs` | `false` | Register the LFS routes. Opt-in, like `-allow-hooks`. | +| `-allow-lfs-locks` | `true` | Register the four lock routes. | +| `-external-url` | `""` | The public base URL, for `git-lfs-authenticate`. | +| `-lfs-url-ttl` | `15m` | How long a presigned URL stays valid. | +| `-lfs-max-size` | `5368709120` | The largest object. One PUT caps at 5 GiB. | +| `-lfs-max-batch` | `500` | Objects per batch, over which the answer is 413. | Startup checks, in the style of `main.go:76-84`: `-external-url` must be absolute, carry a scheme, and end without a slash. It is required when @@ -337,8 +345,7 @@ regression in header hoisting, and a fake presigner would hide it forever. - Download of a present marker returns a download action. - Download of a missing marker returns a per-object 404, and the request is still 200. -- Download of an oid whose global blob exists but whose marker does not returns - 404. This is the read-oracle test. +- Download of an oid whose global blob exists but whose marker does not returns 404. This is the read-oracle test. - Upload of a present marker of the same size returns no actions. - Upload of a present marker of a different size returns 422. - Upload of a new oid returns an upload action and a verify action. @@ -476,7 +483,7 @@ A client now uploads to `//lfs/staging/`. Verify reads that key, not the shared one, and promotes it with `RenameObject`. Membership therefore requires having written the bytes, which Tigris checked against the oid as it accepted them. The same change fixes a second fault the plan had: verify used to -delete the *shared* blob on a size mismatch, so any writer could destroy an +delete the _shared_ blob on a size mismatch, so any writer could destroy an object other repositories depend on. It now deletes only the caller's own staged key. diff --git a/docs/plans/pack-compaction-implementation.md b/docs/plans/pack-compaction-implementation.md index 04afac8..d1ee5ac 100644 --- a/docs/plans/pack-compaction-implementation.md +++ b/docs/plans/pack-compaction-implementation.md @@ -1,10 +1,14 @@ +--- +title: "Pack Compaction Implementation Plan" +--- + # Pack Compaction Implementation Plan > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. **Goal:** Merge small pack containers into larger ones and delete the sources after a grace period, so pack count and duplicate storage stop growing without bound. -**Architecture:** A pass bin-packs *whole* source packs into an output container and copies every payload verbatim. Because the write-side containment rule already guarantees a delta's base sits in the same container as the delta, packing whole packs preserves containment for free — no decode, no delta resolution, no compressor. Deletion is deferred through a tombstone object with a deadline, so a request that built its pack index before the merge never loses the bytes under it. A background sweeper in the daemon runs one pass per repository. +**Architecture:** A pass bin-packs _whole_ source packs into an output container and copies every payload verbatim. Because the write-side containment rule already guarantees a delta's base sits in the same container as the delta, packing whole packs preserves containment for free — no decode, no delta resolution, no compressor. Deletion is deferred through a tombstone object with a deadline, so a request that built its pack index before the merge never loses the bytes under it. A background sweeper in the daemon runs one pass per repository. **Tech Stack:** Go 1.26, `github.com/aws/aws-sdk-go-v2/service/s3`, `github.com/go-git/go-git/v6`, `github.com/klauspost/compress/zstd`, `github.com/prometheus/client_golang`, `golang.org/x/sync/errgroup`. @@ -29,9 +33,11 @@ The fake returns only `Key` from `ListObjectsV2`. Compaction needs the object size and the last-modified time to make grace-period and bin-packing decisions, and the sweeper needs `CommonPrefixes` to enumerate repositories. **Files:** + - Modify: `internal/storage/tigris/client_test.go` **Interfaces:** + - Consumes: nothing. - Produces: `fakeObject.mtime time.Time`; `fakeS3.clock func() time.Time`; `(*fakeS3).setMtime(key string, t time.Time)`. `ListObjectsV2` now populates `types.Object.Size` and `types.Object.LastModified`, and honors `p.Delimiter` by returning `types.CommonPrefix` values. @@ -235,10 +241,12 @@ git commit -m "test(storage/tigris): fake reports object size, mtime, and common `listKeys` throws away the size and the last-modified time that `ListObjectsV2` already returned. Compaction needs both, and paying a second round trip for them would be waste. **Files:** + - Modify: `internal/storage/tigris/iter.go:17-47` - Test: `internal/storage/tigris/iter_test.go` **Interfaces:** + - Consumes: Task 1's fake. - Produces: `type objectEntry struct { key string; size int64; mtime time.Time }` and `func (s *Storer) listEntries(prefix string) ([]objectEntry, error)`. `listKeys` keeps its exact current signature and becomes a wrapper. @@ -372,10 +380,12 @@ git commit -m "feat(storage/tigris): listEntries reports key size and mtime" The tombstone object records which packs to delete and when. Its format copies `packed-refs` byte for byte, so a reader of one format can read the other by eye. **Files:** + - Create: `internal/storage/tigris/tombstones.go` - Test: `internal/storage/tigris/tombstones_test.go` **Interfaces:** + - Consumes: `compressBlock` and the zstd decoder from `compress.go`, as `packedrefs.go` uses them. - Produces: `const tombstonesKey = "pack-tombstones"`; `type tombstone struct { id string; deadline time.Time }`; `func encodeTombstones(ts []tombstone) []byte`; `func decodeTombstones(raw []byte) ([]tombstone, error)`; `var errBadTombstones error`. @@ -635,11 +645,13 @@ git commit -m "feat(storage/tigris): add the pack-tombstones binary format" `commitRefs` inlines its conditional `PutObject`. The tombstone object needs exactly the same compare-and-swap, so factor the call out rather than writing it twice. **Files:** + - Modify: `internal/storage/tigris/refcache.go:196-215` - Modify: `internal/storage/tigris/tombstones.go` - Test: `internal/storage/tigris/tombstones_test.go` **Interfaces:** + - Consumes: `tombstonesKey`, `encodeTombstones`, `decodeTombstones` from Task 3; `fetchSmallETag` from `refs.go:343`; `isPreconditionFailed` from `tigris.go:358`. - Produces: `var errCASConflict error`; `func (s *Storer) casPut(key string, body []byte, etag string) (string, error)`; `func (s *Storer) loadTombstones() ([]tombstone, string, error)`; `func (s *Storer) storeTombstones(ts []tombstone, etag string) (string, error)`. @@ -816,10 +828,12 @@ git commit -m "refactor(storage/tigris): extract casPut, add tombstone load and The pass and the cold index build both list `packs/` and parse every `.cue`. Share one function so the two can never disagree about what a pack is. **Files:** + - Modify: `internal/storage/tigris/packindex.go:553-595` - Test: `internal/storage/tigris/compact_test.go` (create) **Interfaces:** + - Consumes: `listEntries` from Task 2; `parseCue` from `packindex.go:207`. - Produces: `type packSurvey struct { id string; recs []cueRecord; size int64; mtime time.Time }` and `func (s *Storer) surveyPacks() (packs []packSurvey, orphanBins []objectEntry, err error)`. `ensurePacksBuilt` keeps its exact current signature and behavior. @@ -1032,10 +1046,12 @@ git commit -m "refactor(storage/tigris): share one pack survey between the index Which packs a pass merges is pure arithmetic over the survey. Keep it a pure function so every edge case is a table row and no test needs a bucket. **Files:** + - Create: `internal/storage/tigris/compact.go` - Test: `internal/storage/tigris/compact_test.go` **Interfaces:** + - Consumes: `packSurvey` from Task 5. - Produces: `func selectForMerge(packs []packSurvey, maxPacks int, capBytes int64) [][]packSurvey`. @@ -1205,11 +1221,13 @@ git commit -m "feat(storage/tigris): select small packs for compaction with a tr The merge writes payloads that are already encoded. `packSegment.add` re-encodes through a compressor, which is exactly the cost the design avoids. And `seal` hangs off `packWriter`, which the merge does not have. **Files:** + - Modify: `internal/storage/tigris/packwriter.go:412-445` (add a method beside `add`) - Modify: `internal/storage/tigris/packwriter.go:644-697` (`seal`) - Test: `internal/storage/tigris/pack_test.go` **Interfaces:** + - Consumes: `packSegment`, `cueRecord`. - Produces: `func (g *packSegment) appendVerbatim(rec cueRecord, r io.Reader) error`; `func sealSegment(s *Storer, seg *packSegment) (string, error)` returning the new pack id. `(*packWriter).seal` becomes a wrapper that discards the id. @@ -1365,10 +1383,12 @@ git commit -m "feat(storage/tigris): add verbatim segment append and free seal f This is the heart of the pass: choose the surviving record for each hash, then copy payloads verbatim into a new container. **Files:** + - Modify: `internal/storage/tigris/compact.go` - Test: `internal/storage/tigris/compact_test.go` **Interfaces:** + - Consumes: `selectForMerge` (Task 6), `appendVerbatim` and `sealSegment` (Task 7), `streamPack` from `packindex.go:735`, `newPackSegment` from `packwriter.go:402`. - Produces: `func chooseRecords(group []packSurvey) map[string][]cueRecord` and `func (s *Storer) mergeGroup(group []packSurvey) (string, int, error)` returning the new pack id and the number of duplicate records dropped. @@ -1634,10 +1654,12 @@ git commit -m "feat(storage/tigris): merge whole packs into one container with v ### Task 9: The reap step deletes expired tombstones **Files:** + - Modify: `internal/storage/tigris/compact.go` - Test: `internal/storage/tigris/compact_test.go` **Interfaces:** + - Consumes: `loadTombstones`, `storeTombstones`, `errCASConflict` (Task 4); `maxDeleteBatch` and the `DeleteObjects` idiom from `refcache.go:271`. - Produces: `func (s *Storer) reapTombstones(now time.Time) (deleted int, err error)`. @@ -1828,10 +1850,12 @@ git commit -m "feat(storage/tigris): reap expired pack tombstones" Tombstones are state that can disagree with the bucket. Two cases leak packs, and both close with data the survey already holds. **Files:** + - Modify: `internal/storage/tigris/compact.go` - Test: `internal/storage/tigris/compact_test.go` **Interfaces:** + - Consumes: `surveyPacks` (Task 5), `loadTombstones`/`storeTombstones` (Task 4), `deleteKeys` (Task 9). - Produces: `func subsumedPacks(packs []packSurvey) []string`; `func (s *Storer) recordMissingTombstones(packs []packSurvey, grace time.Duration) (int, error)`; `func (s *Storer) deleteOrphanBins(orphans []objectEntry, now time.Time, grace time.Duration) (int, error)`. @@ -2112,11 +2136,13 @@ git commit -m "feat(storage/tigris): back tombstones with subsumption and orphan ### Task 11: `Compact` orchestration and its metrics **Files:** + - Modify: `internal/storage/tigris/compact.go` - Modify: `internal/metrics/metrics.go` - Test: `internal/storage/tigris/compact_test.go` **Interfaces:** + - Consumes: every function from Tasks 5 through 10. - Produces: `type CompactOptions struct { MaxPacks int; Grace time.Duration; Now func() time.Time }`; `type CompactStats struct { PacksMerged, PacksWritten, PacksDeleted, ObjectsDeduped, OrphansDeleted int }`; `func (s *Storer) Compact(opt CompactOptions) (CompactStats, error)`; `func metrics.ObserveCompact(result string, dur time.Duration, st CompactCounts)`. @@ -2458,10 +2484,12 @@ git commit -m "feat(storage/tigris): add the Compact pass and its metrics" The sweeper needs the list of repositories in the bucket, and a way to avoid doing the same work on every daemon at once. **Files:** + - Modify: `internal/storage/tigris/compact.go` - Test: `internal/storage/tigris/compact_test.go` **Interfaces:** + - Consumes: Task 1's delimiter support in the fake. - Produces: `func (s *Storer) ListRepoPrefixes() ([]string, error)`; `func (s *Storer) TakeCompactLease(owner string, ttl time.Duration, now time.Time) (bool, error)`; `func (s *Storer) ReleaseCompactLease() error`. @@ -2674,10 +2702,12 @@ git commit -m "feat(storage/tigris): enumerate repositories and lease compaction ### Task 13: The sweeper **Files:** + - Create: `internal/compactor/compactor.go` - Test: `internal/compactor/compactor_test.go` **Interfaces:** + - Consumes: `(*tigris.Storer).ListRepoPrefixes`, `.Scoped`, `.Compact`, `.TakeCompactLease`, `.ReleaseCompactLease`, `tigris.CompactOptions`, `tigris.CompactStats`; `metrics.ObserveCompact` and `metrics.CompactCounts` from Task 11. - Produces: `type Sweeper struct { Base *tigris.Storer; Owner string; Interval time.Duration; Opts tigris.CompactOptions; Rand *rand.Rand }`; `func (sw *Sweeper) Run(ctx context.Context) error`; `func (sw *Sweeper) SweepOnce(ctx context.Context) error`. @@ -2885,9 +2915,11 @@ git commit -m "feat(compactor): add the background pack compaction sweeper" ### Task 14: Wire the sweeper into the daemon **Files:** + - Modify: `cmd/objgitd/main.go:36-50` (flags) and `cmd/objgitd/main.go:151` (errgroup) **Interfaces:** + - Consumes: `compactor.Sweeper` (Task 13), `tigris.CompactOptions` (Task 11). - Produces: the flags `-compact-interval`, `-compact-max-packs`, `-compact-grace`. @@ -2969,10 +3001,12 @@ It is the regression net for the containment invariant: if a merge ever stranded a delta from its base, the read below fails outright. **Files:** + - Modify: `internal/storage/tigris/compact_test.go` - Modify: `internal/storage/tigris/livebucket_test.go` **Interfaces:** + - Consumes: `buildPackFixture(t, numFiles int, withDeltaEdit bool) packFixture` and `writePack(t, s *Storer, fx packFixture)` from `pack_test.go:672,760`; `packFixture.hashes []plumbing.Hash`; `withMaxPackBytes` from `tigris.go:233`. - [ ] **Step 1: Write the test** @@ -3161,10 +3195,10 @@ git commit -m "test(storage/tigris): prove compaction preserves a real delta-bea --- - ### Task 16: Documentation **Files:** + - Modify: `docs/architecture/tigris-storer.md` (the "Remaining gap" section at the end) - Modify: `docs/reference/tigris-backend.md:476-503` (the build order) - Modify: `AGENTS.md` (the "Where the code lives" table) @@ -3252,7 +3286,7 @@ Change the first line of the section from `Items 1 to 4 and item 6 are complete. In `AGENTS.md`, add a row to the "Where the code lives" table after `internal/bundler`: ```markdown -| `internal/compactor` | The timer that runs pack compaction over every repository. | +| `internal/compactor` | The timer that runs pack compaction over every repository. | ``` - [ ] **Step 4: Check the prose** @@ -3272,23 +3306,23 @@ git commit -m "docs: describe pack compaction and close build order item 5" Checked against `docs/plans/pack-compaction.md`: -| Spec section | Task | -| --- | --- | -| Rewrite merges whole packs | 7, 8 | -| Prefer the whole copy over the delta copy | 8 (`chooseRecords`) | -| Selection by pack count, smallest first | 6 | -| Treadmill guard | 6 | -| No recompression | 7 (`appendVerbatim` carries `codec`) | -| Tombstone object and format | 3, 4 | -| Reap step, delete before tombstone rewrite | 9 | -| Backstop: subsumption, deadline from the newest pack | 10 | -| Backstop: orphan `.bin` | 10 | -| Sweeper, jitter, two-level enumeration | 12, 13 | -| Lease | 12, 13 | -| Concurrency with a live push | documented in 16; the overlap window is exercised in 15 | -| Flags | 14 | -| Metrics | 11 | -| Testing (every listed case) | 3, 6, 8, 9, 10, 11, 15 | +| Spec section | Task | +| ---------------------------------------------------- | ------------------------------------------------------- | +| Rewrite merges whole packs | 7, 8 | +| Prefer the whole copy over the delta copy | 8 (`chooseRecords`) | +| Selection by pack count, smallest first | 6 | +| Treadmill guard | 6 | +| No recompression | 7 (`appendVerbatim` carries `codec`) | +| Tombstone object and format | 3, 4 | +| Reap step, delete before tombstone rewrite | 9 | +| Backstop: subsumption, deadline from the newest pack | 10 | +| Backstop: orphan `.bin` | 10 | +| Sweeper, jitter, two-level enumeration | 12, 13 | +| Lease | 12, 13 | +| Concurrency with a live push | documented in 16; the overlap window is exercised in 15 | +| Flags | 14 | +| Metrics | 11 | +| Testing (every listed case) | 3, 6, 8, 9, 10, 11, 15 | Two items in the spec's testing list are folded rather than dropped. "A refused compare-and-swap on the tombstone retries" is covered by `TestTombstoneStoreDetectsConflict` in Task 4, plus the retry loops in Tasks 9 through 11. "A delta still reads after a merge" is covered in Task 15 by pushing a real git pack whose deltas git itself produced, then re-hashing every object after the merge and again after the reap. That is stronger than a synthetic delta, because a stranded base fails the read outright and a corrupt payload fails the hash. diff --git a/docs/plans/pack-compaction.md b/docs/plans/pack-compaction.md index 5ea5d59..f1a8a02 100644 --- a/docs/plans/pack-compaction.md +++ b/docs/plans/pack-compaction.md @@ -1,3 +1,7 @@ +--- +title: "Plan: pack compaction and tombstone reaping" +--- + # Plan: pack compaction and tombstone reaping ## Context @@ -224,11 +228,11 @@ The pass needs no lock against a push, for four reasons: ### Flags -| Flag | Default | Purpose | -| --------------------- | ------- | ------------------------------------------- | -| `-compact-interval` | `0` | Time between sweeps. `0` disables the sweeper. | -| `-compact-max-packs` | `16` | Pack count that triggers a pass. | -| `-compact-grace` | `1h` | Time between the merge and the delete. | +| Flag | Default | Purpose | +| -------------------- | ------- | ---------------------------------------------- | +| `-compact-interval` | `0` | Time between sweeps. `0` disables the sweeper. | +| `-compact-max-packs` | `16` | Pack count that triggers a pass. | +| `-compact-grace` | `1h` | Time between the merge and the delete. | Each flag is kebab-case with a `flagenv` fallback, such as `COMPACT_INTERVAL`. diff --git a/docs/plans/pack-prefetch.md b/docs/plans/pack-prefetch.md index e60f706..96b6c62 100644 --- a/docs/plans/pack-prefetch.md +++ b/docs/plans/pack-prefetch.md @@ -1,3 +1,7 @@ +--- +title: "Plan: non-blocking pack prefetch" +--- + # Plan: non-blocking pack prefetch ## Context @@ -114,12 +118,12 @@ and it gives that read a chance at a stream another request has warmed. ## Files -| Action | Path | Purpose | -| ------ | --------------------------------------- | -------------------------------------------------- | -| Edit | `internal/storage/tigris/packindex.go` | `packStream`, `packAccess`, `startPackFetch`, tiers | -| Edit | `internal/storage/tigris/packcache.go` | `cacheEntry.stream`, `partial`, progress plumbing | -| Edit | `internal/storage/tigris/tigris.go` | `Storer.fetchSem`, observer contract | -| Edit | `internal/storage/tigris/iter.go` | doc comment for the deleted threshold | +| Action | Path | Purpose | +| ------ | -------------------------------------- | --------------------------------------------------- | +| Edit | `internal/storage/tigris/packindex.go` | `packStream`, `packAccess`, `startPackFetch`, tiers | +| Edit | `internal/storage/tigris/packcache.go` | `cacheEntry.stream`, `partial`, progress plumbing | +| Edit | `internal/storage/tigris/tigris.go` | `Storer.fetchSem`, observer contract | +| Edit | `internal/storage/tigris/iter.go` | doc comment for the deleted threshold | ## Tests diff --git a/docs/plans/pack-streaming.md b/docs/plans/pack-streaming.md index 0eeaf14..aa9699f 100644 --- a/docs/plans/pack-streaming.md +++ b/docs/plans/pack-streaming.md @@ -1,3 +1,7 @@ +--- +title: "Stream pack downloads: serve reads while downloading (don't block clones)" +--- + # Stream pack downloads: serve reads while downloading (don't block clones) ## Context diff --git a/docs/plans/pack-temp-file-cache.md b/docs/plans/pack-temp-file-cache.md index f2c9092..80f94fd 100644 --- a/docs/plans/pack-temp-file-cache.md +++ b/docs/plans/pack-temp-file-cache.md @@ -1,3 +1,7 @@ +--- +title: "Local temp-file pack cache" +--- + # Local temp-file pack cache ## Problem diff --git a/docs/plans/per-repo-bucket.md b/docs/plans/per-repo-bucket.md index 3ffddc6..fc9e68b 100644 --- a/docs/plans/per-repo-bucket.md +++ b/docs/plans/per-repo-bucket.md @@ -1,3 +1,7 @@ +--- +title: "Per-keypair Tigris resolver: a bucket per repo" +--- + # Per-keypair Tigris resolver: a bucket per repo ## Context diff --git a/docs/plans/per-repo-fs-dispatch.md b/docs/plans/per-repo-fs-dispatch.md index 2a9fd0b..251f9b1 100644 --- a/docs/plans/per-repo-fs-dispatch.md +++ b/docs/plans/per-repo-fs-dispatch.md @@ -1,3 +1,7 @@ +--- +title: "Per-repo filesystem resolution + `{orgID}/{repoName}` paths (HTTP focus)" +--- + # Per-repo filesystem resolution + `{orgID}/{repoName}` paths (HTTP focus) ## Context diff --git a/docs/plans/prometheus-metrics.md b/docs/plans/prometheus-metrics.md index d972722..86d2840 100644 --- a/docs/plans/prometheus-metrics.md +++ b/docs/plans/prometheus-metrics.md @@ -1,3 +1,7 @@ +--- +title: "Plan: Prometheus metrics for objgitd" +--- + # Plan: Prometheus metrics for objgitd ## Context diff --git a/docs/plans/push-memory.md b/docs/plans/push-memory.md index a9c2796..b81a48e 100644 --- a/docs/plans/push-memory.md +++ b/docs/plans/push-memory.md @@ -1,3 +1,7 @@ +--- +title: "Plan: bounded memory for large pushes" +--- + # Plan: bounded memory for large pushes ## Goal diff --git a/docs/plans/s3fs-groupcache-listobjectsv2.md b/docs/plans/s3fs-groupcache-listobjectsv2.md index c5e2b7a..ca9af3d 100644 --- a/docs/plans/s3fs-groupcache-listobjectsv2.md +++ b/docs/plans/s3fs-groupcache-listobjectsv2.md @@ -1,3 +1,7 @@ +--- +title: "Plan: directory-listing cache for `internal/s3fs`" +--- + # Plan: directory-listing cache for `internal/s3fs` > **Status note.** This started as a groupcache-backed, fleet-shareable cache @@ -149,7 +153,7 @@ invalidation — and (c) carrying user metadata when the caller needs it (`hasMe `CacheConfig.DisableHeadPrefetch` turns seeding off (zero value = on). - **`headInfo(ctx, key, needMeta) (*s3.HeadObjectOutput, error)`** serves a foreground lookup. A warm hit costs no round-trip. `needMeta` reports whether the caller needs the - x-amz-meta-* user metadata (true iff unix-metadata is enabled): when true, a + x-amz-meta-\* user metadata (true iff unix-metadata is enabled): when true, a listing-seeded (`hasMeta=false`) entry is treated as a miss and filled via one real `HeadObject` (which stores `hasMeta=true`). `headSF` (a `singleflight.Group`) dedupes concurrent fills for the same key. @@ -168,12 +172,12 @@ walk folder-by-folder — `refs/` above all (`refs/`, `refs/heads/`, `refs/tags/ listing and every negative lookup beneath it is then synthesised in memory. - **A second `sync.Map`** (`subtrees`, keyed by root) holds `subtreeData{ Objects - []subtreeObject; Truncated bool }` — the flat key+size+mtime set under a root — in a +[]subtreeObject; Truncated bool }` — the flat key+size+mtime set under a root — in a `subtreeEntry{gen, expires}`, the same TTL/generation scheme as folder listings. - **`list(prefix)` routes**: if `recursiveRoot(prefix)` matches a configured root, serve from `subtree(root)` via `synthesizeListing(objects, prefix)` (remainder-after-prefix with a `/` ⇒ child dir, deduped; else child file). Otherwise the existing delimited - path. A complete subtree is authoritative for negative lookups (it has *all* keys under + path. A complete subtree is authoritative for negative lookups (it has _all_ keys under the root); only `!Truncated` subtrees are trusted. - **Bounded.** `listSubtree` stops once it exceeds `MaxSubtreeKeys` (default 50000) and reports `Truncated`; `list` then **falls back to the delimited per-folder listing**, so @@ -241,7 +245,7 @@ listing and every negative lookup beneath it is then synthesised in memory. ### `internal/metrics/metrics.go` - A Prometheus collector that reads `ListingCache.Stats()` (`{Hits, Misses, ListingItems, - SubtreeItems, HeadItems}`) and exports `objgit_s3_listing_cache_hits_total`, +SubtreeItems, HeadItems}`) and exports `objgit_s3_listing_cache_hits_total`, `_misses_total`, and `_items{kind=listing|subtree|head}`. No `repo` label. (Cache fills are already counted as `ListObjectsV2`/`HeadObject` via `observeS3`.) `main` registers it only when the cache is enabled. diff --git a/docs/plans/s3fs-lazy-read.md b/docs/plans/s3fs-lazy-read.md index 29353ea..cb928eb 100644 --- a/docs/plans/s3fs-lazy-read.md +++ b/docs/plans/s3fs-lazy-read.md @@ -1,3 +1,7 @@ +--- +title: "Lazy, range-based reads for s3fs" +--- + # Lazy, range-based reads for s3fs ## Context diff --git a/docs/plans/s3fs-reduce-listobjectsv2.md b/docs/plans/s3fs-reduce-listobjectsv2.md index 56a4146..e43ba35 100644 --- a/docs/plans/s3fs-reduce-listobjectsv2.md +++ b/docs/plans/s3fs-reduce-listobjectsv2.md @@ -1,3 +1,7 @@ +--- +title: "Reduce ListObjectsV2 volume in objgitd" +--- + # Reduce ListObjectsV2 volume in objgitd ## Context diff --git a/docs/plans/s3fs-unix-metadata.md b/docs/plans/s3fs-unix-metadata.md index 29f5fa6..d87f18c 100644 --- a/docs/plans/s3fs-unix-metadata.md +++ b/docs/plans/s3fs-unix-metadata.md @@ -1,3 +1,7 @@ +--- +title: "Plan: Optional Unix-permission metadata in s3fs" +--- + # Plan: Optional Unix-permission metadata in s3fs ## Context @@ -76,7 +80,7 @@ Thread `*unixMetaConfig` into `newS3WriteFile` and `newS3MultipartUploadFile` - If config is nil → unchanged (no `Metadata`). - Else set `PutObjectInput.Metadata = unixmeta.Encode(unixmeta.Attrs{UID, GID, - Mode: 0o666 &^ umask, Mtime: time.Now()})`. (Multipart: `Metadata` on +Mode: 0o666 &^ umask, Mtime: time.Now()})`. (Multipart: `Metadata` on `CreateMultipartUploadInput` at construction — `CompleteMultipartUpload` cannot attach user metadata.) @@ -87,7 +91,7 @@ Thread `*unixMetaConfig` into `newS3WriteFile` and `newS3MultipartUploadFile` consumers read the raw numeric ownership and resolve to names themselves. - Add `newFileInfoFromHead(name, head, cfg)` that, when `cfg != nil`, runs `unixmeta.Decode(head.Metadata, defaults)` with defaults `{UID: cfg.uid, - GID: cfg.gid, Mode: 0o666 &^ cfg.umask, Mtime: head.LastModified}` and builds +GID: cfg.gid, Mode: 0o666 &^ cfg.umask, Mtime: head.LastModified}` and builds a fully populated `simpleFileInfo`. When `cfg == nil`, keep the current `0666` path by delegating to `newFileInfo`. - Wire this into `Stat` (`internal/s3fs/basic.go`). `Lstat` already delegates diff --git a/docs/plans/ssh-protocol.md b/docs/plans/ssh-protocol.md index 8313b22..a5cc70a 100644 --- a/docs/plans/ssh-protocol.md +++ b/docs/plans/ssh-protocol.md @@ -1,3 +1,7 @@ +--- +title: "Plan: Git over SSH + a generic auth seam for objgitd" +--- + # Plan: Git over SSH + a generic auth seam for objgitd > **For agentic workers:** implement task-by-task in build order (bottom of this @@ -36,14 +40,14 @@ HTTP path's assumptions do **not** — see [Two protocol gotchas](#two-protocol- Each transport carries a different credential, and HTTP needs a challenge flow the others don't: -| Transport | Credential it presents | "please authenticate" mechanism | -| --------- | ----------------------------- | ------------------------------------- | -| git:// | none (anonymous) | n/a | +| Transport | Credential it presents | "please authenticate" mechanism | +| --------- | --------------------------------- | --------------------------------- | +| git:// | none (anonymous) | n/a | | SSH | public key (validated at connect) | handled in the SSH auth handshake | -| HTTP | `Authorization: Basic …` / none | `401` + `WWW-Authenticate` | +| HTTP | `Authorization: Basic …` / none | `401` + `WWW-Authenticate` | The transport must **not** validate credentials itself (a password is only -checkable against a user store the *policy* owns). Each transport's job shrinks +checkable against a user store the _policy_ owns). Each transport's job shrinks to three things: **collect** the raw credential, **map** the git service to a read/write `Operation`, and **enforce** the returned `Decision` in its own dialect. All real auth logic lives behind one interface, so the same mechanism @@ -69,17 +73,17 @@ lifecycle, `t.Helper()` helpers, table-driven subtests with `tt`). ## Files -| Action | Path | Purpose | -| ------ | ----------------------------- | ------------------------------------------------------ | +| Action | Path | Purpose | +| ------ | ----------------------------- | -------------------------------------------------------------------------- | | Create | `internal/auth/auth.go` | transport-neutral `Authorizer`, `Credential`, `Decision`, `AllowAnonymous` | -| Create | `internal/auth/auth_test.go` | unit tests for `AllowAnonymous` decisions | -| Create | `cmd/objgitd/ssh.go` | SSH server, host key, command dispatch | -| Create | `cmd/objgitd/ssh_test.go` | table-driven clone/push/deny/create/hook tests | -| Edit | `cmd/objgitd/git_protocol.go` | `authz` field on `daemon`; git:// goes through authz | -| Edit | `cmd/objgitd/http.go` | HTTP goes through authz (Basic-auth parse, 401/403) | -| Edit | `cmd/objgitd/main.go` | `-ssh-bind` flag, default authz, SSH errgroup listener | -| Edit | `go.mod` / `go.sum` | add `github.com/gliderlabs/ssh` (direct) | -| Edit | `CLAUDE.md` | document the auth seam + third transport | +| Create | `internal/auth/auth_test.go` | unit tests for `AllowAnonymous` decisions | +| Create | `cmd/objgitd/ssh.go` | SSH server, host key, command dispatch | +| Create | `cmd/objgitd/ssh_test.go` | table-driven clone/push/deny/create/hook tests | +| Edit | `cmd/objgitd/git_protocol.go` | `authz` field on `daemon`; git:// goes through authz | +| Edit | `cmd/objgitd/http.go` | HTTP goes through authz (Basic-auth parse, 401/403) | +| Edit | `cmd/objgitd/main.go` | `-ssh-bind` flag, default authz, SSH errgroup listener | +| Edit | `go.mod` / `go.sum` | add `github.com/gliderlabs/ssh` (direct) | +| Edit | `CLAUDE.md` | document the auth seam + third transport | The `daemon` struct stays the shared backend. The `allowPush bool` field is **removed** from `daemon` (the flag survives in `main.go` only, to build the @@ -313,6 +317,7 @@ e.g. `["git-upload-pack", "/foo/bar.git"]` — no manual parsing. | `git-receive-pack` | `transport.ReceivePackService` | Unknown → stderr error + `s.Exit(1)`. + 3. **Clean path:** `repoPath := strings.TrimPrefix(cmd[1], "/")` so `ssh://host/foo.git` and scp-style `host:foo.git` resolve identically. 4. **Authorize:** @@ -444,12 +449,12 @@ Receive-pack dispatches through **`d.receivePack`** (the wrapper), not `AllowAnonymous` decisions: -| AllowWrite | Operation | want | -| ---------- | --------- | ------ | -| false | Read | Allow | -| false | Write | Deny | -| true | Read | Allow | -| true | Write | Allow | +| AllowWrite | Operation | want | +| ---------- | --------- | ----- | +| false | Read | Allow | +| false | Write | Deny | +| true | Read | Allow | +| true | Write | Allow | One table, `tt`, asserting `(AllowAnonymous{tt.allowWrite}).Authorize(ctx, auth.Request{Operation: tt.op}) == tt.want`. @@ -469,6 +474,7 @@ listen on `127.0.0.1:0`, serve `newSSHServer`'s server in a goroutine, return th resolved addr. Skip the subtest if `ssh`/`ssh-keygen` are missing. Client identity via env on `runGit`: + ```go // ssh-keygen -t ed25519 -N "" -f /id_ed25519 env := append(os.Environ(), @@ -476,6 +482,7 @@ env := append(os.Environ(), "GIT_TERMINAL_PROMPT=0", ) ``` + URL: `ssh://git@127.0.0.1:/test.git`. Cases (one table, `tt`, subtests): @@ -511,19 +518,19 @@ Cases (one table, `tt`, subtests): ## Build order - [ ] **Task 1 — `internal/auth` package.** `auth.go` (interface, credentials, - `Decision`, `AllowAnonymous`) + `auth_test.go` table. `go test ./internal/auth/...`. + `Decision`, `AllowAnonymous`) + `auth_test.go` table. `go test ./internal/auth/...`. - [ ] **Task 2 — `authz` on daemon + git:// retrofit.** Add `authz auth.Authorizer` - to `daemon`, remove `allowPush` field, add `operationFor`, route `handle` - through authz. Update `main.go` to construct `AllowAnonymous`. Existing git:// - tests pass. + to `daemon`, remove `allowPush` field, add `operationFor`, route `handle` + through authz. Update `main.go` to construct `AllowAnonymous`. Existing git:// + tests pass. - [ ] **Task 3 — HTTP retrofit.** `credFromRequest`, `resolve(w, r, service, repoPath)` - with 401/403/Allow handling; update the two call sites. Existing HTTP tests pass; - add the anonymous-read parity case. + with 401/403/Allow handling; update the two call sites. Existing HTTP tests pass; + add the anonymous-read parity case. - [ ] **Task 4 — Host key.** `loadOrCreateHostKey` over `d.fs`; unit test on memfs - (first call creates+writes, second reads identical bytes). + (first call creates+writes, second reads identical bytes). - [ ] **Task 5 — SSH server + dispatch.** `newSSHServer`, `pubKeyContextKey`, - `PublicKeyHandler`, `handleSSH` with the git://-style wrapping and authz check. + `PublicKeyHandler`, `handleSSH` with the git://-style wrapping and authz check. - [ ] **Task 6 — main.go SSH wiring.** `-ssh-bind` flag, errgroup listener + - graceful shutdown, slog line. `go build -o objgitd ./cmd/objgitd`. + graceful shutdown, slog line. `go build -o objgitd ./cmd/objgitd`. - [ ] **Task 7 — SSH tests.** `ssh_test.go` table per the cases above; TDD each case. - [ ] **Task 8 — Docs.** Add the auth seam + SSH transport to `CLAUDE.md`. diff --git a/docs/plans/streaming-container-sink.md b/docs/plans/streaming-container-sink.md index 1fd66cb..1a20fe6 100644 --- a/docs/plans/streaming-container-sink.md +++ b/docs/plans/streaming-container-sink.md @@ -1,3 +1,7 @@ +--- +title: "Replace the scratch go-git storage with a container sink" +--- + # Replace the scratch go-git storage with a container sink **Status:** stage 1 of 4 implemented (`internal/storage/tigris/segmentstore.go`, @@ -46,11 +50,11 @@ pack allocates: By allocation site: -| Site | Share | Per push | -| --- | --- | --- | -| `bytes.growSlice` | 47.9% | 2.09 GB | -| `plumbing.(*MemoryObject).Write` | 31.3% | 1.36 GB | -| everything else | 20.8% | ~0.9 GB | +| Site | Share | Per push | +| -------------------------------- | ----- | -------- | +| `bytes.growSlice` | 47.9% | 2.09 GB | +| `plumbing.(*MemoryObject).Write` | 31.3% | 1.36 GB | +| everything else | 20.8% | ~0.9 GB | By objgit entry point, cumulative: @@ -92,7 +96,7 @@ if p.scanner.seeker == nil { and with low memory mode off, `scanner.go:526` retains `oh.content` for every object rather than re-inflating from disk on demand. Driving the parser -directly off the network reader would therefore *increase* memory, not reduce +directly off the network reader would therefore _increase_ memory, not reduce it. So this plan keeps a seekable temp `.pack` and removes the `filesystem.Storage` @@ -158,7 +162,7 @@ To turn offsets into hashes, register a `packfile.Observer` via `WithScannerObservers`. `parser.go:146` and `:150` call `OnInflatedObjectHeader(oh.Type, oh.Size, oh.Offset)` and `OnInflatedObjectContent(oh.Hash, oh.Offset, oh.Crc32, nil)` for every object, -where `oh.Type` and `oh.Size` are the *resolved* values and `oh.Offset` is the +where `oh.Type` and `oh.Size` are the _resolved_ values and `oh.Offset` is the pack offset. Joining on offset gives `hash -> base hash`, which is precisely what `plannedObject.base` holds today, obtained without a single delta application. diff --git a/docs/plans/tekton-kubernetes-commands.md b/docs/plans/tekton-kubernetes-commands.md index 78bf415..6f8ca3d 100644 --- a/docs/plans/tekton-kubernetes-commands.md +++ b/docs/plans/tekton-kubernetes-commands.md @@ -1,3 +1,7 @@ +--- +title: "Plan: Kubernetes and Tekton commands in the hook shell" +--- + # Plan: Kubernetes and Tekton commands in the hook shell ## Context diff --git a/docs/plans/tigris-bin-cue-packs.md b/docs/plans/tigris-bin-cue-packs.md index 85cc256..948bafb 100644 --- a/docs/plans/tigris-bin-cue-packs.md +++ b/docs/plans/tigris-bin-cue-packs.md @@ -1,3 +1,7 @@ +--- +title: "Plan: bin/cue pack containers for `internal/storage/tigris`" +--- + # Plan: bin/cue pack containers for `internal/storage/tigris` **Status:** Implemented. This document records the design and the reasons diff --git a/docs/plans/tigris-delta-reuse.md b/docs/plans/tigris-delta-reuse.md index 825548b..ea09ee8 100644 --- a/docs/plans/tigris-delta-reuse.md +++ b/docs/plans/tigris-delta-reuse.md @@ -1,3 +1,7 @@ +--- +title: "Keep the deltas that a push already computed" +--- + # Keep the deltas that a push already computed ## Context @@ -6,10 +10,10 @@ A clone of a large repository spent almost all of its CPU time in one place. A 30-second profile of `objgitd` while it served a clone of `tigris-os` gave these numbers: -| Frame | Flat | Cumulative | -| ------------------------------ | ----- | ---------- | -| `(*deltaIndex).findMatch` | 47.9% | 85.0% | -| `hashBlock` | 34.1% | 34.1% | +| Frame | Flat | Cumulative | +| ----------------------------------- | ----- | ---------- | +| `(*deltaIndex).findMatch` | 47.9% | 85.0% | +| `hashBlock` | 34.1% | 34.1% | | `diffDelta` and everything under it | — | 92.8% | No code from this repository was in the profile. Every frame belonged to @@ -108,11 +112,11 @@ measurement with the search off suggested. The reason is a limit of the method, not a fault in it. Counters in the writer gave these numbers for one push of 59,459 objects: -| Result | Objects | -| ---------------------------- | ------- | -| Delta kept | 26,514 | -| Client sent no delta | 32,349 | -| Base in another container | 596 | +| Result | Objects | +| ------------------------- | ------- | +| Delta kept | 26,514 | +| Client sent no delta | 32,349 | +| Base in another container | 596 | The writer keeps 97.8% of the deltas that the client sends. Only 596 objects lose their delta to a container boundary. diff --git a/docs/plans/tigris-gogit-storer.md b/docs/plans/tigris-gogit-storer.md index 071c847..6e4fb2f 100644 --- a/docs/plans/tigris-gogit-storer.md +++ b/docs/plans/tigris-gogit-storer.md @@ -1,3 +1,7 @@ +--- +title: "Plan: Tigris-backed go-git `storage.Storer`" +--- + # Plan: Tigris-backed go-git `storage.Storer` > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. @@ -121,27 +125,29 @@ Ref encoding mirrors dotgit loose-ref files. Key form is `refs/` + full ref name ## Files -| Action | Path | Purpose | -| ------ | --------------------------------- | -------------------------------------------------------------- | -| Create | `internal/storage/tigris/tigris.go` | doc, constants, sentinels, `Storer`, Options, `New`, `s3API`, pointer shims, temp stubs | -| Create | `internal/storage/tigris/client_test.go` | `fakeS3`, seed helpers, constructor tables | -| Create | `internal/storage/tigris/objects.go` (+test) | HEAD reads, GET decode | -| Create | `internal/storage/tigris/writer.go` (+test) | staging writer, `SetEncodedObject` | -| Create | `internal/storage/tigris/iter.go` (+test) | pagination + lazy filtered iteration | -| Create | `internal/storage/tigris/refs.go` (+test) | six ref methods, encode/decode, shallow | -| Create | `internal/storage/tigris/index.go`, `config.go` (+test) | index/config codecs, `Module` | -| Create | `internal/storage/tigris/livebucket_test.go` | env-gated real-bucket verification | -| Modify | `AGENTS.md` | architecture blurb | +| Action | Path | Purpose | +| ------ | ------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| Create | `internal/storage/tigris/tigris.go` | doc, constants, sentinels, `Storer`, Options, `New`, `s3API`, pointer shims, temp stubs | +| Create | `internal/storage/tigris/client_test.go` | `fakeS3`, seed helpers, constructor tables | +| Create | `internal/storage/tigris/objects.go` (+test) | HEAD reads, GET decode | +| Create | `internal/storage/tigris/writer.go` (+test) | staging writer, `SetEncodedObject` | +| Create | `internal/storage/tigris/iter.go` (+test) | pagination + lazy filtered iteration | +| Create | `internal/storage/tigris/refs.go` (+test) | six ref methods, encode/decode, shallow | +| Create | `internal/storage/tigris/index.go`, `config.go` (+test) | index/config codecs, `Module` | +| Create | `internal/storage/tigris/livebucket_test.go` | env-gated real-bucket verification | +| Modify | `AGENTS.md` | architecture blurb | --- ### Task 1: Package skeleton, S3 seam, constructor, fake client **Files:** + - Create: `internal/storage/tigris/tigris.go` - Create: `internal/storage/tigris/client_test.go` **Interfaces:** + - Consumes: nothing new. - Produces (every later task depends on these exact names): @@ -844,11 +850,13 @@ git commit -s -m "feat(storage/tigris): scaffold Tigris-backed go-git storer sea ### Task 2: HEAD-backed reads — `headInfo`, `HasEncodedObject`, `EncodedObjectSize` **Files:** + - Create: `internal/storage/tigris/objects.go` - Create: `internal/storage/tigris/objects_test.go` - Modify: `internal/storage/tigris/tigris.go` (delete replaced stubs) **Interfaces:** + - Consumes: Task 1 `s3API`, `keyOf`, `metaType`/`metaSize`, `sp`/`ip`… wait — the production-side shim set is `sp/i32/sv/bv`; the fake side adds `ip` (int64 ptr) locally in tests. Both spelled below. - Produces: @@ -1163,11 +1171,13 @@ git commit -s -m "feat(storage/tigris): HEAD-backed Has and Size lookups" ### Task 3: Full-body read — `loadObject` and `EncodedObject` **Files:** + - Modify: `internal/storage/tigris/objects.go` - Modify: `internal/storage/tigris/objects_test.go` - Modify: `internal/storage/tigris/tigris.go` (delete `EncodedObject` stub) **Interfaces:** + - Consumes: Task 2's `headInfo`/`objHead`. - Produces: @@ -1341,12 +1351,14 @@ git commit -s -m "feat(storage/tigris): body fetch and type-filtered EncodedObje ### Task 4: The staging writer — `RawObjectWriter` **Files:** + - Create: `internal/storage/tigris/writer.go` - Create: `internal/storage/tigris/writer_test.go` - Modify: `internal/storage/tigris/client_test.go` (extract `hashForBody`) - Modify: `internal/storage/tigris/tigris.go` (delete `RawObjectWriter` stub; real `NewEncodedObject` ships here too) **Interfaces:** + - Consumes: `keyOf`, metadata consts, `sp`/`bv`, `observe`, plumbing hasher trio. - Produces: @@ -1764,11 +1776,13 @@ git commit -s -m "feat(storage/tigris): hash-verifying staging writer for RawObj ### Task 5: `SetEncodedObject` and `AddAlternate` **Files:** + - Modify: `internal/storage/tigris/writer.go` - Modify: `internal/storage/tigris/writer_test.go` - Modify: `internal/storage/tigris/tigris.go` (delete `SetEncodedObject`, `AddAlternate` stubs) **Interfaces:** + - Consumes: Task 4's `stageWriter.Discard`. - Produces: `SetEncodedObject` implementation; test-only `lyingObject` wrapper (defined here, used nowhere else). @@ -1939,11 +1953,13 @@ git commit -s -m "feat(storage/tigris): SetEncodedObject with forged-hash refusa ### Task 6: Iteration — paginated `listKeys` and the lazy `objectIter` **Files:** + - Create: `internal/storage/tigris/iter.go` - Create: `internal/storage/tigris/iter_test.go` - Modify: `internal/storage/tigris/tigris.go` (delete `IterEncodedObjects` stub) **Interfaces:** + - Consumes: `objectPrefix`, `headInfo`, `loadObject`, `plumbing.FromHex`, `sp`/`sv`/`bv`. - Produces: @@ -2346,11 +2362,13 @@ git commit -s -m "feat(storage/tigris): lazy paginated EncodedObject iteration" ### Task 7: References (six methods) and shallow marks **Files:** + - Create: `internal/storage/tigris/refs.go` - Create: `internal/storage/tigris/refs_test.go` - Modify: `internal/storage/tigris/tigris.go` (delete nine ref/shallow stubs) **Interfaces:** + - Consumes: `refPrefix`, `symRefPrefix`, `refKey`, `fetchSmall`, `removeSimple` (both born here), `listKeys`, `sp`/`sv`, `isNotFound`, `storage.ErrReferenceHasChanged`, `errMalformedRef` (declared Task 1). - Produces: @@ -2934,12 +2952,14 @@ git commit -s -m "feat(storage/tigris): loose references, CAS updates, shallow m ### Task 8: Worktree index, repository config, `Module` — interface completion **Files:** + - Create: `internal/storage/tigris/index.go` - Create: `internal/storage/tigris/config.go` - Create: `internal/storage/tigris/index_test.go` - Modify: `internal/storage/tigris/tigris.go` (delete final five stubs; retire the block) **Interfaces:** + - Consumes: `indexKey`, `configKey`, `fetchSmall` (Task 7), `of`, `ErrModulesNotSupported`, codec APIs verified in the header. - Produces: @@ -3216,10 +3236,12 @@ git commit -s -m "feat(storage/tigris): complete storage.Storer with index, conf ### Task 9: Live-bucket verification (env-gated), docs blurb, final sweep **Files:** + - Create: `internal/storage/tigris/livebucket_test.go` - Modify: `AGENTS.md` **Interfaces:** + - Consumes: everything built so far; real credentials through the standard AWS chain (`AWS_PROFILE` / env keys) via `New(ctx, bucket)`'s default `storage-go` construction. - Produces: empirical confirmation of the spec's build-order item 2 ("record which error a real Tigris bucket returns for a missing key"), plus operator documentation. diff --git a/docs/plans/webhook-push-events.md b/docs/plans/webhook-push-events.md index a75357d..5a62b3d 100644 --- a/docs/plans/webhook-push-events.md +++ b/docs/plans/webhook-push-events.md @@ -1,3 +1,7 @@ +--- +title: "Plan: push webhook events" +--- + # Plan: push webhook events ## Contract diff --git a/docs/reference/how-tigris-fs-unix-metadata.md b/docs/reference/how-tigris-fs-unix-metadata.md index 3047859..ccec269 100644 --- a/docs/reference/how-tigris-fs-unix-metadata.md +++ b/docs/reference/how-tigris-fs-unix-metadata.md @@ -1,3 +1,7 @@ +--- +title: "How Unix metadata is stored on Tigris objects" +--- + # How Unix metadata is stored on Tigris objects A POSIX filesystem mapped onto S3-compatible object storage needs somewhere to keep @@ -13,14 +17,14 @@ with `aws-sdk-go-v2`. Every value is a string. Integers are decimal-encoded. -| Header | Meaning | Value format | -|---|---|---| -| `x-amz-meta-uid` | Owner user ID | decimal `uint32` | -| `x-amz-meta-gid` | Owner group ID | decimal `uint32` | -| `x-amz-meta-mode` | POSIX file mode (type + permission bits) | decimal `uint32` | -| `x-amz-meta-rdev` | Device number (block/character devices only) | decimal `uint32` | -| `x-amz-meta-mtime` | Modification time | Unix seconds, decimal | -| `x-amz-meta---symlink-target` | Target path of a symlink | URL-percent-escaped string | +| Header | Meaning | Value format | +| ----------------------------- | -------------------------------------------- | -------------------------- | +| `x-amz-meta-uid` | Owner user ID | decimal `uint32` | +| `x-amz-meta-gid` | Owner group ID | decimal `uint32` | +| `x-amz-meta-mode` | POSIX file mode (type + permission bits) | decimal `uint32` | +| `x-amz-meta-rdev` | Device number (block/character devices only) | decimal `uint32` | +| `x-amz-meta-mtime` | Modification time | Unix seconds, decimal | +| `x-amz-meta---symlink-target` | Target path of a symlink | URL-percent-escaped string | The header names are lowercase. Headers are written only when the value differs from the consumer's default; a missing header means "use the default" rather than @@ -37,15 +41,15 @@ The `mode` value is the standard POSIX `mode_t` integer: the low 9 bits encode `rwxrwxrwx`, the next 3 encode setuid/setgid/sticky, and the file type lives in the high bits. -| Type | Octal mask | -|---|---| -| Regular file | `0o100000` | -| Directory | `0o040000` | -| Symbolic link | `0o120000` | -| Block device | `0o060000` | +| Type | Octal mask | +| ---------------- | ---------- | +| Regular file | `0o100000` | +| Directory | `0o040000` | +| Symbolic link | `0o120000` | +| Block device | `0o060000` | | Character device | `0o020000` | -| FIFO | `0o010000` | -| Socket | `0o140000` | +| FIFO | `0o010000` | +| Socket | `0o140000` | So `33188` (decimal) = `0o100644` = regular file with `rw-r--r--`. diff --git a/docs/reference/tigris-backend.md b/docs/reference/tigris-backend.md index 8a414b3..b6cdb0b 100644 --- a/docs/reference/tigris-backend.md +++ b/docs/reference/tigris-backend.md @@ -1,3 +1,7 @@ +--- +title: "A Tigris backend for go-git object storage" +--- + # A Tigris backend for go-git object storage This document gives a design for `storer.EncodedObjectStorer` backed by @@ -134,7 +138,7 @@ only container that can be, and it always holds exactly one object. CAUTION: `enqueue` in `upload.go` clamps the size it gives the bundler to the `BufferedByteLimit` of that bundler. The bundler takes the size -from a semaphore whose capacity *is* that limit, and +from a semaphore whose capacity _is_ that limit, and `x/sync/semaphore` parks a size above the whole capacity until the context is done. Without the clamp, a container above the limit hangs its push. Only the lone-object container of the paragraph above can @@ -162,15 +166,15 @@ header byte at offset 5 says whether that block is one zstd frame. The block holds seven columns, not one struct for each record. N comes from the header, so every column boundary is a calculation: -| Column | Size | Field | -| ------ | --------------- | -------------------------------------------- | -| hashes | N × hash width | Object hash, raw bytes | -| types | N × 1 | Object type: 1 commit, 2 tree, 3 blob, 4 tag | -| codecs | N × 1 | Payload codec: 0 raw, 1 zstd | -| offset | N × 8 | Offset of the payload in the `.bin` | -| stored | N × 8 | Stored length: the byte span to fetch | -| raw | N × 8 | Raw length: the size of the git object | -| bases | N × hash width | Delta base hash, or all zero bytes | +| Column | Size | Field | +| ------ | -------------- | -------------------------------------------- | +| hashes | N × hash width | Object hash, raw bytes | +| types | N × 1 | Object type: 1 commit, 2 tree, 3 blob, 4 tag | +| codecs | N × 1 | Payload codec: 0 raw, 1 zstd | +| offset | N × 8 | Offset of the payload in the `.bin` | +| stored | N × 8 | Stored length: the byte span to fetch | +| raw | N × 8 | Raw length: the size of the git object | +| bases | N × hash width | Delta base hash, or all zero bytes | One record therefore costs `(2 × hash width) + 26` bytes before compression. diff --git a/docs/superpowers/plans/2026-08-28-packed-refs-cas.md b/docs/superpowers/plans/2026-08-28-packed-refs-cas.md index 3385300..0ddd17b 100644 --- a/docs/superpowers/plans/2026-08-28-packed-refs-cas.md +++ b/docs/superpowers/plans/2026-08-28-packed-refs-cas.md @@ -1,3 +1,7 @@ +--- +title: "Packed refs with compare-and-swap — Implementation Plan" +--- + # Packed refs with compare-and-swap — Implementation Plan > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. @@ -30,9 +34,11 @@ The spec's risk table makes this the gate on everything else. If Tigris does not return 412 for these two headers, the whole design is wrong and no other task is worth starting. **Files:** + - Modify: `internal/storage/tigris/livebucket_test.go` (append a new test function) **Interfaces:** + - Consumes: nothing. - Produces: nothing consumed by code. It produces the go-ahead for Tasks 2 through 8. @@ -185,10 +191,12 @@ Signed-off-by: Xe Iaso " Pure functions over byte slices. No S3, no Storer, so this task is testable on its own and needs none of the test-fake work in Task 3. **Files:** + - Create: `internal/storage/tigris/packedrefs.go` - Create: `internal/storage/tigris/packedrefs_test.go` **Interfaces:** + - Consumes: `encodeRefValue` and `decodeRefValue` from `refs.go:26` and `refs.go:33`, unchanged. `compressBlock` and `cueDecoder` from `compress.go:161` and `compress.go:107`. `codecRaw` and `codecZstd` from `compress.go:12`. - Produces: - `const packedRefsKey = "packed-refs"` @@ -640,10 +648,12 @@ Signed-off-by: Xe Iaso " The spec calls this the one piece of test infrastructure the change needs. It buys the whole compare-and-swap path with no bucket. **Files:** + - Modify: `internal/storage/tigris/client_test.go:31-35` (the `fakeObject` struct), `:212-220` (`put`), `:298-333` (`GetObject`), `:355-390` (`PutObject`) - Create: `internal/storage/tigris/conditional_test.go` **Interfaces:** + - Consumes: `isPreconditionFailed` from Task 1. - Produces: - `fakeObject` gains an `etag string` field. @@ -893,6 +903,7 @@ Signed-off-by: Xe Iaso " Reads become unconditional: every read merges `packed-refs` with the legacy loose layer, whether or not writes are enabled. That asymmetry is the whole rollback story. **Files:** + - Create: `internal/storage/tigris/refcache.go` - Create: `internal/storage/tigris/refcache_test.go` - Modify: `internal/storage/tigris/tigris.go:83-113` (`Storer` fields), `:203-231` (`New`), `:251-264` (`Scoped`) @@ -900,6 +911,7 @@ Reads become unconditional: every read merges `packed-refs` with the legacy loos - Modify: `internal/storage/tigris/refs_test.go:134-178` (`TestIterReferencesSortedAndComplete`) **Interfaces:** + - Consumes: `decodePackedRefs`, `packedRefsKey`, `errBadPackedRefs` from Task 2. `listLooseRefs` from `refs.go:103`, unchanged. - Produces: - `type refCache struct` with fields `mu`, `built`, `etag`, `packed`, `loose` @@ -1521,6 +1533,7 @@ Signed-off-by: Xe Iaso " ### Task 5: The compare-and-swap write path **Files:** + - Modify: `internal/storage/tigris/refcache.go` (append `commitRefs` and friends) - Modify: `internal/storage/tigris/tigris.go:54-69` (error vars) - Modify: `internal/storage/tigris/refs.go:48-81` (`SetReference`, `CheckAndSetReference`), `:137-142` (`RemoveReference`) @@ -1529,6 +1542,7 @@ Signed-off-by: Xe Iaso " - Create: `internal/storage/tigris/refcommit_test.go` **Interfaces:** + - Consumes: `encodePackedRefs` (Task 2), `isPreconditionFailed` (Task 1), `preconditionFailed` in tests (Task 3), `refCache` and `refView` (Task 4), `s.up.flush()` from `upload.go`. - Produces: - `const maxRefCASRetries = 8` @@ -2256,6 +2270,7 @@ Signed-off-by: Xe Iaso " ### Task 6: The legacy fold and its loose-key cleanup **Files:** + - Modify: `internal/storage/tigris/refcache.go` (replace the `dropFoldedLooseRefs` stub) - Modify: `internal/storage/tigris/tigris.go:74-80` (`s3API`) - Modify: `internal/storage/tigris/refs.go` (`PackRefs`) @@ -2264,6 +2279,7 @@ Signed-off-by: Xe Iaso " - Create: `internal/storage/tigris/reffold_test.go` **Interfaces:** + - Consumes: `commitRefs` (Task 5), `refCache` (Task 4). - Produces: - `s3API` gains `DeleteObjects`. @@ -2580,7 +2596,7 @@ func bp(v bool) *bool { return &v } and delete the copy at `client_test.go:464`. -**Note:** removing a ref that has *no* loose key still issues a DeleteObjects for its name. S3 treats deleting an absent key as success, so this is harmless, and checking first would cost a round trip to save one. +**Note:** removing a ref that has _no_ loose key still issues a DeleteObjects for its name. S3 treats deleting an absent key as success, so this is harmless, and checking first would cost a round trip to save one. - [ ] **Step 5: Give `PackRefs` its meaning** @@ -2674,11 +2690,13 @@ Signed-off-by: Xe Iaso " ### Task 7: The batch seam in receive-pack **Files:** + - Modify: `internal/storage/tigris/refs.go` (add `UpdateReferences`) - Modify: `cmd/objgitd/receivepack.go:287-324` (`updateReferences`) - Create: `cmd/objgitd/refbatch_test.go` **Interfaces:** + - Consumes: `commitRefs` (Task 5). - Produces: - `func (s *Storer) UpdateReferences(sets []*plumbing.Reference, removes []plumbing.ReferenceName) error` @@ -3263,12 +3281,14 @@ Signed-off-by: Xe Iaso " This task writes down what changed and prepares the flip. **It does not flip the default.** That is release 2's commit, made once the fleet runs a binary that can read packed refs. **Files:** + - Modify: `docs/architecture/tigris-storer.md` - Modify: `docs/architecture/transports.md` - Modify: `internal/storage/tigris/tigris.go:6-21` (the package doc's layout diagram) - Modify: `AGENTS.md` **Interfaces:** + - Consumes: everything above. - Produces: no code. diff --git a/docs/superpowers/plans/2026-09-23-erofs-snapshots.md b/docs/superpowers/plans/2026-09-23-erofs-snapshots.md index e3d338d..f475e9b 100644 --- a/docs/superpowers/plans/2026-09-23-erofs-snapshots.md +++ b/docs/superpowers/plans/2026-09-23-erofs-snapshots.md @@ -1,3 +1,7 @@ +--- +title: "erofs Snapshots Implementation Plan" +--- + # erofs Snapshots Implementation Plan > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. diff --git a/docs/superpowers/specs/2026-08-27-cue-compression-design.md b/docs/superpowers/specs/2026-08-27-cue-compression-design.md index e6da2fa..f504b70 100644 --- a/docs/superpowers/specs/2026-08-27-cue-compression-design.md +++ b/docs/superpowers/specs/2026-08-27-cue-compression-design.md @@ -1,3 +1,7 @@ +--- +title: "Compression for pack containers in `internal/storage/tigris`" +--- + # Compression for pack containers in `internal/storage/tigris` ## Context @@ -20,7 +24,7 @@ The `.cue` sheet is where the fix has to live. `cueRecord.length` currently does double duty — it is both the byte span in the `.bin` and the git object size — and the comment at `packindex.go:66` says so outright: "`== the git object's size, since .bin payloads are raw`". Splitting that one field into -*stored span* and *raw size* is the whole enabling change. +_stored span_ and _raw size_ is the whole enabling change. Intended outcome, measured against this repository: **2.40x on stored pack bytes** (3,297,625 → 1,376,080), recovering 58.3% of what containers hold @@ -51,14 +55,14 @@ existing v1 containers in live buckets stay readable forever. Every object in this repository (901 objects, 3.3 MB), one zstd level-3 frame each: -| band | n | raw | zstd | ratio | objects that shrink | -| --- | --- | --- | --- | --- | --- | -| <512B | 342 | 59,757 | 56,064 | **1.07x** | 143 / 342 | -| 512B–2K | 213 | 201,466 | 137,184 | 1.47x | 213 / 213 | -| 2K–8K | 228 | 1,165,860 | 459,811 | 2.54x | 228 / 228 | -| 8K–32K | 115 | 1,472,502 | 561,729 | 2.62x | 115 / 115 | -| 32K–64K | 1 | 55,453 | 16,709 | 3.32x | 1 / 1 | -| ≥64K | 2 | 342,587 | 76,608 | 4.47x | 2 / 2 | +| band | n | raw | zstd | ratio | objects that shrink | +| ------- | --- | --------- | ------- | --------- | ------------------- | +| <512B | 342 | 59,757 | 56,064 | **1.07x** | 143 / 342 | +| 512B–2K | 213 | 201,466 | 137,184 | 1.47x | 213 / 213 | +| 2K–8K | 228 | 1,165,860 | 459,811 | 2.54x | 228 / 228 | +| 8K–32K | 115 | 1,472,502 | 561,729 | 2.62x | 115 / 115 | +| 32K–64K | 1 | 55,453 | 16,709 | 3.32x | 1 / 1 | +| ≥64K | 2 | 342,587 | 76,608 | 4.47x | 2 / 2 | Cumulative bytes recovered by floor choice: **64 KiB → 8.1%**, 32 KiB → 9.2%, 8 KiB → 36.9%, **2 KiB → 58.3%**, 512 B → 60.2%, no floor → 60.4%. @@ -179,7 +183,7 @@ bytes against a declared object size would silently destroy it. **The 128 MiB container cap keeps working unchanged.** `packwriter.go:113` compares `seg.offset + obj.Size()` against the cap. `seg.offset` is now stored bytes while `obj.Size()` is raw — apples to oranges, except the -win-or-don't policy makes `obj.Size()` a valid *upper bound*, so the +win-or-don't policy makes `obj.Size()` a valid _upper bound_, so the expression stays correct and merely seals slightly early. Update the comment; leave the logic. @@ -209,12 +213,12 @@ every caller outside `packindex.go` are untouched. All four tiers in `packObject` (`packindex.go:468`) change one word — `e.length` → `e.stored` — and route through one new helper: -| Tier | Line | Change | -| --- | --- | --- | -| staged local `.bin` | `:485` | `SectionReader(f, e.offset, e.stored)` | -| already-downloaded bulk copy | `:494` | same | -| bulk-download trigger | `:500` | same | -| ranged GET | `:511` | `bytes=%d-%d` computed over `e.stored` | +| Tier | Line | Change | +| ---------------------------- | ------ | -------------------------------------- | +| staged local `.bin` | `:485` | `SectionReader(f, e.offset, e.stored)` | +| already-downloaded bulk copy | `:494` | same | +| bulk-download trigger | `:500` | same | +| ranged GET | `:511` | `bytes=%d-%d` computed over `e.stored` | New helper `decodePacked(h, hs, codec, r)`: when `codec == zstd`, read the stored bytes and `DecodeAll` before handing off to the existing `decodeBody`. @@ -242,7 +246,7 @@ but it means **rolling back a deploy loses access to every pack written in the window.** So gate writing, not reading. Reader support for both versions is -unconditional. v2 *writing* goes behind: +unconditional. v2 _writing_ goes behind: - `WithPackCompression(bool)` Option in `tigris.go`, alongside `WithPackCache` (`tigris.go:124`); field on `Storer`, copied by `Scoped` @@ -264,17 +268,17 @@ the existing vector + thin-helper shape (`ObserveS3`, `:99`). ## Files to modify -| Path | Change | -| --- | --- | -| `internal/storage/tigris/packindex.go` | cue v2 codec, `parseCue` version branch, `cueRecord`/`packEntry` fields, four read tiers, `decodePacked`, `binSize` | -| `internal/storage/tigris/packwriter.go` | sniff + two counters in `add`, rewind on misprediction, sha256 moves to `seal`, cap comment | -| `internal/storage/tigris/tigris.go` | `WithPackCompression`, `Storer` field, package doc layout comment (`:15`) | -| `internal/storage/tigris/objects.go` | `e.length` → `e.raw` at `:38` | -| `cmd/objgitd/main.go` | `-pack-compression` flag, wire the Option | -| `internal/metrics/metrics.go` | codec counter + ratio histogram | -| `internal/storage/tigris/pack_test.go` | see below | -| `docs/reference/tigris-backend.md` | cue layout table (`:151`), payload description (`:41`, `:50`, `:72`), retire build-order item 7 (`:371`) | -| `docs/architecture/tigris-storer.md` | "payload is raw" claims at `:117`, `:173` | +| Path | Change | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `internal/storage/tigris/packindex.go` | cue v2 codec, `parseCue` version branch, `cueRecord`/`packEntry` fields, four read tiers, `decodePacked`, `binSize` | +| `internal/storage/tigris/packwriter.go` | sniff + two counters in `add`, rewind on misprediction, sha256 moves to `seal`, cap comment | +| `internal/storage/tigris/tigris.go` | `WithPackCompression`, `Storer` field, package doc layout comment (`:15`) | +| `internal/storage/tigris/objects.go` | `e.length` → `e.raw` at `:38` | +| `cmd/objgitd/main.go` | `-pack-compression` flag, wire the Option | +| `internal/metrics/metrics.go` | codec counter + ratio histogram | +| `internal/storage/tigris/pack_test.go` | see below | +| `docs/reference/tigris-backend.md` | cue layout table (`:151`), payload description (`:41`, `:50`, `:72`), retire build-order item 7 (`:371`) | +| `docs/architecture/tigris-storer.md` | "payload is raw" claims at `:117`, `:173` | ## Tests — `internal/storage/tigris/pack_test.go` diff --git a/docs/superpowers/specs/2026-08-28-packed-refs-cas-design.md b/docs/superpowers/specs/2026-08-28-packed-refs-cas-design.md index 6ea422b..86adcd0 100644 --- a/docs/superpowers/specs/2026-08-28-packed-refs-cas-design.md +++ b/docs/superpowers/specs/2026-08-28-packed-refs-cas-design.md @@ -1,3 +1,7 @@ +--- +title: "Packed refs with compare-and-swap" +--- + # Packed refs with compare-and-swap Status: approved, not implemented. @@ -53,10 +57,10 @@ two steps are not atomic. The comment at `refs.go:22` records this hole. Two keys hold ref state. Both sit under the per-Storer prefix from `Scoped`. -| Key | Role | -| ------------ | -------------------------------------------------------- | -| `packed-refs`| Every ref for this repository. Its ETag is the CAS token. | -| `refs/`| A legacy loose ref. Read-only. Deleted by the fold. | +| Key | Role | +| ------------- | --------------------------------------------------------- | +| `packed-refs` | Every ref for this repository. Its ETag is the CAS token. | +| `refs/` | A legacy loose ref. Read-only. Deleted by the fold. | `packed-refs` sits at the root, beside `shallow`, `index` and `config`. The name comes from the git file with the same role. The key does not start with @@ -67,13 +71,13 @@ name comes from the git file with the same role. The key does not start with The object has a 16-byte plaintext header, and then a body. The header copies the shape that `.cue` uses in `packindex.go`: -| Offset | Bytes | Content | -| ------ | ----- | ---------------------------------------------- | -| 0 | 3 | Magic `OGR`. | -| 3 | 1 | Format version. This design writes version 1. | -| 4 | 1 | Body codec: `codecRaw` or `codecZstd`. | -| 5 | 4 | Ref count, big-endian `uint32`. | -| 9 | 7 | Reserved. Zero. | +| Offset | Bytes | Content | +| ------ | ----- | --------------------------------------------- | +| 0 | 3 | Magic `OGR`. | +| 3 | 1 | Format version. This design writes version 1. | +| 4 | 1 | Body codec: `codecRaw` or `codecZstd`. | +| 5 | 4 | Ref count, big-endian `uint32`. | +| 9 | 7 | Reserved. Zero. | The body is raw, or one zstd frame. It reuses the encoder in `compress.go`. @@ -282,10 +286,10 @@ The rollout copies the `WithPackCompression` pattern, for the reason its own comment gives at `tigris.go:150`. Ship the reader first, so the rollback window is empty. -| Release | `WithPackedRefs` | Reads | Writes | -| ------- | ---------------- | -------------------------------- | ------------ | -| 1 | off by default | `packed-refs`, then loose on top | loose keys | -| 2 | on by default | `packed-refs`, then loose on top | `packed-refs`| +| Release | `WithPackedRefs` | Reads | Writes | +| ------- | ---------------- | -------------------------------- | ------------- | +| 1 | off by default | `packed-refs`, then loose on top | loose keys | +| 2 | on by default | `packed-refs`, then loose on top | `packed-refs` | Release 1 changes nothing in production, because no `packed-refs` object exists yet. It exists so that release 2 can be rolled back. A rollback to release 1 @@ -302,21 +306,21 @@ and `IfNoneMatch`. A failed condition returns a `smithy.APIError` whose code is Tests are table-driven with `tt`, as `AGENTS.md` requires. -| Test | Asserts | -| ----------------------- | ----------------------------------------------------------- | -| Format round-trip | Empty set, one ref, symbolic refs, 100,000 refs. | -| Byte stability | One set of refs encodes to identical bytes every time. | -| Merge, loose only | A ref that exists only as a loose key is visible. | -| Merge, both layers | The loose ref wins. | -| Fold | The fold deletes every key it folded. | -| Fold, stopped part-way | PUT succeeds and `DeleteObjects` fails. Refs still resolve. The next write retries the delete. | -| CAS retry | One 412, then a rebuild, then success. | -| CAS exhaustion | Eight 412 responses give `ErrRefContention`. | -| Expectation failure | A concurrent writer moves the ref. The result is `storage.ErrReferenceHasChanged`, and not `ErrRefContention`. | -| Call count, write | A push of 500 refs is exactly one `PutObject` call. | -| Call count, read | An advertisement is exactly one `GetObject` and one `ListObjectsV2` call. | -| Live bucket | Real 412 behavior for `IfMatch` and `IfNoneMatch`. Goes in `livebucket_test.go`. | -| End to end | `git push` of many tags through `runGit` and `seedRepo`, with the same call counts. | +| Test | Asserts | +| ---------------------- | -------------------------------------------------------------------------------------------------------------- | +| Format round-trip | Empty set, one ref, symbolic refs, 100,000 refs. | +| Byte stability | One set of refs encodes to identical bytes every time. | +| Merge, loose only | A ref that exists only as a loose key is visible. | +| Merge, both layers | The loose ref wins. | +| Fold | The fold deletes every key it folded. | +| Fold, stopped part-way | PUT succeeds and `DeleteObjects` fails. Refs still resolve. The next write retries the delete. | +| CAS retry | One 412, then a rebuild, then success. | +| CAS exhaustion | Eight 412 responses give `ErrRefContention`. | +| Expectation failure | A concurrent writer moves the ref. The result is `storage.ErrReferenceHasChanged`, and not `ErrRefContention`. | +| Call count, write | A push of 500 refs is exactly one `PutObject` call. | +| Call count, read | An advertisement is exactly one `GetObject` and one `ListObjectsV2` call. | +| Live bucket | Real 412 behavior for `IfMatch` and `IfNoneMatch`. Goes in `livebucket_test.go`. | +| End to end | `git push` of many tags through `runGit` and `seedRepo`, with the same call counts. | The two call-count tests are the point of the whole change. They must fail before the work and pass after it. @@ -331,26 +335,26 @@ way this design degrades quietly. ## Files -| Path | Change | -| ------------------------------------------ | ------------------------------------------------------------------------- | -| `internal/storage/tigris/packedrefs.go` | New. Header and text body, encode and decode. | -| `internal/storage/tigris/refcache.go` | New. The cache, and the CAS commit loop. | -| `internal/storage/tigris/refs.go` | Rewire all seven ref methods onto the cache. Add `UpdateReferences`. | -| `internal/storage/tigris/tigris.go` | `packedRefsKey`, `WithPackedRefs`, the cache field, `Scoped`, `DeleteObjects` on `s3API`. | -| `cmd/objgitd/receivepack.go` | The `refUpdater` assertion in `updateReferences`. | -| `internal/metrics` | The CAS retry counter. | -| `docs/architecture/tigris-storer.md` | The new ref layout. | -| `docs/architecture/transports.md` | The atomicity change. | +| Path | Change | +| --------------------------------------- | ----------------------------------------------------------------------------------------- | +| `internal/storage/tigris/packedrefs.go` | New. Header and text body, encode and decode. | +| `internal/storage/tigris/refcache.go` | New. The cache, and the CAS commit loop. | +| `internal/storage/tigris/refs.go` | Rewire all seven ref methods onto the cache. Add `UpdateReferences`. | +| `internal/storage/tigris/tigris.go` | `packedRefsKey`, `WithPackedRefs`, the cache field, `Scoped`, `DeleteObjects` on `s3API`. | +| `cmd/objgitd/receivepack.go` | The `refUpdater` assertion in `updateReferences`. | +| `internal/metrics` | The CAS retry counter. | +| `docs/architecture/tigris-storer.md` | The new ref layout. | +| `docs/architecture/transports.md` | The atomicity change. | ## Risks -| Risk | Response | -| -------------------------------------------------------- | ------------------------------------------------------------------------ | -| Tigris rejects conditional writes on some bucket type. | The live-bucket test runs before any other work. It is the gate. | +| Risk | Response | +| ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | +| Tigris rejects conditional writes on some bucket type. | The live-bucket test runs before any other work. It is the gate. | | A Global or Dual-region bucket reads eventually. A stale read makes CAS unsafe. | Document that this design needs a Single-region or Multi-region bucket. See . | -| Two releases run at once and both write refs. | The merge rule makes the loose write win, so the older binary's push is honored. | -| Write amplification: every push rewrites up to 2 MB. | Accepted at the target scale. The next section names the fix if it hurts. | -| The 100,000-ref fold makes one push slow. | `PackRefs()` lets an operator run the fold first. | +| Two releases run at once and both write refs. | The merge rule makes the loose write win, so the older binary's push is honored. | +| Write amplification: every push rewrites up to 2 MB. | Accepted at the target scale. The next section names the fix if it hurts. | +| The 100,000-ref fold makes one push slow. | `PackRefs()` lets an operator run the fold first. | ## Rejected alternatives diff --git a/docs/superpowers/specs/2026-09-23-erofs-snapshots-design.md b/docs/superpowers/specs/2026-09-23-erofs-snapshots-design.md index cdaa5e9..985134e 100644 --- a/docs/superpowers/specs/2026-09-23-erofs-snapshots-design.md +++ b/docs/superpowers/specs/2026-09-23-erofs-snapshots-design.md @@ -1,3 +1,7 @@ +--- +title: "Design: erofs snapshots of pushed trees" +--- + # Design: erofs snapshots of pushed trees ## Goal diff --git a/docs/superpowers/specs/2026-09-23-erofs-streaming-builder-design.md b/docs/superpowers/specs/2026-09-23-erofs-streaming-builder-design.md index 0a3c65a..9b3f7da 100644 --- a/docs/superpowers/specs/2026-09-23-erofs-streaming-builder-design.md +++ b/docs/superpowers/specs/2026-09-23-erofs-streaming-builder-design.md @@ -1,3 +1,7 @@ +--- +title: "Design: a streaming file source for `erofs.Builder`" +--- + # Design: a streaming file source for `erofs.Builder` This spec is for work in the `github.com/Xe/erofs` repository, not in objgit. diff --git a/docs/usage/format-benchmark.md b/docs/usage/format-benchmark.md index 7a333c2..4a71548 100644 --- a/docs/usage/format-benchmark.md +++ b/docs/usage/format-benchmark.md @@ -1,3 +1,7 @@ +--- +title: "Measuring the storage format change" +--- + # Measuring the storage format change `cmd/formatbench` answers one question: what did the move from git's native @@ -11,10 +15,10 @@ comparable. ## The two sides -| Side | Ref | What it runs | -| --- | --- | --- | -| before | `v1.0.2` | `internal/s3fs` and go-git's dotgit layout. One bucket. Listing cache and pack cache on. | -| after | current checkout | `internal/storage/tigris`. `.bin`/`.cue` containers, zstd, cue v3. | +| Side | Ref | What it runs | +| ------ | ---------------- | ---------------------------------------------------------------------------------------- | +| before | `v1.0.2` | `internal/s3fs` and go-git's dotgit layout. One bucket. Listing cache and pack cache on. | +| after | current checkout | `internal/storage/tigris`. `.bin`/`.cue` containers, zstd, cue v3. | The format landed in commits `f4a1419` and `ebfe4e3` on 2026-08-27, released as v1.2.0. `v1.0.2` is the last release before it where repositories live in the @@ -148,20 +152,20 @@ spellings. ## Flags -| Flag | Default | What it does | -| --- | --- | --- | -| `-repos` | every entry | Comma-separated corpus entries to run. | -| `-builds` | `before,after` | Which columns to run. See below. | -| `-before-ref` | `v1.0.2` | The ref the "before" daemon is built from. | -| `-reps` | per entry | Override the repetition count for every repository. | -| `-out` | `var/bench/out` | Parent directory for run output. | -| `-org` | `formatbench` | Org segment every benchmark repository is created under. | -| `-bucket` | `BUCKET` | The bucket the daemon under test writes to. | -| `-push-timeout` | `45m` | Time limit for one push. | -| `-clone-timeout` | `30m` | Time limit for one clone. | -| `-verify` | `true` | Compare each clone against its mirror. | -| `-refresh-mirrors` | `false` | Fetch each mirror before the run. | -| `-cleanup` | none | Delete every prefix in a `repos.txt` and exit. | +| Flag | Default | What it does | +| ------------------ | --------------- | -------------------------------------------------------- | +| `-repos` | every entry | Comma-separated corpus entries to run. | +| `-builds` | `before,after` | Which columns to run. See below. | +| `-before-ref` | `v1.0.2` | The ref the "before" daemon is built from. | +| `-reps` | per entry | Override the repetition count for every repository. | +| `-out` | `var/bench/out` | Parent directory for run output. | +| `-org` | `formatbench` | Org segment every benchmark repository is created under. | +| `-bucket` | `BUCKET` | The bucket the daemon under test writes to. | +| `-push-timeout` | `45m` | Time limit for one push. | +| `-clone-timeout` | `30m` | Time limit for one clone. | +| `-verify` | `true` | Compare each clone against its mirror. | +| `-refresh-mirrors` | `false` | Fetch each mirror before the run. | +| `-cleanup` | none | Delete every prefix in a `repos.txt` and exit. | Every flag also reads from the environment through `flagenv`, which maps each one to UPPER_SNAKE. `-before-ref` becomes `BEFORE_REF`. @@ -170,10 +174,10 @@ one to UPPER_SNAKE. `-before-ref` becomes `BEFORE_REF`. `-builds` takes three spellings, comma-separated: -| Entry | What it builds | -| --- | --- | -| `before` | The ref in `-before-ref`, which defaults to `v1.0.2`. | -| `after` | The current checkout. | +| Entry | What it builds | +| -------------- | --------------------------------------------------------------------------------- | +| `before` | The ref in `-before-ref`, which defaults to `v1.0.2`. | +| `after` | The current checkout. | | `=` | That ref, in a column labelled ``. An empty ref means the current checkout. | The third form is how you measure a candidate fix. Put the old build, the @@ -195,8 +199,8 @@ Slashes in a ref name become dashes in that directory name. The default corpus is three repositories, chosen for three shapes rather than three names. Edit the `corpus` table in `cmd/formatbench/main.go` to change it. -| Name | Shape | -| --- | --- | -| objgit | Small. Mostly text. | -| x | Medium. Large object count. | +| Name | Shape | +| ----------- | ----------------------------------- | +| objgit | Small. Mostly text. | +| x | Medium. Large object count. | | tigris-blog | Blob-heavy. Images do not compress. | diff --git a/docs/usage/hooks.md b/docs/usage/hooks.md index 0ca19cc..32cc17b 100644 --- a/docs/usage/hooks.md +++ b/docs/usage/hooks.md @@ -1,3 +1,7 @@ +--- +title: "Push hooks" +--- + # Push hooks `objgitd` can run a script after a successful push. The script lives **inside the diff --git a/docs/usage/kubernetes-hooks.md b/docs/usage/kubernetes-hooks.md index 6f20683..c3f8996 100644 --- a/docs/usage/kubernetes-hooks.md +++ b/docs/usage/kubernetes-hooks.md @@ -1,3 +1,7 @@ +--- +title: "Kubernetes and Tekton hook commands" +--- + # Kubernetes and Tekton hook commands A push hook can render a Kustomize bundle and send it to the Kubernetes diff --git a/docs/usage/kubernetes.md b/docs/usage/kubernetes.md index 2bdb737..c38c0e3 100644 --- a/docs/usage/kubernetes.md +++ b/docs/usage/kubernetes.md @@ -1,3 +1,7 @@ +--- +title: "Kubernetes deployment" +--- + # Kubernetes deployment `manifest/` holds a kustomize deployment for `objgitd`. The pod stores all diff --git a/docs/usage/lfs.md b/docs/usage/lfs.md index 3ba81cd..96206c8 100644 --- a/docs/usage/lfs.md +++ b/docs/usage/lfs.md @@ -1,3 +1,7 @@ +--- +title: "Git LFS" +--- + # Git LFS `objgitd` serves the Git LFS API. Large files stay out of the git repository: @@ -27,14 +31,14 @@ refuses to start with `-allow-lfs` and an empty `-http-bind`. ## Flags -| Flag | Default | Purpose | -| --- | --- | --- | -| `-allow-lfs` | `false` | Serve the Git LFS API. | -| `-allow-lfs-locks` | `true` | Serve the file locking API. | -| `-external-url` | `""` | Public base URL, such as `https://git.example.com`. | -| `-lfs-url-ttl` | `15m` | How long a presigned transfer URL stays valid. | -| `-lfs-max-size` | `5368709120` | Largest object accepted, in bytes. | -| `-lfs-max-batch` | `500` | Most objects in one batch request. | +| Flag | Default | Purpose | +| ------------------ | ------------ | --------------------------------------------------- | +| `-allow-lfs` | `false` | Serve the Git LFS API. | +| `-allow-lfs-locks` | `true` | Serve the file locking API. | +| `-external-url` | `""` | Public base URL, such as `https://git.example.com`. | +| `-lfs-url-ttl` | `15m` | How long a presigned transfer URL stays valid. | +| `-lfs-max-size` | `5368709120` | Largest object accepted, in bytes. | +| `-lfs-max-batch` | `500` | Most objects in one batch request. | Each flag also reads from an environment variable, in upper snake case. `-allow-lfs` is `ALLOW_LFS`. diff --git a/docs/usage/memory-benchmark.md b/docs/usage/memory-benchmark.md index 6f9b992..10f7158 100644 --- a/docs/usage/memory-benchmark.md +++ b/docs/usage/memory-benchmark.md @@ -1,3 +1,7 @@ +--- +title: "Measuring push memory" +--- + # Measuring push memory `cmd/membench` answers one question: how much memory does `objgitd` need while diff --git a/docs/usage/webhooks.md b/docs/usage/webhooks.md index 02f3771..0eba8f4 100644 --- a/docs/usage/webhooks.md +++ b/docs/usage/webhooks.md @@ -1,3 +1,7 @@ +--- +title: "Push webhooks" +--- + # Push webhooks Store one `settings.json` object for each repository in the daemon's Tigris From bf95c48b074dedd560106d8112b17a100f80ff81 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Wed, 28 Oct 2026 11:33:06 -0400 Subject: [PATCH 2/2] feat(manifest): serve the docs with auxesia Add a kustomization in manifest/docs that serves docs/ at objgit.auxesia.automuse.art. An init container clones main with git-sync once. Auxesia then serves the docs folder from that clone. A ConfigMap volume does not work, because Kubernetes builds it from symbolic links and Auxesia rejects them. A git-sync sidecar does not work either: Auxesia resolves its root once at startup, so it keeps the old files after git-sync moves its link. To publish new docs, restart the deployment. Assisted-by: Claude Opus 5.5 via Claude Code Signed-off-by: Xe Iaso --- manifest/docs/deployment.yaml | 97 ++++++++++++++++++++++++++++++++ manifest/docs/ingress.yaml | 25 ++++++++ manifest/docs/kustomization.yaml | 13 +++++ manifest/docs/service.yaml | 14 +++++ 4 files changed, 149 insertions(+) create mode 100644 manifest/docs/deployment.yaml create mode 100644 manifest/docs/ingress.yaml create mode 100644 manifest/docs/kustomization.yaml create mode 100644 manifest/docs/service.yaml diff --git a/manifest/docs/deployment.yaml b/manifest/docs/deployment.yaml new file mode 100644 index 0000000..484f787 --- /dev/null +++ b/manifest/docs/deployment.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: objgit-docs + labels: + app.kubernetes.io/name: objgit-docs +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: objgit-docs + template: + metadata: + labels: + app.kubernetes.io/name: objgit-docs + spec: + runtimeClassName: kata + securityContext: + fsGroup: 65532 + # Auxesia refuses symbolic links, so a ConfigMap volume cannot hold the + # docs. Auxesia also resolves its root once at startup, so a git-sync + # sidecar that swaps the root link would leave it serving stale files. + # Clone once, before the server starts. + initContainers: + - name: clone + image: registry.k8s.io/git-sync/git-sync:v4.4.0 + args: + - --repo=https://github.com/tigrisdata/objgit + - --ref=main + - --depth=1 + - --root=/git + - --link=objgit + - --one-time + resources: + limits: + cpu: "500m" + memory: "256Mi" + requests: + cpu: "100m" + memory: "64Mi" + volumeMounts: + - name: git + mountPath: /git + - name: tmp + mountPath: /tmp + securityContext: + runAsUser: 65533 + runAsGroup: 65533 + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + containers: + - name: main + image: ghcr.io/tigrisdata/auxesia:latest + imagePullPolicy: "Always" + args: + - /git/objgit/docs + - --listen + - 0.0.0.0:8080 + ports: + - name: http + containerPort: 8080 + readinessProbe: + httpGet: + path: /llms.txt + port: http + resources: + limits: + cpu: "500m" + memory: "256Mi" + requests: + cpu: "250m" + memory: "128Mi" + volumeMounts: + - name: git + mountPath: /git + readOnly: true + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumes: + - name: git + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/manifest/docs/ingress.yaml b/manifest/docs/ingress.yaml new file mode 100644 index 0000000..894fe06 --- /dev/null +++ b/manifest/docs/ingress.yaml @@ -0,0 +1,25 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: objgit-docs + labels: + app.kubernetes.io/name: objgit-docs + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" +spec: + ingressClassName: nginx + tls: + - hosts: + - objgit.auxesia.automuse.art + secretName: objgit-docs-public-tls + rules: + - host: objgit.auxesia.automuse.art + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: objgit-docs + port: + name: http diff --git a/manifest/docs/kustomization.yaml b/manifest/docs/kustomization.yaml new file mode 100644 index 0000000..f75eb65 --- /dev/null +++ b/manifest/docs/kustomization.yaml @@ -0,0 +1,13 @@ +# Serves the docs/ folder of this repository with Auxesia at +# https://objgit.auxesia.automuse.art. +# +# kubectl apply -k manifest/docs/ +# +# An init container clones main from GitHub at pod start. To publish new +# docs, restart the deployment: +# +# kubectl rollout restart deployment/objgit-docs +resources: + - deployment.yaml + - service.yaml + - ingress.yaml diff --git a/manifest/docs/service.yaml b/manifest/docs/service.yaml new file mode 100644 index 0000000..83f3977 --- /dev/null +++ b/manifest/docs/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: objgit-docs + labels: + app.kubernetes.io/name: objgit-docs +spec: + selector: + app.kubernetes.io/name: objgit-docs + ports: + - port: 80 + targetPort: 8080 + name: http + type: ClusterIP