Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .github/workflows/configurator-production.yml
Original file line number Diff line number Diff line change
Expand Up @@ -188,8 +188,11 @@ jobs:
LZC_RELEASE_BINDINGS: ${{ secrets.LZC_RELEASE_BINDINGS }}
LZC_PROJECT_ID: ${{ vars.LZC_PROJECT_ID }}
- name: Prepare authenticated production runtime
run: node infra/ci/prepare-release.mjs application
run: |
test -n "$LZC_MAILOUT_BINDINGS"
node infra/ci/prepare-release.mjs application
env:
LZC_MAILOUT_BINDINGS: ${{ secrets.LZC_MAILOUT_BINDINGS }}
LZC_AUTH_ENABLED: 'true'
LZC_ASSISTANT_ENABLED: 'true'
LZC_STACKIT_DEVICE_ENABLED: 'true'
Expand All @@ -215,6 +218,7 @@ jobs:
run: node infra/ci/check-production-route.mjs
- name: Verify runtime service and isolated Runner connections
run: |
node infra/ci/verify-mailout-binding.mjs
cf run-task landing-zone-configurator --command './runtime/bin/node apps/api/dist/check-connections.js' --name "connections-$GITHUB_RUN_ID" -m 128M -k 4096M --wait
cf run-task landing-zone-configurator --command "./runtime/bin/node apps/api/dist/check-plan-runner.js $GITHUB_RUN_ID" --name "runner-$GITHUB_RUN_ID" -m 128M -k 4096M --wait
for attempt in 1 2 3; do
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
-- A reported native Apply failure has already published its output and any
-- errored State. Retire its compute process, never its retained platform State.
ALTER TABLE lzc.plan_runs ADD COLUMN runner_stop_confirmed_at timestamptz;
CREATE INDEX reported_runner_retirement_queue ON lzc.plan_runs(finished_at,id)
WHERE operation='apply' AND status='recovery_required' AND error_code='apply_failed'
AND NOT cleaned AND runner_stop_confirmed_at IS NULL;

CREATE FUNCTION lzc_auth.reported_s3_apply_runners(p_run uuid DEFAULT NULL)
RETURNS TABLE(id uuid,runner_app_id uuid,runner_droplet_id uuid)
LANGUAGE sql SECURITY DEFINER SET search_path=pg_catalog AS $$
SELECT r.id,r.runner_app_id,r.runner_droplet_id
FROM lzc.plan_runs r JOIN lzc.platform_states s ON s.state_key=r.state_key
WHERE (p_run IS NULL OR r.id=p_run)
AND r.operation='apply' AND r.status='recovery_required'
AND r.error_code='apply_failed' AND r.input_claimed
AND r.finished_at<now()-interval '30 seconds'
AND r.runner_app_id IS NOT NULL AND r.runner_droplet_id IS NOT NULL
AND NOT r.cleaned AND r.runner_stop_confirmed_at IS NULL
AND s.backend_id IS NOT NULL AND s.ciphertext IS NULL AND s.lock_run_id IS NULL
AND (s.pending_backend_id IS NULL OR s.pending_backend_id=s.backend_id)
AND NOT EXISTS(SELECT 1 FROM lzc.state_recoveries recovery WHERE recovery.run_id=r.id)
AND NOT EXISTS(SELECT 1 FROM lzc.plan_runs other WHERE other.id<>r.id
AND other.runner_app_id=r.runner_app_id AND other.status IN ('starting','initializing','validating','planning','applying'))
ORDER BY r.finished_at,r.id LIMIT 20;
$$;
REVOKE ALL ON FUNCTION lzc_auth.reported_s3_apply_runners(uuid) FROM PUBLIC;
GRANT EXECUTE ON FUNCTION lzc_auth.reported_s3_apply_runners(uuid) TO configurator_app;

-- This timestamp only avoids repeated maintenance work. Checkpoint/reconcile
-- still independently inspect CF process/task termination on every request.
CREATE FUNCTION lzc_auth.record_reported_s3_runner_stopped(p_run uuid,p_app uuid,p_source uuid)
RETURNS void LANGUAGE sql SECURITY DEFINER SET search_path=pg_catalog AS $$
UPDATE lzc.plan_runs r SET runner_stop_confirmed_at=clock_timestamp()
WHERE r.id=p_run AND r.runner_app_id=p_app AND r.runner_droplet_id=p_source
AND EXISTS(SELECT 1 FROM lzc_auth.reported_s3_apply_runners(p_run) candidate WHERE candidate.id=r.id);
$$;
REVOKE ALL ON FUNCTION lzc_auth.record_reported_s3_runner_stopped(uuid,uuid,uuid) FROM PUBLIC;
GRANT EXECUTE ON FUNCTION lzc_auth.record_reported_s3_runner_stopped(uuid,uuid,uuid) TO configurator_app;
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ export interface PlanRunner {
sourceDropletId: string,
): Promise<boolean>;
removeRecovery?(id: string, appId: string | null): Promise<void>;
stopReportedFailure?(
id: string,
appId: string,
sourceDropletId: string,
): Promise<void>;
isStopped?(
id: string,
appId: string,
Expand Down Expand Up @@ -428,6 +433,17 @@ export class CloudFoundryPlanRunner implements PlanRunner {
async isStopped(id: string, appId: string, sourceDropletId: string) {
return this.inspectStopped(id, appId, sourceDropletId, false);
}
async stopReportedFailure(
id: string,
appId: string,
sourceDropletId: string,
) {
// Verify exact app, space and immutable source before any CF mutation.
// Only the broker's reported-failure retirement path may call this method.
if (await this.inspectStopped(id, appId, sourceDropletId, false)) return;
const call = await this.authorized();
await call(`apps/${appId}/actions/stop`, "POST");
}
async isRecoveryStopped(id: string, appId: string, sourceDropletId: string) {
return this.inspectStopped(id, appId, sourceDropletId, true);
}
Expand Down
31 changes: 31 additions & 0 deletions landing-zone-configurator/app/apps/api/src/plans/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1508,12 +1508,43 @@ export class Plans {
});
}
async maintain() {
let retirementError: unknown;
if (this.runner.stopReportedFailure && this.runner.isStopped) {
const failed = (
await this.pool.query(
"SELECT * FROM lzc_auth.reported_s3_apply_runners()",
)
).rows;
for (const run of failed) {
try {
await this.runner.stopReportedFailure(
run.id,
run.runner_app_id,
run.runner_droplet_id,
);
if (
await this.runner.isStopped(
run.id,
run.runner_app_id,
run.runner_droplet_id,
)
)
await this.pool.query(
"SELECT lzc_auth.record_reported_s3_runner_stopped($1,$2,$3)",
[run.id, run.runner_app_id, run.runner_droplet_id],
);
} catch (error) {
retirementError ??= error;
}
}
}
const rows = (
await this.pool.query("SELECT * FROM lzc_auth.expire_plan_runs()")
).rows;
for (const row of rows) {
await this.runner.remove(row.id, row.runner_app_id);
await this.pool.query("SELECT lzc_auth.mark_plan_cleaned($1)", [row.id]);
}
if (retirementError) throw retirementError;
}
}
151 changes: 151 additions & 0 deletions landing-zone-configurator/app/tests/platform-plan.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2093,6 +2093,157 @@ describe.skipIf(!enabled)(
return { ...test, run, saved, descriptor, stopped };
}

it("automatically stops a reported failed S3 runner without changing State or clearing the recovery hold", async () => {
const test = await failedRemoteApply();
test.stopped.mockResolvedValue(false);
await expect(
test.plans.checkpoint(test.owner, test.run.id),
).rejects.toMatchObject({ code: "runner_not_stopped" });
await migration!.query(
"UPDATE lzc.plan_runs SET finished_at=now()-interval '1 minute' WHERE id=$1",
[test.run.id],
);
const key = stableStateKey(test.owner, test.manifest);
// Cover the normal Configurator bootstrap migration marker as well.
await migration!.query(
"UPDATE lzc.platform_states SET pending_backend_id=backend_id WHERE state_key=$1",
[key],
);
const before = (
await migration!.query(
"SELECT * FROM lzc.platform_states WHERE state_key=$1",
[key],
)
).rows[0];
const remoteBefore = [...test.remote];
const stop = vi.fn(async (_id: string, _app: string, _source: string) => {
test.stopped.mockResolvedValue(true);
});
Object.assign(test.runner, { stopReportedFailure: stop });
await test.plans.maintain();
expect(stop).toHaveBeenCalledWith(
test.run.id,
expect.any(String),
expect.any(String),
);
const run = (
await migration!.query(
"SELECT status,cleaned,runner_stop_confirmed_at FROM lzc.plan_runs WHERE id=$1",
[test.run.id],
)
).rows[0];
expect(run.status).toBe("recovery_required");
expect(run.cleaned).toBe(false);
expect(run.runner_stop_confirmed_at).not.toBeNull();
expect(
(
await migration!.query(
"SELECT * FROM lzc.platform_states WHERE state_key=$1",
[key],
)
).rows[0],
).toEqual(before);
expect([...test.remote]).toEqual(remoteBefore);
expect(
(await test.plans.checkpoint(test.owner, test.run.id)).canResume,
).toBe(true);
// A prior maintenance receipt never substitutes for a fresh process check.
test.stopped.mockResolvedValue(false);
await expect(
test.plans.checkpoint(test.owner, test.run.id),
).rejects.toMatchObject({ code: "runner_not_stopped" });
stop.mockClear();
await test.plans.maintain();
expect(stop.mock.calls.filter((c) => c[0] === test.run.id)).toHaveLength(
0,
);
});

it.each([
"fresh",
"timed_out",
"state_failed",
"runner_unavailable",
"pending-target",
"backup",
"recovery",
"stop-unconfirmed",
"stop-error",
])(
"keeps S3 failure recovery blocked for retirement case %s",
async (reason) => {
const test = await failedRemoteApply();
const key = stableStateKey(test.owner, test.manifest);
await migration!.query(
"UPDATE lzc.plan_runs SET finished_at=now()-interval '1 minute' WHERE id=$1",
[test.run.id],
);
if (reason === "fresh")
await migration!.query(
"UPDATE lzc.plan_runs SET finished_at=now() WHERE id=$1",
[test.run.id],
);
if (
["timed_out", "state_failed", "runner_unavailable"].includes(reason)
)
await migration!.query(
"UPDATE lzc.plan_runs SET error_code=$2 WHERE id=$1",
[test.run.id, reason],
);
if (reason === "pending-target") {
const other = await test.backends.register(test.owner, {
descriptor: { ...test.descriptor, key: "pending.tfstate" },
credentials: {
accessKeyId: "fixture-only",
secretAccessKey: "fixture-only",
},
});
await migration!.query(
"UPDATE lzc.platform_states SET pending_backend_id=$2 WHERE state_key=$1",
[key, other.id],
);
}
if (reason === "backup")
await migration!.query(
"UPDATE lzc.platform_states SET backend_id=NULL,ciphertext=$2 WHERE state_key=$1",
[key, Buffer.from("retained-bootstrap-backup")],
);
if (reason === "recovery")
await migration!.query(
"INSERT INTO lzc.state_recoveries(run_id,tenant_id,state_key,sha256,ciphertext) SELECT $1,tenant_id,state_key,$2,$3 FROM lzc.platform_states WHERE state_key=$4",
[
test.run.id,
"a".repeat(64),
Buffer.from("separate-recovery"),
key,
],
);
test.stopped.mockResolvedValue(false);
const stop = vi.fn(
async (_id: string, _app: string, _source: string) => {
if (reason === "stop-error") throw new Error("stop-unavailable");
},
);
Object.assign(test.runner, { stopReportedFailure: stop });
await test.plans.maintain().catch((error) => {
expect(reason).toBe("stop-error");
expect(error.message).toBe("stop-unavailable");
});
if (!["stop-unconfirmed", "stop-error"].includes(reason))
expect(
stop.mock.calls.filter((c) => c[0] === test.run.id),
).toHaveLength(0);
const run = (
await migration!.query(
"SELECT status,runner_stop_confirmed_at FROM lzc.plan_runs WHERE id=$1",
[test.run.id],
)
).rows[0];
expect(run.status).toBe("recovery_required");
expect(run.runner_stop_confirmed_at).toBeNull();
},
);

it("reconciles a reported S3 Apply failure without changing remote state or replaying the approved plan", async () => {
const test = await failedRemoteApply();
const key = stableStateKey(test.owner, test.manifest);
Expand Down
28 changes: 28 additions & 0 deletions landing-zone-configurator/app/tests/runner-stopped.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -180,3 +180,31 @@ it("checks the exact recovery task without accepting normal or warm runners", as
);
await expect(f.run()).rejects.toThrow("Runner identity mismatch");
});

it("stops an exact failed runner only after source verification, then still requires independent process proof", async () => {
const f = fixture();
(f.data[`apps/${f.app}`] as { state: string }).state = "STARTED";
f.data[`apps/${f.app}/actions/stop`] = {};
await f.runner.stopReportedFailure(f.id, f.app, f.source);
expect(
f.calls
.filter((c) => c.method === "POST" && c.url.includes("/v3/"))
.map((c) => c.url),
).toEqual([expect.stringContaining(`/v3/apps/${f.app}/actions/stop`)]);
await expect(f.run()).resolves.toBe(false);
(f.data[`apps/${f.app}`] as { state: string }).state = "STOPPED";
await expect(f.run()).resolves.toBe(true);
});
it("never stops a runner with a different source checksum", async () => {
const f = fixture();
(f.data[`apps/${f.app}`] as { state: string }).state = "STARTED";
(
f.data[`apps/${f.app}/droplets/current`] as { checksum: { value: string } }
).checksum = { value: "b".repeat(64) };
await expect(
f.runner.stopReportedFailure(f.id, f.app, f.source),
).rejects.toThrow();
expect(
f.calls.some((c) => c.method === "POST" && c.url.includes("/v3/")),
).toBe(false);
});
8 changes: 8 additions & 0 deletions landing-zone-configurator/docs/plan-execution.de.md
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,14 @@ Destroy-Aktion. Die Migration muss vor Nutzung der neuen API aktiviert sein.

Eine abgeschlossene Bootstrap-Migration behält `pending_backend_id` als Nachweis. Entspricht diese Referenz dem aktiven Backend und wurde der Bootstrap-Ciphertext entfernt, ist die Migration abgeschlossen. Ein anderes vorgemerktes Backend oder ein noch vorhandener Bootstrap-State bleibt gesperrt.

Nach einem gemeldeten S3-Fehler `apply_failed` stoppt die Hintergrundwartung nach
einer kurzen Abschlussfrist genau den geprüften Runner. Die App bleibt zur Prüfung
gestoppt; die normale Bereinigung folgt dem Abgleich. Dies gibt den Lauf nicht frei
und verändert keinen State. Timeouts, unklare Fehler, separate Recovery-States und
offene Migrationen sind ausgeschlossen. Ein Wartungszeitpunkt verhindert nur
wiederholte Stop-Anfragen; Checkpoint und Bestätigung prüfen den aktuellen CF-App-,
Task- und Prozesszustand weiterhin unabhängig.

Bei einem bestehenden S3-Backend prüft die Recovery den registrierten Remote-State
anstelle eines Bootstrap-Checkpoints. Nur der Eigentümer eines mit `apply_failed`
beendeten Laufs darf die erneute Planung bestätigen. Genau dieser Runner muss
Expand Down
7 changes: 7 additions & 0 deletions landing-zone-configurator/docs/plan-execution.md
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,13 @@ The user can then save the corrected configuration, choose a new preparation and

A completed bootstrap migration retains `pending_backend_id` as evidence. When it matches the active backend and bootstrap ciphertext has been removed, it is a completed migration, not an open migration. A different pending backend or retained bootstrap State remains blocked.

After a reported S3 `apply_failed`, background maintenance stops the exact verified
Runner after a short completion grace period. The app remains stopped for inspection;
normal cleanup follows reconciliation. This neither reconciles the run nor changes
State. Timeouts, ambiguous failures, separate recovery State and open migrations are
excluded. A maintenance timestamp only avoids repeated stop requests; checkpoint and
confirmation always independently verify current CF app, task and process status.

For an existing S3 backend, recovery inspects the registered remote State rather
than a bootstrap checkpoint. Only the owner of a reported `apply_failed` run may
confirm replanning. The exact Runner must be stopped or already cleaned; local
Expand Down
8 changes: 8 additions & 0 deletions landing-zone-configurator/infra/ci/release-notes.mjs
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
const tag=process.env.LZC_RELEASE_TAG;
if(!/^v(?:[2-9]|[1-9][0-9]+)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$/.test(tag??""))throw new Error("Expected a stable product release tag");
const notes={
"v2.2.2": {
introduction:"Production email delivery and reliable recovery after reported S3 Apply failures.",
changes:[
"Provision a dedicated production MailOut sender and bind SMTP only to the application backend, with DNS validation and deployment verification.",
"Stop source-verified runners after explicitly reported S3 Apply failures so checkpoint inspection can proceed; retain State and require independent process checks and owner confirmation.",
"Preserve recovery holds for timeouts, ambiguous interruptions, pending migrations and separate recovery States.",
],
},
"v2.2.1": {
introduction:"Context-aware service consultation in the Landing Zone Configurator.",
changes:[
Expand Down
Loading
Loading