diff --git a/.github/workflows/configurator-production.yml b/.github/workflows/configurator-production.yml index 64010a17..7cb743ce 100644 --- a/.github/workflows/configurator-production.yml +++ b/.github/workflows/configurator-production.yml @@ -188,8 +188,11 @@ jobs: LZC_RELEASE_BINDINGS: ${{ secrets.LZC_RELEASE_BINDINGS }} LZC_PROJECT_ID: ${{ vars.LZC_PROJECT_ID }} - name: Prepare authenticated production runtime - run: node infra/ci/prepare-release.mjs application + run: | + test -n "$LZC_MAILOUT_BINDINGS" + node infra/ci/prepare-release.mjs application env: + LZC_MAILOUT_BINDINGS: ${{ secrets.LZC_MAILOUT_BINDINGS }} LZC_AUTH_ENABLED: 'true' LZC_ASSISTANT_ENABLED: 'true' LZC_STACKIT_DEVICE_ENABLED: 'true' @@ -215,6 +218,7 @@ jobs: run: node infra/ci/check-production-route.mjs - name: Verify runtime service and isolated Runner connections run: | + node infra/ci/verify-mailout-binding.mjs cf run-task landing-zone-configurator --command './runtime/bin/node apps/api/dist/check-connections.js' --name "connections-$GITHUB_RUN_ID" -m 128M -k 4096M --wait cf run-task landing-zone-configurator --command "./runtime/bin/node apps/api/dist/check-plan-runner.js $GITHUB_RUN_ID" --name "runner-$GITHUB_RUN_ID" -m 128M -k 4096M --wait for attempt in 1 2 3; do diff --git a/landing-zone-configurator/app/apps/api/db/101_reported_runner_retirement.sql b/landing-zone-configurator/app/apps/api/db/101_reported_runner_retirement.sql new file mode 100644 index 00000000..5b58f25f --- /dev/null +++ b/landing-zone-configurator/app/apps/api/db/101_reported_runner_retirement.sql @@ -0,0 +1,38 @@ +-- A reported native Apply failure has already published its output and any +-- errored State. Retire its compute process, never its retained platform State. +ALTER TABLE lzc.plan_runs ADD COLUMN runner_stop_confirmed_at timestamptz; +CREATE INDEX reported_runner_retirement_queue ON lzc.plan_runs(finished_at,id) + WHERE operation='apply' AND status='recovery_required' AND error_code='apply_failed' + AND NOT cleaned AND runner_stop_confirmed_at IS NULL; + +CREATE FUNCTION lzc_auth.reported_s3_apply_runners(p_run uuid DEFAULT NULL) +RETURNS TABLE(id uuid,runner_app_id uuid,runner_droplet_id uuid) +LANGUAGE sql SECURITY DEFINER SET search_path=pg_catalog AS $$ + SELECT r.id,r.runner_app_id,r.runner_droplet_id + FROM lzc.plan_runs r JOIN lzc.platform_states s ON s.state_key=r.state_key + WHERE (p_run IS NULL OR r.id=p_run) + AND r.operation='apply' AND r.status='recovery_required' + AND r.error_code='apply_failed' AND r.input_claimed + AND r.finished_atr.id + AND other.runner_app_id=r.runner_app_id AND other.status IN ('starting','initializing','validating','planning','applying')) + ORDER BY r.finished_at,r.id LIMIT 20; +$$; +REVOKE ALL ON FUNCTION lzc_auth.reported_s3_apply_runners(uuid) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION lzc_auth.reported_s3_apply_runners(uuid) TO configurator_app; + +-- This timestamp only avoids repeated maintenance work. Checkpoint/reconcile +-- still independently inspect CF process/task termination on every request. +CREATE FUNCTION lzc_auth.record_reported_s3_runner_stopped(p_run uuid,p_app uuid,p_source uuid) +RETURNS void LANGUAGE sql SECURITY DEFINER SET search_path=pg_catalog AS $$ + UPDATE lzc.plan_runs r SET runner_stop_confirmed_at=clock_timestamp() + WHERE r.id=p_run AND r.runner_app_id=p_app AND r.runner_droplet_id=p_source + AND EXISTS(SELECT 1 FROM lzc_auth.reported_s3_apply_runners(p_run) candidate WHERE candidate.id=r.id); +$$; +REVOKE ALL ON FUNCTION lzc_auth.record_reported_s3_runner_stopped(uuid,uuid,uuid) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION lzc_auth.record_reported_s3_runner_stopped(uuid,uuid,uuid) TO configurator_app; diff --git a/landing-zone-configurator/app/apps/api/src/plans/cloud-foundry.ts b/landing-zone-configurator/app/apps/api/src/plans/cloud-foundry.ts index 0baa12f5..b4914965 100644 --- a/landing-zone-configurator/app/apps/api/src/plans/cloud-foundry.ts +++ b/landing-zone-configurator/app/apps/api/src/plans/cloud-foundry.ts @@ -25,6 +25,11 @@ export interface PlanRunner { sourceDropletId: string, ): Promise; removeRecovery?(id: string, appId: string | null): Promise; + stopReportedFailure?( + id: string, + appId: string, + sourceDropletId: string, + ): Promise; isStopped?( id: string, appId: string, @@ -428,6 +433,17 @@ export class CloudFoundryPlanRunner implements PlanRunner { async isStopped(id: string, appId: string, sourceDropletId: string) { return this.inspectStopped(id, appId, sourceDropletId, false); } + async stopReportedFailure( + id: string, + appId: string, + sourceDropletId: string, + ) { + // Verify exact app, space and immutable source before any CF mutation. + // Only the broker's reported-failure retirement path may call this method. + if (await this.inspectStopped(id, appId, sourceDropletId, false)) return; + const call = await this.authorized(); + await call(`apps/${appId}/actions/stop`, "POST"); + } async isRecoveryStopped(id: string, appId: string, sourceDropletId: string) { return this.inspectStopped(id, appId, sourceDropletId, true); } diff --git a/landing-zone-configurator/app/apps/api/src/plans/service.ts b/landing-zone-configurator/app/apps/api/src/plans/service.ts index af98fe4a..e43e0522 100644 --- a/landing-zone-configurator/app/apps/api/src/plans/service.ts +++ b/landing-zone-configurator/app/apps/api/src/plans/service.ts @@ -1508,6 +1508,36 @@ export class Plans { }); } async maintain() { + let retirementError: unknown; + if (this.runner.stopReportedFailure && this.runner.isStopped) { + const failed = ( + await this.pool.query( + "SELECT * FROM lzc_auth.reported_s3_apply_runners()", + ) + ).rows; + for (const run of failed) { + try { + await this.runner.stopReportedFailure( + run.id, + run.runner_app_id, + run.runner_droplet_id, + ); + if ( + await this.runner.isStopped( + run.id, + run.runner_app_id, + run.runner_droplet_id, + ) + ) + await this.pool.query( + "SELECT lzc_auth.record_reported_s3_runner_stopped($1,$2,$3)", + [run.id, run.runner_app_id, run.runner_droplet_id], + ); + } catch (error) { + retirementError ??= error; + } + } + } const rows = ( await this.pool.query("SELECT * FROM lzc_auth.expire_plan_runs()") ).rows; @@ -1515,5 +1545,6 @@ export class Plans { await this.runner.remove(row.id, row.runner_app_id); await this.pool.query("SELECT lzc_auth.mark_plan_cleaned($1)", [row.id]); } + if (retirementError) throw retirementError; } } diff --git a/landing-zone-configurator/app/tests/platform-plan.test.ts b/landing-zone-configurator/app/tests/platform-plan.test.ts index f0fbc203..6aeb63d2 100644 --- a/landing-zone-configurator/app/tests/platform-plan.test.ts +++ b/landing-zone-configurator/app/tests/platform-plan.test.ts @@ -2093,6 +2093,157 @@ describe.skipIf(!enabled)( return { ...test, run, saved, descriptor, stopped }; } + it("automatically stops a reported failed S3 runner without changing State or clearing the recovery hold", async () => { + const test = await failedRemoteApply(); + test.stopped.mockResolvedValue(false); + await expect( + test.plans.checkpoint(test.owner, test.run.id), + ).rejects.toMatchObject({ code: "runner_not_stopped" }); + await migration!.query( + "UPDATE lzc.plan_runs SET finished_at=now()-interval '1 minute' WHERE id=$1", + [test.run.id], + ); + const key = stableStateKey(test.owner, test.manifest); + // Cover the normal Configurator bootstrap migration marker as well. + await migration!.query( + "UPDATE lzc.platform_states SET pending_backend_id=backend_id WHERE state_key=$1", + [key], + ); + const before = ( + await migration!.query( + "SELECT * FROM lzc.platform_states WHERE state_key=$1", + [key], + ) + ).rows[0]; + const remoteBefore = [...test.remote]; + const stop = vi.fn(async (_id: string, _app: string, _source: string) => { + test.stopped.mockResolvedValue(true); + }); + Object.assign(test.runner, { stopReportedFailure: stop }); + await test.plans.maintain(); + expect(stop).toHaveBeenCalledWith( + test.run.id, + expect.any(String), + expect.any(String), + ); + const run = ( + await migration!.query( + "SELECT status,cleaned,runner_stop_confirmed_at FROM lzc.plan_runs WHERE id=$1", + [test.run.id], + ) + ).rows[0]; + expect(run.status).toBe("recovery_required"); + expect(run.cleaned).toBe(false); + expect(run.runner_stop_confirmed_at).not.toBeNull(); + expect( + ( + await migration!.query( + "SELECT * FROM lzc.platform_states WHERE state_key=$1", + [key], + ) + ).rows[0], + ).toEqual(before); + expect([...test.remote]).toEqual(remoteBefore); + expect( + (await test.plans.checkpoint(test.owner, test.run.id)).canResume, + ).toBe(true); + // A prior maintenance receipt never substitutes for a fresh process check. + test.stopped.mockResolvedValue(false); + await expect( + test.plans.checkpoint(test.owner, test.run.id), + ).rejects.toMatchObject({ code: "runner_not_stopped" }); + stop.mockClear(); + await test.plans.maintain(); + expect(stop.mock.calls.filter((c) => c[0] === test.run.id)).toHaveLength( + 0, + ); + }); + + it.each([ + "fresh", + "timed_out", + "state_failed", + "runner_unavailable", + "pending-target", + "backup", + "recovery", + "stop-unconfirmed", + "stop-error", + ])( + "keeps S3 failure recovery blocked for retirement case %s", + async (reason) => { + const test = await failedRemoteApply(); + const key = stableStateKey(test.owner, test.manifest); + await migration!.query( + "UPDATE lzc.plan_runs SET finished_at=now()-interval '1 minute' WHERE id=$1", + [test.run.id], + ); + if (reason === "fresh") + await migration!.query( + "UPDATE lzc.plan_runs SET finished_at=now() WHERE id=$1", + [test.run.id], + ); + if ( + ["timed_out", "state_failed", "runner_unavailable"].includes(reason) + ) + await migration!.query( + "UPDATE lzc.plan_runs SET error_code=$2 WHERE id=$1", + [test.run.id, reason], + ); + if (reason === "pending-target") { + const other = await test.backends.register(test.owner, { + descriptor: { ...test.descriptor, key: "pending.tfstate" }, + credentials: { + accessKeyId: "fixture-only", + secretAccessKey: "fixture-only", + }, + }); + await migration!.query( + "UPDATE lzc.platform_states SET pending_backend_id=$2 WHERE state_key=$1", + [key, other.id], + ); + } + if (reason === "backup") + await migration!.query( + "UPDATE lzc.platform_states SET backend_id=NULL,ciphertext=$2 WHERE state_key=$1", + [key, Buffer.from("retained-bootstrap-backup")], + ); + if (reason === "recovery") + await migration!.query( + "INSERT INTO lzc.state_recoveries(run_id,tenant_id,state_key,sha256,ciphertext) SELECT $1,tenant_id,state_key,$2,$3 FROM lzc.platform_states WHERE state_key=$4", + [ + test.run.id, + "a".repeat(64), + Buffer.from("separate-recovery"), + key, + ], + ); + test.stopped.mockResolvedValue(false); + const stop = vi.fn( + async (_id: string, _app: string, _source: string) => { + if (reason === "stop-error") throw new Error("stop-unavailable"); + }, + ); + Object.assign(test.runner, { stopReportedFailure: stop }); + await test.plans.maintain().catch((error) => { + expect(reason).toBe("stop-error"); + expect(error.message).toBe("stop-unavailable"); + }); + if (!["stop-unconfirmed", "stop-error"].includes(reason)) + expect( + stop.mock.calls.filter((c) => c[0] === test.run.id), + ).toHaveLength(0); + const run = ( + await migration!.query( + "SELECT status,runner_stop_confirmed_at FROM lzc.plan_runs WHERE id=$1", + [test.run.id], + ) + ).rows[0]; + expect(run.status).toBe("recovery_required"); + expect(run.runner_stop_confirmed_at).toBeNull(); + }, + ); + it("reconciles a reported S3 Apply failure without changing remote state or replaying the approved plan", async () => { const test = await failedRemoteApply(); const key = stableStateKey(test.owner, test.manifest); diff --git a/landing-zone-configurator/app/tests/runner-stopped.test.ts b/landing-zone-configurator/app/tests/runner-stopped.test.ts index 8d6a292d..705d29bb 100644 --- a/landing-zone-configurator/app/tests/runner-stopped.test.ts +++ b/landing-zone-configurator/app/tests/runner-stopped.test.ts @@ -180,3 +180,31 @@ it("checks the exact recovery task without accepting normal or warm runners", as ); await expect(f.run()).rejects.toThrow("Runner identity mismatch"); }); + +it("stops an exact failed runner only after source verification, then still requires independent process proof", async () => { + const f = fixture(); + (f.data[`apps/${f.app}`] as { state: string }).state = "STARTED"; + f.data[`apps/${f.app}/actions/stop`] = {}; + await f.runner.stopReportedFailure(f.id, f.app, f.source); + expect( + f.calls + .filter((c) => c.method === "POST" && c.url.includes("/v3/")) + .map((c) => c.url), + ).toEqual([expect.stringContaining(`/v3/apps/${f.app}/actions/stop`)]); + await expect(f.run()).resolves.toBe(false); + (f.data[`apps/${f.app}`] as { state: string }).state = "STOPPED"; + await expect(f.run()).resolves.toBe(true); +}); +it("never stops a runner with a different source checksum", async () => { + const f = fixture(); + (f.data[`apps/${f.app}`] as { state: string }).state = "STARTED"; + ( + f.data[`apps/${f.app}/droplets/current`] as { checksum: { value: string } } + ).checksum = { value: "b".repeat(64) }; + await expect( + f.runner.stopReportedFailure(f.id, f.app, f.source), + ).rejects.toThrow(); + expect( + f.calls.some((c) => c.method === "POST" && c.url.includes("/v3/")), + ).toBe(false); +}); diff --git a/landing-zone-configurator/docs/plan-execution.de.md b/landing-zone-configurator/docs/plan-execution.de.md index 9f68cf23..076793a0 100644 --- a/landing-zone-configurator/docs/plan-execution.de.md +++ b/landing-zone-configurator/docs/plan-execution.de.md @@ -430,6 +430,14 @@ Destroy-Aktion. Die Migration muss vor Nutzung der neuen API aktiviert sein. Eine abgeschlossene Bootstrap-Migration behält `pending_backend_id` als Nachweis. Entspricht diese Referenz dem aktiven Backend und wurde der Bootstrap-Ciphertext entfernt, ist die Migration abgeschlossen. Ein anderes vorgemerktes Backend oder ein noch vorhandener Bootstrap-State bleibt gesperrt. +Nach einem gemeldeten S3-Fehler `apply_failed` stoppt die Hintergrundwartung nach +einer kurzen Abschlussfrist genau den geprüften Runner. Die App bleibt zur Prüfung +gestoppt; die normale Bereinigung folgt dem Abgleich. Dies gibt den Lauf nicht frei +und verändert keinen State. Timeouts, unklare Fehler, separate Recovery-States und +offene Migrationen sind ausgeschlossen. Ein Wartungszeitpunkt verhindert nur +wiederholte Stop-Anfragen; Checkpoint und Bestätigung prüfen den aktuellen CF-App-, +Task- und Prozesszustand weiterhin unabhängig. + Bei einem bestehenden S3-Backend prüft die Recovery den registrierten Remote-State anstelle eines Bootstrap-Checkpoints. Nur der Eigentümer eines mit `apply_failed` beendeten Laufs darf die erneute Planung bestätigen. Genau dieser Runner muss diff --git a/landing-zone-configurator/docs/plan-execution.md b/landing-zone-configurator/docs/plan-execution.md index 86bdbcc8..f208a538 100644 --- a/landing-zone-configurator/docs/plan-execution.md +++ b/landing-zone-configurator/docs/plan-execution.md @@ -196,6 +196,13 @@ The user can then save the corrected configuration, choose a new preparation and A completed bootstrap migration retains `pending_backend_id` as evidence. When it matches the active backend and bootstrap ciphertext has been removed, it is a completed migration, not an open migration. A different pending backend or retained bootstrap State remains blocked. +After a reported S3 `apply_failed`, background maintenance stops the exact verified +Runner after a short completion grace period. The app remains stopped for inspection; +normal cleanup follows reconciliation. This neither reconciles the run nor changes +State. Timeouts, ambiguous failures, separate recovery State and open migrations are +excluded. A maintenance timestamp only avoids repeated stop requests; checkpoint and +confirmation always independently verify current CF app, task and process status. + For an existing S3 backend, recovery inspects the registered remote State rather than a bootstrap checkpoint. Only the owner of a reported `apply_failed` run may confirm replanning. The exact Runner must be stopped or already cleaned; local diff --git a/landing-zone-configurator/infra/ci/release-notes.mjs b/landing-zone-configurator/infra/ci/release-notes.mjs index f323349c..63e52b94 100644 --- a/landing-zone-configurator/infra/ci/release-notes.mjs +++ b/landing-zone-configurator/infra/ci/release-notes.mjs @@ -1,6 +1,14 @@ const tag=process.env.LZC_RELEASE_TAG; if(!/^v(?:[2-9]|[1-9][0-9]+)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$/.test(tag??""))throw new Error("Expected a stable product release tag"); const notes={ + "v2.2.2": { + introduction:"Production email delivery and reliable recovery after reported S3 Apply failures.", + changes:[ + "Provision a dedicated production MailOut sender and bind SMTP only to the application backend, with DNS validation and deployment verification.", + "Stop source-verified runners after explicitly reported S3 Apply failures so checkpoint inspection can proceed; retain State and require independent process checks and owner confirmation.", + "Preserve recovery holds for timeouts, ambiguous interruptions, pending migrations and separate recovery States.", + ], + }, "v2.2.1": { introduction:"Context-aware service consultation in the Landing Zone Configurator.", changes:[ diff --git a/landing-zone-configurator/infra/mailout/README.de.md b/landing-zone-configurator/infra/mailout/README.de.md index 77375dc4..0f1423e1 100644 --- a/landing-zone-configurator/infra/mailout/README.de.md +++ b/landing-zone-configurator/infra/mailout/README.de.md @@ -5,17 +5,17 @@ Dieses Verzeichnis richtet die Absender-Anbindung des Betreibers ein, kein Kunden-Landing-Zone-Feature. Da der gepinnte STACKIT-Terraform-Provider keine MailOut-Ressourcen bereitstellt, verwendet es die offiziellen MailOut-v1beta1- -und DNS-v1-APIs. `environments/lzc-dev.json` beschreibt den geprüften Sollzustand. +und DNS-v1-APIs. `environments/lzc-dev.json` und `environments/lzc-prod.json` definieren getrennte Absenderdomains, Projekte und Sender. Der Reconciler erzeugt nur fehlende dedizierte Ressourcen und prüft bestehende. Er überschreibt keine DNS-Richtlinien und rotiert oder löscht keine Zugangsdaten -automatisch. Produktion ist für diesen ersten Dev-Rollout ausdrücklich kein Ziel. +automatisch. Für Produktion die geprüfte Umgebung ausdrücklich auswählen; Standard bleibt Dev. ## Einrichten und anbinden Node 24.21.0 und die offizielle STACKIT CLI verwenden. Der Betreiber-Key benötigt -MailOut-Zugriff in Dev und DNS-Zugriff auf die vorhandene Elternzone. Schlüssel +MailOut-Zugriff im gewählten Projekt und DNS-Zugriff auf die vorhandene Elternzone. Schlüssel nicht versionieren. Die App-Route bleibt unverändert; verwaltet werden ausschließlich -SPF, DMARC, DKIM und Return-Path der Absender-Subdomain. MailOut prüft die Einträge +SPF, DMARC, DKIM und Return-Path der Absenderdomain. MailOut prüft die Einträge fortlaufend; nach erfolgreicher Verifikation nicht entfernen. ```sh @@ -68,6 +68,29 @@ MailOut-Werte mit der freigegebenen Bindung, bevor er SMTP prüft. Fehlende, deaktivierte oder veraltete Bindungen lassen den Release fehlschlagen; Verbindungstests können eingerichteten E-Mail-Versand nicht unbemerkt überspringen. +## Produktion einrichten + +Mit `provision.mjs plan lzc-prod` den Umfang prüfen und anschließend +`provision.mjs apply lzc-prod` ausführen. Während der Provider die Domain anlegt, +kann `ready:false` ohne DNS-Prüfdaten erscheinen; später denselben Abgleich +wiederholen. Produktion verwendet `notifications@landing-zone.runs.onstackit.cloud` +und eine eigene private Bindung in `.local/mailout-prod/bindings.private.json`. +Dev-Zugangsdaten sind nicht wiederverwendbar. Nur eine geprüfte, bereite Bindung +veröffentlichen: + +```sh +gh secret set LZC_MAILOUT_BINDINGS --repo stackitcloud/stackit-landing-zone \ + --env lzc-prod-release < landing-zone-configurator/.local/mailout-prod/bindings.private.json +``` + +Das Prod-Deployment verlangt die SMTP-Bindung und bricht bei fehlender Bindung vor der Aktivierung ab. + +Production Release übergibt dieses Secret an die Runtime-Vorbereitung und prüft +die tatsächliche CF-MailOut-Bindung vor dem SMTP-Verbindungstest. Staging, Runner +und Migrations-App erhalten keine SMTP-Zugangsdaten und versenden keine Mails. +Vorlagen im STACKIT-Erscheinungsbild, Sprache und persönliche Themenpräferenzen +entsprechen Dev; den Einladungsversand wählt der Einladende. + ## Zustellungsdiagnose und Wiederherstellung - Posteingang und Einladungsliste zeigen vorgemerkt/laufend/angenommen/unterdrückt/ diff --git a/landing-zone-configurator/infra/mailout/README.md b/landing-zone-configurator/infra/mailout/README.md index c216d3a6..6ece1b0b 100644 --- a/landing-zone-configurator/infra/mailout/README.md +++ b/landing-zone-configurator/infra/mailout/README.md @@ -5,16 +5,16 @@ This directory provisions the operator's sending integration, not a customer Landing Zone feature. It uses the official MailOut v1beta1 and DNS v1 APIs because the pinned STACKIT Terraform provider does not provide MailOut resources. -`environments/lzc-dev.json` is the reviewed declarative desired state. The reconciler +`environments/lzc-dev.json` and `environments/lzc-prod.json` declare separate sending domains, projects and senders. The reconciler only creates missing dedicated resources and validates existing ones; it never rewrites existing DNS policies or automatically rotates/deletes credentials. -Production is deliberately not an accepted target in this first Dev rollout. +Select the reviewed environment explicitly for production; the default remains Dev. ## Provision and bind Use Node 24.21.0 and the official STACKIT CLI. The operator key needs MailOut access -in Dev and DNS access to the existing parent zone. Keep keys outside version control. -The application route/domain is not changed; only the sending subdomain's SPF, +in the selected project and DNS access to the existing parent zone. Keep keys outside version control. +The application route/domain is not changed; only the sending domain's SPF, DMARC, DKIM and return-path records are managed. Do not remove these records after validation: MailOut checks them continuously. @@ -64,6 +64,28 @@ fail the release; connection checks cannot silently skip configured email delive The existing artifact-encryption key encrypts invitation tokens with separate, context-bound purposes; preserve that key while encrypted pending jobs exist. +## Production setup + +Use `provision.mjs plan lzc-prod` and, after reviewing the scoped plan, +`provision.mjs apply lzc-prod`. Provider creation can initially report `ready:false` +without authentication records; rerun reconciliation once the domain is ready. +Production uses `notifications@landing-zone.runs.onstackit.cloud`, with a separate +private binding in `.local/mailout-prod/bindings.private.json`. Dev credentials +cannot be reused. Publish only a validated, ready binding: + +```sh +gh secret set LZC_MAILOUT_BINDINGS --repo stackitcloud/stackit-landing-zone \ + --env lzc-prod-release < landing-zone-configurator/.local/mailout-prod/bindings.private.json +``` + +Production deployment requires the SMTP binding and fails before activation if it is missing. + +The Production Release workflow passes this secret to runtime preparation and +verifies the actual CF MailOut binding before SMTP connection checks. Staging, +Runner and migration applications never receive SMTP credentials or send mail. +Production sending uses the same branded templates, language and per-user topic +preferences as Dev; invitation delivery is selected by the inviter. + ## Delivery diagnostics and recovery - The user inbox and invitation list expose bounded queued/sending/accepted/ diff --git a/landing-zone-configurator/infra/mailout/bindings.test.mjs b/landing-zone-configurator/infra/mailout/bindings.test.mjs index fc42bfec..04cfb033 100644 --- a/landing-zone-configurator/infra/mailout/bindings.test.mjs +++ b/landing-zone-configurator/infra/mailout/bindings.test.mjs @@ -20,3 +20,11 @@ test('the real CF manifest carries enabled and disabled bindings through to the for(const name of ['../../deploy/runner/manifest.yml','../../deploy/cloud-foundry/migration-manifest.yml']) assert.doesNotMatch(readFileSync(new URL(name,import.meta.url),'utf8'),/LZC_MAILOUT_/); }); + +test('binds production SMTP only to the production sender and validates it after activation',()=>{ + const b={...binding,projectId:prod,from:'notifications@landing-zone.runs.onstackit.cloud'}; + assert.equal(mailoutEnvironment(JSON.stringify(b),'lzc-prod',prod).LZC_MAILOUT_ENABLED,'true'); + const workflow=readFileSync(new URL('../../../.github/workflows/configurator-production.yml',import.meta.url),'utf8'); + assert.match(workflow,/LZC_MAILOUT_BINDINGS: \$\{\{ secrets.LZC_MAILOUT_BINDINGS \}\}/); + assert.match(workflow,/node infra\/ci\/verify-mailout-binding.mjs/); +}); diff --git a/landing-zone-configurator/infra/mailout/environments/lzc-prod.json b/landing-zone-configurator/infra/mailout/environments/lzc-prod.json new file mode 100644 index 00000000..2d5d7d00 --- /dev/null +++ b/landing-zone-configurator/infra/mailout/environments/lzc-prod.json @@ -0,0 +1,11 @@ +{ + "environment": "lzc-prod", + "projectId": "b601468a-e897-4f74-8e10-ac4ab1e448af", + "region": "eu01", + "domain": "landing-zone.runs.onstackit.cloud", + "from": "notifications@landing-zone.runs.onstackit.cloud", + "senderDescription": "Landing Zone Configurator Prod notifications v1", + "dnsProjectId": "b601468a-e897-4f74-8e10-ac4ab1e448af", + "dnsZoneId": "59775d2f-b237-4c71-97dd-5d8624223ef5", + "dnsZone": "landing-zone.runs.onstackit.cloud" +} diff --git a/landing-zone-configurator/infra/mailout/provision.mjs b/landing-zone-configurator/infra/mailout/provision.mjs index ce455ef7..6fe05cb0 100644 --- a/landing-zone-configurator/infra/mailout/provision.mjs +++ b/landing-zone-configurator/infra/mailout/provision.mjs @@ -8,7 +8,11 @@ import {spawnSync} from 'node:child_process'; const uuid=/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/; const hostname=/^(?:[a-z0-9_][a-z0-9_-]*\.)+[a-z0-9-]+$/; export function validateManifest(c) { - if(c.environment!=='lzc-dev'||c.projectId!=='7dbff805-6788-4d5d-88c0-a5212ac5e90f'||c.region!=='eu01'||c.domain!=='dev.landing-zone.runs.onstackit.cloud'||c.from!==`notifications@${c.domain}`||!uuid.test(c.dnsProjectId)||!uuid.test(c.dnsZoneId)||c.dnsZone!=='landing-zone.runs.onstackit.cloud'||!c.senderDescription||c.senderDescription.length>100)throw Error('mailout_manifest_invalid'); + const target={ + 'lzc-dev':{project:'7dbff805-6788-4d5d-88c0-a5212ac5e90f',domain:'dev.landing-zone.runs.onstackit.cloud'}, + 'lzc-prod':{project:'b601468a-e897-4f74-8e10-ac4ab1e448af',domain:'landing-zone.runs.onstackit.cloud'}, + }[c.environment]; + if(!target||c.projectId!==target.project||c.region!=='eu01'||c.domain!==target.domain||c.from!==`notifications@${c.domain}`||!uuid.test(c.dnsProjectId)||!uuid.test(c.dnsZoneId)||c.dnsZone!=='landing-zone.runs.onstackit.cloud'||!c.senderDescription||c.senderDescription.length>100)throw Error('mailout_manifest_invalid'); return c; } export function dnsRecords(c,config) { @@ -19,7 +23,7 @@ export function dnsRecords(c,config) { if(host==='@'||host==='')host=c.domain; else if(host!==c.domain&&!host.endsWith(`.${c.domain}`))host=`${host}.${c.domain}`; if(!hostname.test(host)||!(host===c.domain||host.endsWith(`.${c.domain}`))||!['TXT','CNAME'].includes(r.type)||typeof r.value!=='string'||!r.value||/[\r\n]/.test(r.value))throw Error('mailout_dns_record_outside_scope'); - return {name:`${host}.`,type:r.type,ttl:300,records:[{content:r.type==='CNAME'?`${r.value.replace(/\.$/,'')}.`:r.value}],comment:'Landing Zone Configurator Dev MailOut'}; + return {name:`${host}.`,type:r.type,ttl:300,records:[{content:r.type==='CNAME'?`${r.value.replace(/\.$/,'')}.`:r.value}],comment:`Landing Zone Configurator ${c.environment} MailOut`}; }); } export async function reconcile(c,request,{apply=false,binding,saveBinding=()=>{},saveRecovery=()=>{},record=()=>{}}={}) { @@ -37,6 +41,7 @@ export async function reconcile(c,request,{apply=false,binding,saveBinding=()=>{ if(!uuid.test(domain.id))throw Error('mailout_domain_identity_invalid'); const root=`${base}/${domain.id}`; const detail=await request('mailout',root); + if(detail.state==='creating'&&!detail.config?.dnsRecords?.spf)return {domainId:domain.id,state:'creating',ready:false,dnsCreates:0}; const desired=dnsRecords(c,detail.config); const existing=await request('dns',`${dns}/rrsets?pageSize=10000`); if(existing.totalPages>1||!Array.isArray(existing.rrSets))throw Error('mailout_dns_inventory_incomplete'); @@ -74,8 +79,10 @@ export async function reconcile(c,request,{apply=false,binding,saveBinding=()=>{ async function main() { process.umask(0o077); const mode=process.argv[2];if(!['plan','apply'].includes(mode))throw Error('expected_plan_or_apply'); - const c=validateManifest(JSON.parse(readFileSync(new URL('./environments/lzc-dev.json',import.meta.url)))); - const dir=resolve(process.env.LZC_MAILOUT_PRIVATE??fileURLToPath(new URL('../../.local/mailout/',import.meta.url)));mkdirSync(dir,{recursive:true,mode:0o700}); + const environment=process.argv[3]??'lzc-dev'; + if(!['lzc-dev','lzc-prod'].includes(environment)||process.argv.length>4)throw Error('mailout_environment_invalid'); + const c=validateManifest(JSON.parse(readFileSync(new URL(`./environments/${environment}.json`,import.meta.url)))); + const dir=resolve(process.env.LZC_MAILOUT_PRIVATE??fileURLToPath(new URL(environment==='lzc-dev'?'../../.local/mailout/':'../../.local/mailout-prod/',import.meta.url)));mkdirSync(dir,{recursive:true,mode:0o700}); if(statSync(dir).mode&0o077)throw Error('mailout_private_directory_permissions'); const file=resolve(dir,'bindings.private.json');let binding; if(existsSync(file)){if(statSync(file).mode&0o077)throw Error('mailout_binding_permissions');binding=JSON.parse(readFileSync(file));} diff --git a/landing-zone-configurator/infra/mailout/provision.test.mjs b/landing-zone-configurator/infra/mailout/provision.test.mjs index 56def8dd..9f1fe776 100644 --- a/landing-zone-configurator/infra/mailout/provision.test.mjs +++ b/landing-zone-configurator/infra/mailout/provision.test.mjs @@ -42,3 +42,19 @@ test('fails closed if an existing sender password has been lost',async()=>{ test('does not issue sender credentials before domain validation succeeds',async()=>{ const f=fixture({state:'validating'});assert.equal((await reconcile(c,f.request,{apply:true})).ready,false);assert.equal(f.saved.length,0);assert.ok(!f.calls.some(r=>r.path.endsWith('/authorized-senders'))); }); + +test('accepts the reviewed production sender scope without accepting cross-stage manifests',()=>{ + const prod=JSON.parse(readFileSync(new URL('./environments/lzc-prod.json',import.meta.url))); + assert.equal(validateManifest(prod),prod); + assert.throws(()=>validateManifest({...prod,projectId:c.projectId})); + assert.throws(()=>validateManifest({...prod,domain:c.domain})); + assert.throws(()=>validateManifest({...prod,from:c.from})); + assert.ok(dnsRecords(prod,config).every(r=>r.name===`${prod.domain}.`||r.name.endsWith(`.${prod.domain}.`))); +}); + +test('waits for provider creation before expecting authentication records or issuing writes',async()=>{ + const f=fixture({state:'creating'}); + const request=async(...args)=>{const response=await f.request(...args);return args[1].endsWith(`/${domainId}`)?{...response,config:{dnsRecords:{}}}:response;}; + const result=await reconcile(c,request,{apply:true});assert.equal(result.ready,false);assert.equal(result.state,'creating'); + assert.ok(f.calls.every(call=>call.method==='GET'));assert.equal(f.saved.length,0); +});