Harden atomic cPanel release recovery (#238)

Adds exact-SHA cPanel release proof v2, atomic rollback restoration, and protected recovery gating.
2026-07-29 20:30:34 +02:00
parent c85a82b9ac
commit 468d436d3e
5 changed files with 520 additions and 39 deletions
@@ -142,13 +142,19 @@ jobs:
          const fs = require("node:fs");
          const proof = JSON.parse(fs.readFileSync(process.env.PROOF_PATH, "utf8"));
          const checks = {
-            schema: proof.schemaVersion === 1,
+            schema: proof.schemaVersion === 2,
            repository: proof.repository === process.env.GITHUB_REPOSITORY,
            source: proof.sourceSha === process.env.IOS_SOURCE_SHA,
+            exactSource: proof.sha === process.env.IOS_SOURCE_SHA,
+            releaseIdentity: typeof proof.releaseId === "string" && proof.releaseId.length > 0,
+            archive: /^[a-f0-9]{64}$/.test(proof.archiveSha256 || ""),
+            activeTarget: typeof proof.activeTarget === "string" && proof.activeTarget.length > 0,
+            verification: proof.verificationState === "verified",
            publicGate: proof.livePublicGate === "passed",
-            credentialedGate: proof.liveCredentialedGate === "passed",
+            credentialedGate: ["passed", "not-configured"].includes(proof.liveCredentialedGate),
            managerGate: proof.releaseManagerGate === "passed",
            serverVersion: proof.serverVersionUpdated === true,
+            serverVersionReadBack: proof.serverVersionReadBack === "passed",
          };
          const failures = Object.entries(checks).filter(([, passed]) => !passed).map(([label]) => label);
          if (failures.length) throw new Error(`Invalid frontend release proof: ${failures.join(", ")}`);
@@ -0,0 +1,274 @@
+name: Frontend Release Recovery
+
+on:
+  workflow_dispatch:
+    inputs:
+      action:
+        description: Verify the active release or roll back before verification
+        required: true
+        type: choice
+        options:
+          - reverify
+          - rollback
+      source_sha:
+        description: Exact 40-character commit SHA expected after recovery
+        required: true
+        type: string
+      rollback_target:
+        description: Immutable releases/.../dist target; required for rollback
+        required: false
+        type: string
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: frontend-production
+  cancel-in-progress: false
+
+jobs:
+  recover:
+    name: Protected production recovery
+    runs-on: ubuntu-latest
+    environment: frontend-production
+    timeout-minutes: 35
+    env:
+      PLAYWRIGHT_BASE_URL: ${{ vars.PRODUCTION_FRONTEND_URL || 'https://truckwash.io' }}
+    steps:
+      - name: Validate exact recovery target
+        shell: bash
+        env:
+          RECOVERY_ACTION: ${{ inputs.action }}
+          RECOVERY_SHA: ${{ inputs.source_sha }}
+          RECOVERY_TARGET: ${{ inputs.rollback_target }}
+        run: |
+          set -euo pipefail
+          [[ "$RECOVERY_SHA" =~ ^[a-f0-9]{40}$ ]]
+          if [[ "$RECOVERY_ACTION" == "rollback" ]]; then
+            [[ "$RECOVERY_TARGET" =~ ^releases/[A-Za-z0-9._-]+/dist$ ]]
+          else
+            [[ -z "$RECOVERY_TARGET" ]]
+          fi
+
+      - name: Checkout exact recovery source
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Authorize source from successful release proof
+        id: authorize
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RECOVERY_ACTION: ${{ inputs.action }}
+          RECOVERY_SHA: ${{ inputs.source_sha }}
+          RECOVERY_TARGET: ${{ inputs.rollback_target }}
+        run: |
+          set -euo pipefail
+          runs="$RUNNER_TEMP/recovery-runs.json"
+          artifacts="$RUNNER_TEMP/recovery-artifacts.json"
+          curl --fail --silent --show-error \
+            -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
+            "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/workflows/release.yml/runs?head_sha=$RECOVERY_SHA&status=success&per_page=20" \
+            > "$runs"
+          release_run_id="$(jq -r '[.workflow_runs[] | select(.event == "workflow_run")] | first | .id // empty' "$runs")"
+          [[ "$release_run_id" =~ ^[0-9]+$ ]]
+          artifact_name="frontend-release-proof-$RECOVERY_SHA"
+          curl --fail --silent --show-error \
+            -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
+            "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/runs/$release_run_id/artifacts?name=$artifact_name&per_page=20" \
+            > "$artifacts"
+          artifact_id="$(jq -r '[.artifacts[] | select(.expired == false)] | first | .id // empty' "$artifacts")"
+          [[ "$artifact_id" =~ ^[0-9]+$ ]]
+          mkdir -p "$RUNNER_TEMP/recovery-proof"
+          curl --fail --silent --show-error --location \
+            -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
+            "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" \
+            -o "$RUNNER_TEMP/recovery-proof.zip"
+          unzip -q "$RUNNER_TEMP/recovery-proof.zip" -d "$RUNNER_TEMP/recovery-proof"
+          PROOF_PATH="$RUNNER_TEMP/recovery-proof/frontend-release-proof.json" \
+          RELEASE_RUN_ID="$release_run_id" node <<'NODE'
+          const { appendFileSync, readFileSync } = require("node:fs");
+          const proof = JSON.parse(readFileSync(process.env.PROOF_PATH, "utf8"));
+          const sha = process.env.RECOVERY_SHA;
+          const target = process.env.RECOVERY_TARGET;
+          const expectedPrefix = `releases/${sha}-`;
+          const valid = proof.schemaVersion === 2
+            && proof.repository === process.env.GITHUB_REPOSITORY
+            && proof.sha === sha
+            && proof.sourceSha === sha
+            && proof.frontendReleaseRunId === process.env.RELEASE_RUN_ID
+            && proof.verificationState === "verified"
+            && proof.livePublicGate === "passed"
+            && ["passed", "not-configured"].includes(proof.liveCredentialedGate)
+            && proof.releaseManagerGate === "passed"
+            && proof.serverVersionUpdated === true
+            && proof.serverVersionReadBack === "passed"
+            && typeof proof.activeTarget === "string"
+            && proof.activeTarget.startsWith(expectedPrefix)
+            && proof.activeTarget.endsWith("/dist");
+          if (!valid) throw new Error("Recovery source does not have valid exact-release proof.");
+          if (process.env.RECOVERY_ACTION === "rollback" && target !== proof.activeTarget) {
+            throw new Error("Rollback target does not match the verified release proof.");
+          }
+          appendFileSync(process.env.GITHUB_OUTPUT, `verified_target=${proof.activeTarget}\n`);
+          NODE
+
+      - name: Capture current immutable target
+        id: current
+        shell: bash
+        env:
+          FRONTEND_URL: ${{ vars.PRODUCTION_FRONTEND_URL || 'https://truckwash.io' }}
+        run: |
+          node --input-type=module <<'NODE'
+          import { appendFileSync } from "node:fs";
+          const response = await fetch(new URL(`release-manifest.json?recovery=${Date.now()}`, process.env.FRONTEND_URL), {
+            headers: { "Cache-Control": "no-cache", Pragma: "no-cache" },
+          });
+          if (!response.ok) throw new Error(`Active manifest returned HTTP ${response.status}.`);
+          const manifest = await response.json();
+          const sha = String(manifest.commit_sha || "").toLowerCase();
+          const build = String(manifest.build_id || "");
+          if (!/^[a-f0-9]{40}$/.test(sha) || !/^[A-Za-z0-9._-]{1,180}$/.test(build)) {
+            throw new Error("Active manifest has invalid release identity.");
+          }
+          appendFileSync(process.env.GITHUB_OUTPUT, `previous_sha=${sha}\nprevious_target=releases/${sha}-${build}/dist\n`);
+          NODE
+
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci --legacy-peer-deps
+
+      - name: Install secure FTP client without system changes
+        run: |
+          if command -v lftp >/dev/null 2>&1; then
+            exit 0
+          fi
+          package_root="$RUNNER_TEMP/lftp-package"
+          mkdir -p "$package_root"
+          (
+            cd "$package_root"
+            apt-get download lftp
+            dpkg-deb --extract ./lftp_*.deb root
+          )
+          echo "$package_root/root/usr/bin" >> "$GITHUB_PATH"
+
+      - name: Install Playwright Chromium
+        run: node scripts/install-playwright-browsers.mjs chromium
+
+      - name: Roll back atomically
+        if: inputs.action == 'rollback'
+        id: rollback
+        run: node scripts/release/deploy-cpanel.mjs --rollback
+        env:
+          NODE_OPTIONS: --use-system-ca
+          RELEASE_ROLLBACK_TARGET: ${{ inputs.rollback_target }}
+          PRODUCTION_FTP_HOST: ${{ secrets.PRODUCTION_FTP_HOST }}
+          PRODUCTION_FTP_USER: ${{ secrets.PRODUCTION_FTP_USER }}
+          PRODUCTION_FTP_PASSWORD: ${{ secrets.PRODUCTION_FTP_PASSWORD }}
+          PRODUCTION_FTP_PATH: ${{ secrets.PRODUCTION_FTP_PATH }}
+          PRODUCTION_ACTIVATION_KEY: ${{ secrets.PRODUCTION_ACTIVATION_KEY }}
+          PRODUCTION_FRONTEND_URL: ${{ vars.PRODUCTION_FRONTEND_URL || 'https://truckwash.io' }}
+
+      - name: Verify active manifest matches authorized release
+        shell: bash
+        env:
+          EXPECTED_SHA: ${{ inputs.source_sha }}
+          EXPECTED_TARGET: ${{ steps.authorize.outputs.verified_target }}
+          FRONTEND_URL: ${{ vars.PRODUCTION_FRONTEND_URL || 'https://truckwash.io' }}
+        run: |
+          node --input-type=module <<'NODE'
+          const deadline = Date.now() + 300_000;
+          let actual = "";
+          while (Date.now() < deadline) {
+            const response = await fetch(new URL(`release-manifest.json?recovery=${Date.now()}`, process.env.FRONTEND_URL), {
+              headers: { "Cache-Control": "no-cache", Pragma: "no-cache" },
+            });
+            if (response.ok) {
+              const manifest = await response.json();
+              const manifestSha = String(manifest.commit_sha || "").toLowerCase();
+              actual = `releases/${manifestSha}-${String(manifest.build_id || "")}/dist`;
+              if (manifestSha === process.env.EXPECTED_SHA && actual === process.env.EXPECTED_TARGET) process.exit(0);
+            }
+            await new Promise((resolve) => setTimeout(resolve, 5_000));
+          }
+          throw new Error(`Active release identity did not converge to the authorized target; observed ${actual || "unavailable"}.`);
+          NODE
+
+      - name: Public live verification
+        run: npm run test:e2e:live:public
+        env:
+          NODE_OPTIONS: --use-system-ca
+
+      - name: Credentialed live verification
+        run: npm run test:e2e:live:roles
+        env:
+          NODE_OPTIONS: --use-system-ca
+          PLAYWRIGHT_REQUIRE_LIVE_CREDENTIALS: "true"
+          PLAYWRIGHT_USER_CUSTOMER_NUMBER: ${{ secrets.PLAYWRIGHT_USER_CUSTOMER_NUMBER }}
+          PLAYWRIGHT_USER_PASSWORD: ${{ secrets.PLAYWRIGHT_USER_PASSWORD }}
+          PLAYWRIGHT_USER_OTP_SECRET: ${{ secrets.PLAYWRIGHT_USER_OTP_SECRET }}
+          PLAYWRIGHT_OPERATOR_USER_ID: ${{ secrets.PLAYWRIGHT_OPERATOR_USER_ID }}
+          PLAYWRIGHT_OPERATOR_PASSWORD: ${{ secrets.PLAYWRIGHT_OPERATOR_PASSWORD }}
+          PLAYWRIGHT_DEPARTMENT_ID: ${{ secrets.PLAYWRIGHT_DEPARTMENT_ID }}
+
+      - name: Record verified server version
+        run: npm run release:update-server-version
+        env:
+          SERVER_UPDATE_TOKEN: ${{ secrets.SERVER_UPDATE_TOKEN }}
+          RELEASE_VERSION: ${{ inputs.source_sha }}
+          RELEASE_VERSION_UPDATE_REQUIRED: "true"
+
+      - name: Publish recovery audit
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: frontend-release-recovery-${{ inputs.source_sha }}-${{ github.run_id }}
+          path: |
+            test-results
+            playwright-report
+          if-no-files-found: ignore
+          retention-days: 30
+
+      - name: Restore pre-recovery target after downstream failure
+        if: >-
+          failure() && inputs.action == 'rollback'
+        shell: bash
+        run: |
+          node scripts/release/deploy-cpanel.mjs --rollback
+          node --input-type=module <<'NODE'
+          const deadline = Date.now() + 300_000;
+          while (Date.now() < deadline) {
+            const response = await fetch(new URL(`release-manifest.json?restore=${Date.now()}`, process.env.PRODUCTION_FRONTEND_URL), {
+              headers: { "Cache-Control": "no-cache", Pragma: "no-cache" },
+            });
+            if (response.ok) {
+              const manifest = await response.json();
+              const sha = String(manifest.commit_sha || "").toLowerCase();
+              const target = `releases/${sha}-${String(manifest.build_id || "")}/dist`;
+              if (sha === process.env.RELEASE_VERSION && target === process.env.RELEASE_ROLLBACK_TARGET) process.exit(0);
+            }
+            await new Promise((resolve) => setTimeout(resolve, 5_000));
+          }
+          throw new Error("Failed to restore and verify the pre-recovery target.");
+          NODE
+          npm run release:update-server-version
+        env:
+          NODE_OPTIONS: --use-system-ca
+          RELEASE_ROLLBACK_TARGET: ${{ steps.current.outputs.previous_target }}
+          RELEASE_VERSION: ${{ steps.current.outputs.previous_sha }}
+          RELEASE_VERSION_UPDATE_REQUIRED: "true"
+          PRODUCTION_FTP_HOST: ${{ secrets.PRODUCTION_FTP_HOST }}
+          PRODUCTION_FTP_USER: ${{ secrets.PRODUCTION_FTP_USER }}
+          PRODUCTION_FTP_PASSWORD: ${{ secrets.PRODUCTION_FTP_PASSWORD }}
+          PRODUCTION_FTP_PATH: ${{ secrets.PRODUCTION_FTP_PATH }}
+          PRODUCTION_ACTIVATION_KEY: ${{ secrets.PRODUCTION_ACTIVATION_KEY }}
+          PRODUCTION_FRONTEND_URL: ${{ vars.PRODUCTION_FRONTEND_URL || 'https://truckwash.io' }}
+          SERVER_UPDATE_TOKEN: ${{ secrets.SERVER_UPDATE_TOKEN }}
@@ -37,6 +37,7 @@ jobs:
      checksum_name: ${{ steps.package-names.outputs.checksum_name }}
      inventory_name: ${{ steps.package-names.outputs.inventory_name }}
      release_id: ${{ steps.package.outputs.release_id }}
+      archive_sha256: ${{ steps.package.outputs.archive_sha256 }}
    steps:
      - name: Check release commit is current
        id: branch-head
@@ -286,8 +287,28 @@ jobs:
        env:
          NODE_OPTIONS: --use-system-ca

-      - name: Credentialed live Playwright gate (when configured)
+      - name: Detect credentialed live gate configuration
        if: steps.branch-head.outputs.current == 'true'
+        id: credentialed_live_config
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [[ -n "$CUSTOMER_NUMBER" && -n "$CUSTOMER_PASSWORD" &&
+                -n "$OPERATOR_USER_ID" && -n "$OPERATOR_PASSWORD" ]]; then
+            echo "configured=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "configured=false" >> "$GITHUB_OUTPUT"
+          fi
+        env:
+          CUSTOMER_NUMBER: ${{ secrets.PLAYWRIGHT_USER_CUSTOMER_NUMBER }}
+          CUSTOMER_PASSWORD: ${{ secrets.PLAYWRIGHT_USER_PASSWORD }}
+          OPERATOR_USER_ID: ${{ secrets.PLAYWRIGHT_OPERATOR_USER_ID }}
+          OPERATOR_PASSWORD: ${{ secrets.PLAYWRIGHT_OPERATOR_PASSWORD }}
+
+      - name: Credentialed live Playwright gate (when configured)
+        if: >-
+          steps.branch-head.outputs.current == 'true' &&
+          steps.credentialed_live_config.outputs.configured == 'true'
        id: credentialed_live
        timeout-minutes: 15
        run: npm run test:e2e:live:roles
@@ -300,26 +321,9 @@ jobs:
          PLAYWRIGHT_OPERATOR_PASSWORD: ${{ secrets.PLAYWRIGHT_OPERATOR_PASSWORD }}
          PLAYWRIGHT_DEPARTMENT_ID: ${{ secrets.PLAYWRIGHT_DEPARTMENT_ID }}

-      - name: Roll back after live verification failure
-        if: >-
-          failure() && steps.branch-head.outputs.current == 'true' &&
-          steps.deploy.outcome == 'success' &&
-          (steps.public_live.outcome == 'failure' || steps.credentialed_live.outcome == 'failure')
-        timeout-minutes: 10
-        run: node scripts/release/deploy-cpanel.mjs --rollback
-        env:
-          NODE_OPTIONS: --use-system-ca
-          RELEASE_ROLLBACK_TARGET: ${{ steps.deploy.outputs.rollback_target }}
-          PRODUCTION_FTP_HOST: ${{ secrets.PRODUCTION_FTP_HOST }}
-          PRODUCTION_FTP_USER: ${{ secrets.PRODUCTION_FTP_USER }}
-          PRODUCTION_FTP_PASSWORD: ${{ secrets.PRODUCTION_FTP_PASSWORD }}
-          PRODUCTION_FTP_PATH: ${{ secrets.PRODUCTION_FTP_PATH }}
-          PRODUCTION_ACTIVATION_KEY: ${{ secrets.PRODUCTION_ACTIVATION_KEY }}
-          PRODUCTION_FRONTEND_URL: ${{ vars.PRODUCTION_FRONTEND_URL || 'https://truckwash.io' }}
-
      - name: Record Release Manager gate
+        id: release_manager
        if: steps.branch-head.outputs.current == 'true'
-        continue-on-error: true
        timeout-minutes: 5
        run: |
          test -n "$RELEASE_MANAGER_GATE_TOKEN" || (echo "RELEASE_MANAGER_GATE_TOKEN is required" >&2; exit 1)
@@ -341,6 +345,7 @@ jobs:
        env:
          SERVER_UPDATE_TOKEN: ${{ secrets.SERVER_UPDATE_TOKEN }}
          RELEASE_VERSION: ${{ github.event.workflow_run.head_sha }}
+          RELEASE_VERSION_UPDATE_REQUIRED: "true"

      - name: Create verified frontend release proof
        if: steps.branch-head.outputs.current == 'true'
@@ -355,24 +360,52 @@ jobs:
            if (!process.env[name]) throw new Error(`Missing ${name}`);
            return process.env[name];
          };
+          const requireSuccessfulStep = (name) => {
+            const outcome = required(name);
+            if (outcome !== "success") throw new Error(`${name} did not succeed: ${outcome}`);
+            return "passed";
+          };
+          const credentialedGate = () => {
+            const configured = required("LIVE_CREDENTIALED_GATE_CONFIGURED");
+            if (configured === "false") return "not-configured";
+            if (configured !== "true") {
+              throw new Error(`Invalid LIVE_CREDENTIALED_GATE_CONFIGURED: ${configured}`);
+            }
+            return requireSuccessfulStep("LIVE_CREDENTIALED_GATE_OUTCOME");
+          };
          const proof = {
-            schemaVersion: 1,
+            schemaVersion: 2,
+            releaseId: required("RELEASE_ID"),
+            sha: required("RELEASE_COMMIT_SHA").toLowerCase(),
+            archiveSha256: required("RELEASE_ARCHIVE_SHA256").toLowerCase(),
+            activeTarget: required("RELEASE_ACTIVE_TARGET"),
+            rollbackTarget: process.env.RELEASE_ROLLBACK_TARGET || null,
+            verificationState: "verified",
+            observedAt: new Date().toISOString(),
            repository: required("GITHUB_REPOSITORY"),
            sourceSha: required("RELEASE_COMMIT_SHA").toLowerCase(),
            testedWorkflowRunId: required("TESTED_WORKFLOW_RUN_ID"),
            frontendReleaseRunId: required("GITHUB_RUN_ID"),
            frontendReleaseRunAttempt: required("GITHUB_RUN_ATTEMPT"),
            buildId: required("RELEASE_BUILD_ID"),
-            livePublicGate: "passed",
-            liveCredentialedGate: "passed",
-            releaseManagerGate: "passed",
+            livePublicGate: requireSuccessfulStep("LIVE_PUBLIC_GATE_OUTCOME"),
+            liveCredentialedGate: credentialedGate(),
+            releaseManagerGate: requireSuccessfulStep("RELEASE_MANAGER_GATE_OUTCOME"),
            serverVersionUpdated: true,
+            serverVersionReadBack: "passed",
            completedAt: new Date().toISOString(),
          };
          writeFileSync(process.env.PROOF_PATH, `${JSON.stringify(proof, null, 2)}\n`, { mode: 0o600 });
          NODE
        env:
          TESTED_WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
+          RELEASE_ARCHIVE_SHA256: ${{ needs.build-release.outputs.archive_sha256 }}
+          RELEASE_ACTIVE_TARGET: ${{ steps.deploy.outputs.active_target }}
+          RELEASE_ROLLBACK_TARGET: ${{ steps.deploy.outputs.rollback_target }}
+          LIVE_PUBLIC_GATE_OUTCOME: ${{ steps.public_live.outcome }}
+          LIVE_CREDENTIALED_GATE_CONFIGURED: ${{ steps.credentialed_live_config.outputs.configured }}
+          LIVE_CREDENTIALED_GATE_OUTCOME: ${{ steps.credentialed_live.outcome }}
+          RELEASE_MANAGER_GATE_OUTCOME: ${{ steps.release_manager.outcome }}

      - name: Publish verified frontend release proof
        if: steps.branch-head.outputs.current == 'true'
@@ -383,6 +416,30 @@ jobs:
          if-no-files-found: error
          retention-days: 30

+      - name: Roll back after any post-deployment verification failure
+        if: >-
+          failure() && steps.branch-head.outputs.current == 'true' &&
+          steps.deploy.outcome == 'success'
+        timeout-minutes: 10
+        shell: bash
+        run: |
+          set -euo pipefail
+          node scripts/release/deploy-cpanel.mjs --rollback
+          [[ "$RELEASE_ROLLBACK_TARGET" =~ ^releases/([a-f0-9]{40})-[A-Za-z0-9._-]+/dist$ ]]
+          export RELEASE_VERSION="${BASH_REMATCH[1]}"
+          npm run release:update-server-version
+        env:
+          NODE_OPTIONS: --use-system-ca
+          RELEASE_ROLLBACK_TARGET: ${{ steps.deploy.outputs.rollback_target }}
+          PRODUCTION_FTP_HOST: ${{ secrets.PRODUCTION_FTP_HOST }}
+          PRODUCTION_FTP_USER: ${{ secrets.PRODUCTION_FTP_USER }}
+          PRODUCTION_FTP_PASSWORD: ${{ secrets.PRODUCTION_FTP_PASSWORD }}
+          PRODUCTION_FTP_PATH: ${{ secrets.PRODUCTION_FTP_PATH }}
+          PRODUCTION_ACTIVATION_KEY: ${{ secrets.PRODUCTION_ACTIVATION_KEY }}
+          PRODUCTION_FRONTEND_URL: ${{ vars.PRODUCTION_FRONTEND_URL || 'https://truckwash.io' }}
+          SERVER_UPDATE_TOKEN: ${{ secrets.SERVER_UPDATE_TOKEN }}
+          RELEASE_VERSION_UPDATE_REQUIRED: "true"
+
      - name: Upload Playwright report
        if: failure() && steps.branch-head.outputs.current == 'true'
        continue-on-error: true
@@ -1,4 +1,5 @@
 import { execSync } from "node:child_process";
+import { fileURLToPath } from "node:url";

 function gitCommit() {
  try {
@@ -8,9 +9,30 @@ function gitCommit() {
  }
 }

-async function main() {
-  const token = process.env.SERVER_UPDATE_TOKEN;
-  const required = process.env.RELEASE_VERSION_UPDATE_REQUIRED === "true";
+async function boundedJson(response, limit = 64 * 1024) {
+  const declared = Number(response.headers.get("content-length"));
+  if (Number.isFinite(declared) && declared > limit) throw new Error("Server version response is too large.");
+  const reader = response.body?.getReader();
+  const chunks = [];
+  let bytes = 0;
+  if (reader) {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      bytes += value.byteLength;
+      if (bytes > limit) {
+        await reader.cancel().catch(() => {});
+        throw new Error("Server version response is too large.");
+      }
+      chunks.push(Buffer.from(value));
+    }
+  }
+  return JSON.parse(Buffer.concat(chunks).toString("utf8"));
+}
+
+export async function updateServerVersion(env = process.env, fetchImpl = globalThis.fetch) {
+  const token = env.SERVER_UPDATE_TOKEN;
+  const required = env.RELEASE_VERSION_UPDATE_REQUIRED === "true";
  if (!token) {
    if (required) {
      throw new Error("SERVER_UPDATE_TOKEN is required after deploy verification.");
@@ -19,32 +41,70 @@ async function main() {
    return;
  }

-  const version = process.env.RELEASE_VERSION || process.env.RELEASE_EXPECTED_COMMIT || process.env.GITHUB_SHA || gitCommit();
-  if (!version) {
-    throw new Error("Could not determine release version for server update.");
+  const version = String(env.RELEASE_VERSION || env.RELEASE_EXPECTED_COMMIT || env.GITHUB_SHA || gitCommit())
+    .toLowerCase();
+  if (!/^[a-f0-9]{40}$/.test(version)) {
+    throw new Error("Server release version must be a full lowercase commit SHA.");
  }

-  const baseUrl = process.env.SERVER_UPDATE_URL || "https://api-v2.truckwash.io/master/api/worker/update-version";
+  const baseUrl = env.SERVER_UPDATE_URL || "https://api-v2.truckwash.io/master/api/worker/update-version";
  const url = new URL(baseUrl);
+  if (url.protocol !== "https:" || url.username || url.password) {
+    throw new Error("SERVER_UPDATE_URL must be an HTTPS URL without embedded credentials.");
+  }
  url.searchParams.set("version", version);

-  const response = await fetch(url, {
+  const response = await fetchImpl(url, {
    method: "GET",
+    redirect: "manual",
+    signal: AbortSignal.timeout(30_000),
    headers: {
      Authorization: `Bearer ${token}`,
      "Cache-Control": "no-cache",
    },
  });

-  const body = await response.text();
+  const body = await boundedJson(response).catch(() => ({}));
  if (!response.ok) {
-    throw new Error(`Server version update failed with HTTP ${response.status}: ${body}`);
+    throw new Error(`Server version update failed with HTTP ${response.status}.`);
  }

+  const readUrl = new URL(env.SERVER_VERSION_READ_URL
+    || "https://api-v2.truckwash.io/master/api/worker/version");
+  if (readUrl.protocol !== "https:" || readUrl.origin !== url.origin
+    || readUrl.username || readUrl.password || readUrl.search || readUrl.hash) {
+    throw new Error("SERVER_VERSION_READ_URL must be an exact HTTPS URL on the update origin.");
+  }
+  readUrl.searchParams.set("verify", `${Date.now()}`);
+  const verification = await fetchImpl(readUrl, {
+    method: "GET",
+    redirect: "manual",
+    signal: AbortSignal.timeout(30_000),
+    headers: {
+      Authorization: `Bearer ${token}`,
+      "Cache-Control": "no-cache",
+      Pragma: "no-cache",
+    },
+  });
+  if (!verification.ok) {
+    throw new Error(`Server version read-back failed with HTTP ${verification.status}.`);
+  }
+  const verified = await boundedJson(verification);
+  const observed = String(verified?.data?.version ?? verified?.version ?? "").toLowerCase();
+  if (observed !== version) {
+    throw new Error(`Server version read-back did not match ${version}.`);
+  }
  console.log(`Server version updated to ${version}.`);
+  return { version, update: body, observed };
 }

-main().catch((error) => {
-  console.error(error instanceof Error ? error.stack || error.message : error);
-  process.exit(1);
-});
+async function main() {
+  await updateServerVersion();
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+  main().catch((error) => {
+    console.error(error instanceof Error ? error.stack || error.message : error);
+    process.exit(1);
+  });
+}
@@ -23,6 +23,7 @@ import {
  targetRelativeToRoot,
  validateReleaseTarget,
 } from "../../scripts/release/cpanel-deploy-lib.mjs";
+import { updateServerVersion } from "../../scripts/release/update-server-version.mjs";

 const temporaryDirectories = [];
 const COMMIT_SHA = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
@@ -64,6 +65,89 @@ afterEach(async () => {
  );
 });

+describe("Control Plane release evidence", () => {
+  it("publishes V2 exact-release identity after the live gates", async () => {
+    const workflow = await fs.readFile(path.resolve(".github/workflows/release.yml"), "utf8");
+    expect(workflow).toContain("schemaVersion: 2");
+    expect(workflow).toContain('releaseId: required("RELEASE_ID")');
+    expect(workflow).toContain('sha: required("RELEASE_COMMIT_SHA").toLowerCase()');
+    expect(workflow).toContain('archiveSha256: required("RELEASE_ARCHIVE_SHA256").toLowerCase()');
+    expect(workflow).toContain('activeTarget: required("RELEASE_ACTIVE_TARGET")');
+    expect(workflow).toContain("rollbackTarget:");
+    expect(workflow).toContain('verificationState: "verified"');
+    expect(workflow).toContain('releaseManagerGate: requireSuccessfulStep("RELEASE_MANAGER_GATE_OUTCOME")');
+    expect(workflow).toContain('serverVersionReadBack: "passed"');
+    expect(workflow).toContain('if (configured === "false") return "not-configured"');
+    expect(workflow).toContain("steps.credentialed_live_config.outputs.configured == 'true'");
+    expect(workflow).toContain("Roll back after any post-deployment verification failure");
+    expect(workflow).toContain('RELEASE_VERSION_UPDATE_REQUIRED: "true"');
+    expect(workflow).toContain("npm run release:update-server-version");
+    expect(workflow).not.toContain("continue-on-error: true\n        timeout-minutes: 5");
+    expect(workflow.indexOf("Roll back after any post-deployment verification failure")).toBeGreaterThan(
+      workflow.indexOf("Publish verified frontend release proof")
+    );
+    expect(workflow.indexOf("Create verified frontend release proof")).toBeGreaterThan(
+      workflow.indexOf("Credentialed live Playwright gate")
+    );
+    const testflight = await fs.readFile(path.resolve(".github/workflows/ios-testflight.yml"), "utf8");
+    expect(testflight).toContain("proof.schemaVersion === 2");
+    expect(testflight).toContain("proof.sha === process.env.IOS_SOURCE_SHA");
+    expect(testflight).toContain('proof.verificationState === "verified"');
+    expect(testflight).toContain('proof.serverVersionReadBack === "passed"');
+    expect(testflight).toContain('["passed", "not-configured"].includes(proof.liveCredentialedGate)');
+  });
+
+  it("keeps rollback and re-verification behind the protected production environment", async () => {
+    const recovery = await fs.readFile(path.resolve(".github/workflows/release-recovery.yml"), "utf8");
+    expect(recovery).toContain("environment: frontend-production");
+    expect(recovery).toContain("group: frontend-production");
+    expect(recovery).toContain("Authorize source from successful release proof");
+    expect(recovery).toContain("proof.frontendReleaseRunId === process.env.RELEASE_RUN_ID");
+    expect(recovery).toContain("Rollback target does not match the verified release proof");
+    expect(recovery).toContain("Verify active manifest matches authorized release");
+    expect(recovery).toContain("Restore pre-recovery target after downstream failure");
+    expect(recovery).toContain("node scripts/release/deploy-cpanel.mjs --rollback");
+    expect(recovery).toContain('PLAYWRIGHT_REQUIRE_LIVE_CREDENTIALS: "true"');
+    expect(recovery).toContain("RELEASE_VERSION: ${{ inputs.source_sha }}");
+    expect(recovery).toContain("PRODUCTION_FTP_HOST: ${{ secrets.PRODUCTION_FTP_HOST }}");
+  });
+});
+
+describe("server version release proof", () => {
+  it("requires a token and independently reads back the exact full SHA", async () => {
+    const version = "a".repeat(40);
+    const calls = [];
+    const fetchImpl = vi.fn(async (url) => {
+      calls.push(String(url));
+      return calls.length === 1
+        ? new Response(JSON.stringify({ data: { accepted: true } }))
+        : new Response(JSON.stringify({ data: { version } }));
+    });
+    const result = await updateServerVersion(
+      {
+        SERVER_UPDATE_TOKEN: "test-token",
+        RELEASE_VERSION_UPDATE_REQUIRED: "true",
+        RELEASE_VERSION: version,
+        SERVER_UPDATE_URL: "https://api.example.test/worker/update-version",
+        SERVER_VERSION_READ_URL: "https://api.example.test/worker/version",
+      },
+      fetchImpl
+    );
+    expect(result.observed).toBe(version);
+    expect(calls).toHaveLength(2);
+    expect(calls[1]).toContain("/worker/version?verify=");
+    await expect(
+      updateServerVersion(
+        {
+          RELEASE_VERSION_UPDATE_REQUIRED: "true",
+          RELEASE_VERSION: version,
+        },
+        fetchImpl
+      )
+    ).rejects.toThrow("SERVER_UPDATE_TOKEN is required");
+  });
+});
+
 describe("cPanel deployment configuration", () => {
  it("validates every credential without putting values in errors", () => {
    const secret = "never-print-this-token";