Jeppe B
d393c8c175
## Summary - fall back to the existing scoped `RELEASE_MANAGER_GATE_TOKEN` when `SERVER_UPDATE_TOKEN` is absent - record the exact frontend SHA through the release-gate endpoint, then independently read it back - preserve the legacy dedicated-token path when it is configured - carry the scoped credential and exact run-attempt build ID through normal releases, rollback recovery, and restore-on-failure ## Dependency Depends on backend PR copenhagentruckwash/api#342 being merged and deployed before this PR is merged. ## Verification - focused release-gate updater test: 1 passed - direct exact-SHA update/readback execution passed - ESLint passed for changed JavaScript/tests - Prettier passed for both workflows and changed JavaScript/tests - Node syntax and `git diff --check` passed The existing broader cPanel release test is also updated; the local cached dependency set cannot collect that file because `jszip` is absent, so protected CI remains the full-suite authority.
284 lines
13 KiB
YAML
284 lines
13 KiB
YAML
name: Frontend Release Recovery
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
action:
|
|
description: Verify the active release or roll back before verification
|
|
required: true
|
|
type: choice
|
|
options:
|
|
- reverify
|
|
- rollback
|
|
source_sha:
|
|
description: Exact 40-character commit SHA expected after recovery
|
|
required: true
|
|
type: string
|
|
rollback_target:
|
|
description: Immutable releases/.../dist target; required for rollback
|
|
required: false
|
|
type: string
|
|
|
|
permissions:
|
|
contents: read
|
|
actions: read
|
|
|
|
concurrency:
|
|
group: frontend-production
|
|
cancel-in-progress: false
|
|
|
|
jobs:
|
|
recover:
|
|
name: Protected production recovery
|
|
runs-on: ubuntu-latest
|
|
environment: frontend-production
|
|
timeout-minutes: 35
|
|
env:
|
|
PLAYWRIGHT_BASE_URL: ${{ vars.PRODUCTION_FRONTEND_URL || 'https://truckwash.io' }}
|
|
steps:
|
|
- name: Validate exact recovery target
|
|
shell: bash
|
|
env:
|
|
RECOVERY_ACTION: ${{ inputs.action }}
|
|
RECOVERY_SHA: ${{ inputs.source_sha }}
|
|
RECOVERY_TARGET: ${{ inputs.rollback_target }}
|
|
run: |
|
|
set -euo pipefail
|
|
[[ "$RECOVERY_SHA" =~ ^[a-f0-9]{40}$ ]]
|
|
if [[ "$RECOVERY_ACTION" == "rollback" ]]; then
|
|
[[ "$RECOVERY_TARGET" =~ ^releases/[A-Za-z0-9._-]+/dist$ ]]
|
|
else
|
|
[[ -z "$RECOVERY_TARGET" ]]
|
|
fi
|
|
|
|
- name: Checkout exact recovery source
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
ref: ${{ inputs.source_sha }}
|
|
|
|
- name: Authorize source from successful release proof
|
|
id: authorize
|
|
shell: bash
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
RECOVERY_ACTION: ${{ inputs.action }}
|
|
RECOVERY_SHA: ${{ inputs.source_sha }}
|
|
RECOVERY_TARGET: ${{ inputs.rollback_target }}
|
|
run: |
|
|
set -euo pipefail
|
|
runs="$RUNNER_TEMP/recovery-runs.json"
|
|
artifacts="$RUNNER_TEMP/recovery-artifacts.json"
|
|
curl --fail --silent --show-error \
|
|
-H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
|
|
"$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/workflows/release.yml/runs?head_sha=$RECOVERY_SHA&status=success&per_page=20" \
|
|
> "$runs"
|
|
release_run_id="$(jq -r '[.workflow_runs[] | select(.event == "workflow_run")] | first | .id // empty' "$runs")"
|
|
[[ "$release_run_id" =~ ^[0-9]+$ ]]
|
|
artifact_name="frontend-release-proof-$RECOVERY_SHA"
|
|
curl --fail --silent --show-error \
|
|
-H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
|
|
"$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/runs/$release_run_id/artifacts?name=$artifact_name&per_page=20" \
|
|
> "$artifacts"
|
|
artifact_id="$(jq -r '[.artifacts[] | select(.expired == false)] | first | .id // empty' "$artifacts")"
|
|
[[ "$artifact_id" =~ ^[0-9]+$ ]]
|
|
mkdir -p "$RUNNER_TEMP/recovery-proof"
|
|
curl --fail --silent --show-error --location \
|
|
-H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
|
|
"$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" \
|
|
-o "$RUNNER_TEMP/recovery-proof.zip"
|
|
unzip -q "$RUNNER_TEMP/recovery-proof.zip" -d "$RUNNER_TEMP/recovery-proof"
|
|
PROOF_PATH="$RUNNER_TEMP/recovery-proof/frontend-release-proof.json" \
|
|
RELEASE_RUN_ID="$release_run_id" node <<'NODE'
|
|
const { appendFileSync, readFileSync } = require("node:fs");
|
|
const proof = JSON.parse(readFileSync(process.env.PROOF_PATH, "utf8"));
|
|
const sha = process.env.RECOVERY_SHA;
|
|
const target = process.env.RECOVERY_TARGET;
|
|
const expectedPrefix = `releases/${sha}-`;
|
|
const valid = proof.schemaVersion === 2
|
|
&& proof.repository === process.env.GITHUB_REPOSITORY
|
|
&& proof.sha === sha
|
|
&& proof.sourceSha === sha
|
|
&& proof.frontendReleaseRunId === process.env.RELEASE_RUN_ID
|
|
&& proof.verificationState === "verified"
|
|
&& proof.livePublicGate === "passed"
|
|
&& ["passed", "not-configured"].includes(proof.liveCredentialedGate)
|
|
&& proof.releaseManagerGate === "passed"
|
|
&& proof.serverVersionUpdated === true
|
|
&& proof.serverVersionReadBack === "passed"
|
|
&& /^[1-9][0-9]*-[1-9][0-9]*$/.test(String(proof.buildId || ""))
|
|
&& typeof proof.activeTarget === "string"
|
|
&& proof.activeTarget.startsWith(expectedPrefix)
|
|
&& proof.activeTarget.endsWith("/dist");
|
|
if (!valid) throw new Error("Recovery source does not have valid exact-release proof.");
|
|
if (process.env.RECOVERY_ACTION === "rollback" && target !== proof.activeTarget) {
|
|
throw new Error("Rollback target does not match the verified release proof.");
|
|
}
|
|
appendFileSync(process.env.GITHUB_OUTPUT, `verified_target=${proof.activeTarget}\n`);
|
|
appendFileSync(process.env.GITHUB_OUTPUT, `build_id=${proof.buildId}\n`);
|
|
NODE
|
|
|
|
- name: Capture current immutable target
|
|
id: current
|
|
shell: bash
|
|
env:
|
|
FRONTEND_URL: ${{ vars.PRODUCTION_FRONTEND_URL || 'https://truckwash.io' }}
|
|
run: |
|
|
node --input-type=module <<'NODE'
|
|
import { appendFileSync } from "node:fs";
|
|
const response = await fetch(new URL(`release-manifest.json?recovery=${Date.now()}`, process.env.FRONTEND_URL), {
|
|
headers: { "Cache-Control": "no-cache", Pragma: "no-cache" },
|
|
});
|
|
if (!response.ok) throw new Error(`Active manifest returned HTTP ${response.status}.`);
|
|
const manifest = await response.json();
|
|
const sha = String(manifest.commit_sha || "").toLowerCase();
|
|
const build = String(manifest.build_id || "");
|
|
if (!/^[a-f0-9]{40}$/.test(sha) || !/^[A-Za-z0-9._-]{1,180}$/.test(build)) {
|
|
throw new Error("Active manifest has invalid release identity.");
|
|
}
|
|
if (!/^[1-9][0-9]*-[1-9][0-9]*$/.test(build)) {
|
|
throw new Error("Active manifest build id is not a release run identity.");
|
|
}
|
|
appendFileSync(process.env.GITHUB_OUTPUT, `previous_sha=${sha}\nprevious_build_id=${build}\nprevious_target=releases/${sha}-${build}/dist\n`);
|
|
NODE
|
|
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
|
with:
|
|
node-version: 22
|
|
cache: npm
|
|
|
|
- name: Install dependencies
|
|
run: npm ci --legacy-peer-deps
|
|
|
|
- name: Install secure FTP client without system changes
|
|
run: |
|
|
if command -v lftp >/dev/null 2>&1; then
|
|
exit 0
|
|
fi
|
|
package_root="$RUNNER_TEMP/lftp-package"
|
|
mkdir -p "$package_root"
|
|
(
|
|
cd "$package_root"
|
|
apt-get download lftp
|
|
dpkg-deb --extract ./lftp_*.deb root
|
|
)
|
|
echo "$package_root/root/usr/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Install Playwright Chromium
|
|
run: node scripts/install-playwright-browsers.mjs chromium
|
|
|
|
- name: Roll back atomically
|
|
if: inputs.action == 'rollback'
|
|
id: rollback
|
|
run: node scripts/release/deploy-cpanel.mjs --rollback
|
|
env:
|
|
NODE_OPTIONS: --use-system-ca
|
|
RELEASE_ROLLBACK_TARGET: ${{ inputs.rollback_target }}
|
|
PRODUCTION_FTP_HOST: ${{ secrets.PRODUCTION_FTP_HOST }}
|
|
PRODUCTION_FTP_USER: ${{ secrets.PRODUCTION_FTP_USER }}
|
|
PRODUCTION_FTP_PASSWORD: ${{ secrets.PRODUCTION_FTP_PASSWORD }}
|
|
PRODUCTION_FTP_PATH: ${{ secrets.PRODUCTION_FTP_PATH }}
|
|
PRODUCTION_ACTIVATION_KEY: ${{ secrets.PRODUCTION_ACTIVATION_KEY }}
|
|
PRODUCTION_FRONTEND_URL: ${{ vars.PRODUCTION_FRONTEND_URL || 'https://truckwash.io' }}
|
|
|
|
- name: Verify active manifest matches authorized release
|
|
shell: bash
|
|
env:
|
|
EXPECTED_SHA: ${{ inputs.source_sha }}
|
|
EXPECTED_TARGET: ${{ steps.authorize.outputs.verified_target }}
|
|
FRONTEND_URL: ${{ vars.PRODUCTION_FRONTEND_URL || 'https://truckwash.io' }}
|
|
run: |
|
|
node --input-type=module <<'NODE'
|
|
const deadline = Date.now() + 300_000;
|
|
let actual = "";
|
|
while (Date.now() < deadline) {
|
|
const response = await fetch(new URL(`release-manifest.json?recovery=${Date.now()}`, process.env.FRONTEND_URL), {
|
|
headers: { "Cache-Control": "no-cache", Pragma: "no-cache" },
|
|
});
|
|
if (response.ok) {
|
|
const manifest = await response.json();
|
|
const manifestSha = String(manifest.commit_sha || "").toLowerCase();
|
|
actual = `releases/${manifestSha}-${String(manifest.build_id || "")}/dist`;
|
|
if (manifestSha === process.env.EXPECTED_SHA && actual === process.env.EXPECTED_TARGET) process.exit(0);
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 5_000));
|
|
}
|
|
throw new Error(`Active release identity did not converge to the authorized target; observed ${actual || "unavailable"}.`);
|
|
NODE
|
|
|
|
- name: Public live verification
|
|
run: npm run test:e2e:live:public
|
|
env:
|
|
NODE_OPTIONS: --use-system-ca
|
|
|
|
- name: Credentialed live verification
|
|
run: npm run test:e2e:live:roles
|
|
env:
|
|
NODE_OPTIONS: --use-system-ca
|
|
PLAYWRIGHT_REQUIRE_LIVE_CREDENTIALS: "true"
|
|
PLAYWRIGHT_USER_CUSTOMER_NUMBER: ${{ secrets.PLAYWRIGHT_USER_CUSTOMER_NUMBER }}
|
|
PLAYWRIGHT_USER_PASSWORD: ${{ secrets.PLAYWRIGHT_USER_PASSWORD }}
|
|
PLAYWRIGHT_USER_OTP_SECRET: ${{ secrets.PLAYWRIGHT_USER_OTP_SECRET }}
|
|
PLAYWRIGHT_OPERATOR_USER_ID: ${{ secrets.PLAYWRIGHT_OPERATOR_USER_ID }}
|
|
PLAYWRIGHT_OPERATOR_PASSWORD: ${{ secrets.PLAYWRIGHT_OPERATOR_PASSWORD }}
|
|
PLAYWRIGHT_DEPARTMENT_ID: ${{ secrets.PLAYWRIGHT_DEPARTMENT_ID }}
|
|
|
|
- name: Record verified server version
|
|
run: npm run release:update-server-version
|
|
env:
|
|
SERVER_UPDATE_TOKEN: ${{ secrets.SERVER_UPDATE_TOKEN }}
|
|
RELEASE_MANAGER_GATE_TOKEN: ${{ secrets.RELEASE_MANAGER_GATE_TOKEN }}
|
|
RELEASE_VERSION: ${{ inputs.source_sha }}
|
|
RELEASE_BUILD_ID: ${{ steps.authorize.outputs.build_id }}
|
|
RELEASE_VERSION_UPDATE_REQUIRED: "true"
|
|
|
|
- name: Publish recovery audit
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: frontend-release-recovery-${{ inputs.source_sha }}-${{ github.run_id }}
|
|
path: |
|
|
test-results
|
|
playwright-report
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
- name: Restore pre-recovery target after downstream failure
|
|
if: >-
|
|
failure() && inputs.action == 'rollback'
|
|
shell: bash
|
|
run: |
|
|
node scripts/release/deploy-cpanel.mjs --rollback
|
|
node --input-type=module <<'NODE'
|
|
const deadline = Date.now() + 300_000;
|
|
while (Date.now() < deadline) {
|
|
const response = await fetch(new URL(`release-manifest.json?restore=${Date.now()}`, process.env.PRODUCTION_FRONTEND_URL), {
|
|
headers: { "Cache-Control": "no-cache", Pragma: "no-cache" },
|
|
});
|
|
if (response.ok) {
|
|
const manifest = await response.json();
|
|
const sha = String(manifest.commit_sha || "").toLowerCase();
|
|
const target = `releases/${sha}-${String(manifest.build_id || "")}/dist`;
|
|
if (sha === process.env.RELEASE_VERSION && target === process.env.RELEASE_ROLLBACK_TARGET) process.exit(0);
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 5_000));
|
|
}
|
|
throw new Error("Failed to restore and verify the pre-recovery target.");
|
|
NODE
|
|
npm run release:update-server-version
|
|
env:
|
|
NODE_OPTIONS: --use-system-ca
|
|
RELEASE_ROLLBACK_TARGET: ${{ steps.current.outputs.previous_target }}
|
|
RELEASE_VERSION: ${{ steps.current.outputs.previous_sha }}
|
|
RELEASE_BUILD_ID: ${{ steps.current.outputs.previous_build_id }}
|
|
RELEASE_VERSION_UPDATE_REQUIRED: "true"
|
|
PRODUCTION_FTP_HOST: ${{ secrets.PRODUCTION_FTP_HOST }}
|
|
PRODUCTION_FTP_USER: ${{ secrets.PRODUCTION_FTP_USER }}
|
|
PRODUCTION_FTP_PASSWORD: ${{ secrets.PRODUCTION_FTP_PASSWORD }}
|
|
PRODUCTION_FTP_PATH: ${{ secrets.PRODUCTION_FTP_PATH }}
|
|
PRODUCTION_ACTIVATION_KEY: ${{ secrets.PRODUCTION_ACTIVATION_KEY }}
|
|
PRODUCTION_FRONTEND_URL: ${{ vars.PRODUCTION_FRONTEND_URL || 'https://truckwash.io' }}
|
|
SERVER_UPDATE_TOKEN: ${{ secrets.SERVER_UPDATE_TOKEN }}
|
|
RELEASE_MANAGER_GATE_TOKEN: ${{ secrets.RELEASE_MANAGER_GATE_TOKEN }}
|