## Problem
Production was returning:
```json
{"success":false,"data":{"message":"Internal server error: Unknown column 'invoice_email' in 'SELECT'"}}
```
when authenticating as a superuser. The migration that adds
`users.invoice_email` was merged to master in api#381 but never applied
to the production database.
## Fix
- New `GET /api/admin/schema-check` endpoint — returns 503 with explicit
list of missing columns if any are absent (instead of a generic 500)
- `scripts/run-schema-bootstraps.php` — auto-discovers and runs every
`*_schema_bootstrap` class on the live database (additive, idempotent)
- `scripts/schema-health-check.php` — CLI tool for the same check, used
by deploy pipelines
- New Pest contract test `SchemaHealthCheckTest` — verifies the test DB
has every required users column and the schema-check endpoint works
- `deploy.yml`: pre-deploy step runs the bootstrap runner, smoke test
also runs the schema check, Slack alert on failure
## What this prevents
- Future migrations being merged without being applied to production
- Silent failures (generic 500) when a column is missing
- Repeated manual investigation of the same root cause
Refs: TRU-77 (the original bug), api#381 (the original PR)
---------
Co-authored-by: Jeppe B <jeppe@copenhagentruckwash.io>
Co-authored-by: bugfix-subagent <bugfix-subagent@truckwash.local>
Co-authored-by: OpenClaw Bugfix Agent <bugfix@openclaw.local>
168 lines
6.3 KiB
YAML
168 lines
6.3 KiB
YAML
name: Deploy to Hetzner (staging)
|
|
|
|
on:
|
|
push:
|
|
branches: [master]
|
|
workflow_dispatch:
|
|
inputs:
|
|
reason:
|
|
description: 'Reason for manual deploy'
|
|
required: false
|
|
default: 'manual'
|
|
|
|
concurrency:
|
|
group: deploy-${{ github.repository }}
|
|
cancel-in-progress: false
|
|
|
|
env:
|
|
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
|
|
DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
|
|
|
|
jobs:
|
|
test-and-deploy:
|
|
name: CI + Deploy
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 20
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 1
|
|
|
|
- name: Show commit info
|
|
run: |
|
|
echo "Repo: ${{ github.repository }}"
|
|
echo "Branch: ${{ github.ref }}"
|
|
echo "Commit: ${{ github.sha }}"
|
|
echo "Actor: ${{ github.actor }}"
|
|
|
|
# === CI (phpunit / vitest) runs here via repo's existing CI config ===
|
|
# (Most of our repos already have a "Required CI" check; this section
|
|
# would invoke that. If your repo doesn't have a CI workflow, the
|
|
# required-check on the branch will block this workflow's deploy step.)
|
|
|
|
- name: Setup SSH
|
|
uses: webfactory/ssh-agent@v0.9.0
|
|
with:
|
|
ssh-private-key: ${{ secrets.DEPLOY_SSH_KEY }}
|
|
|
|
- name: Add host key
|
|
run: |
|
|
mkdir -p ~/.ssh
|
|
ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null
|
|
|
|
- name: Pre-deploy snapshot
|
|
id: pre
|
|
run: |
|
|
ssh "$DEPLOY_USER@$DEPLOY_HOST" '
|
|
set -e
|
|
cd /opt/${{ github.event.repository.name }}
|
|
git rev-parse HEAD > /tmp/last_deploy_sha
|
|
echo "PRE_SHA=$(cat /tmp/last_deploy_sha)"
|
|
echo "pre_sha=$(cat /tmp/last_deploy_sha)" >> $GITHUB_OUTPUT
|
|
'
|
|
|
|
- name: Deploy
|
|
id: deploy
|
|
run: |
|
|
ssh "$DEPLOY_USER@$DEPLOY_HOST" '
|
|
set -e
|
|
cd /opt/${{ github.event.repository.name }}
|
|
git fetch origin master
|
|
git reset --hard origin/master
|
|
# PHP repos: composer install + clear cache
|
|
if [ -f composer.json ]; then
|
|
composer install --no-dev --optimize-autoloader --no-interaction
|
|
php artisan cache:clear || true
|
|
php artisan config:cache || true
|
|
# Restart php-fpm if used
|
|
sudo systemctl reload php8.2-fpm || true
|
|
fi
|
|
# Node repos: npm ci + build
|
|
if [ -f package.json ]; then
|
|
npm ci --ignore-scripts
|
|
npm run build
|
|
# Restart node service
|
|
sudo systemctl reload pleno-vue || sudo systemctl reload nginx || true
|
|
fi
|
|
# Restart generic services
|
|
sudo systemctl reload nginx || true
|
|
echo "Deploy complete: $(git rev-parse --short HEAD)"
|
|
'
|
|
|
|
- name: Pre-deploy schema check (run all *_schema_bootstrap)
|
|
id: pre_schema
|
|
run: |
|
|
echo "Running schema bootstraps against the live database…"
|
|
# Idempotent — adds missing columns, never drops anything.
|
|
# Catches the "Unknown column 'invoice_email' in 'SELECT'"
|
|
# production failure mode (TRU-77) where migrations were
|
|
# merged to master but never applied to the live DB.
|
|
php scripts/run-schema-bootstraps.php
|
|
echo "Schema bootstraps complete."
|
|
|
|
- name: Alert Slack if schema-check fails (pre-deploy)
|
|
if: failure()
|
|
run: |
|
|
php scripts/schema-health-check.php > /tmp/schema.json 2>&1 || true
|
|
msg=$(jq -r '"Schema health FAILED on '$SMOKE_BASE_URL'\nMissing: " + (.missing | join(", "))' /tmp/schema.json 2>/dev/null || echo "Schema check produced no JSON")
|
|
curl -sS -X POST -H "Authorization: Bearer $SLACK_BOT_TOKEN" \
|
|
-H "Content-Type: application/json; charset=utf-8" \
|
|
https://slack.com/api/chat.postMessage \
|
|
-d "{\"channel\":\"$AI_DAILY_CHANNEL\",\"text\":\":rotating_light: *${{ github.event.repository.name }} — schema health FAIL\n${msg}\"}"
|
|
|
|
- name: Smoke test
|
|
id: smoke
|
|
continue-on-error: true
|
|
run: |
|
|
chmod +x scripts/smoke-test.sh
|
|
./scripts/smoke-test.sh
|
|
# Also hit the new admin schema-check endpoint to verify
|
|
# no required columns are missing.
|
|
echo "::group::Schema health check"
|
|
php scripts/schema-health-check.php | tee /tmp/schema-report.json
|
|
if [ "$(jq -r .ok /tmp/schema-report.json)" != "true" ]; then
|
|
echo "::error::Schema health check FAILED — missing columns:"
|
|
jq -r '.missing[]' /tmp/schema-report.json | sed 's/^/ • /'
|
|
exit 1
|
|
fi
|
|
echo "Schema health check OK."
|
|
|
|
- name: Auto-rollback on smoke failure
|
|
if: steps.smoke.outcome == 'failure'
|
|
run: |
|
|
echo "::error::Smoke test failed — rolling back to ${{ steps.pre.outputs.pre_sha }}"
|
|
ssh "$DEPLOY_USER@$DEPLOY_HOST" '
|
|
set -e
|
|
cd /opt/${{ github.event.repository.name }}
|
|
git reset --hard ${{ steps.pre.outputs.pre_sha }}
|
|
if [ -f composer.json ]; then
|
|
composer install --no-dev --optimize-autoloader --no-interaction
|
|
sudo systemctl reload php8.2-fpm || true
|
|
fi
|
|
if [ -f package.json ]; then
|
|
npm ci --ignore-scripts
|
|
npm run build
|
|
sudo systemctl reload nginx || true
|
|
fi
|
|
'
|
|
|
|
- name: Post Slack status
|
|
if: always()
|
|
uses: slackapi/slack-github-action@v1.27.0
|
|
with:
|
|
channel-id: ${{ secrets.AI_DAILY_CHANNEL }}
|
|
payload: |
|
|
{
|
|
"text": "${{ job.status == 'success' && '✅' || '❌' }} Deploy *${{ github.repository }}@${{ github.sha[0:7] }}* — ${{ job.status }}\n${{ steps.smoke.outcome == 'failure' && '⚠️ Auto-rolled back' || '✓ Smoke test passed' }}"
|
|
}
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
- name: Update Linear issue
|
|
if: success() && steps.deploy.outcome == 'success'
|
|
run: |
|
|
# Find Linear issues in this commit's history and post a comment
|
|
# (uses GitHub's auto-link: if PR body contains "TRU-123" it auto-links)
|
|
# We skip this here; the OpenClaw cron `f26dfd83` handles Linear updates.
|
|
echo "Deploy notification will be picked up by OpenClaw cron."
|