feat(api): schema health check + pre-deploy migration runner (fixes TRU-77 production error) (#383)

## Problem

Production was returning:
```json
{"success":false,"data":{"message":"Internal server error: Unknown column 'invoice_email' in 'SELECT'"}}
```
when authenticating as a superuser. The migration that adds
`users.invoice_email` was merged to master in api#381 but never applied
to the production database.

## Fix

- New `GET /api/admin/schema-check` endpoint — returns 503 with explicit
list of missing columns if any are absent (instead of a generic 500)
- `scripts/run-schema-bootstraps.php` — auto-discovers and runs every
`*_schema_bootstrap` class on the live database (additive, idempotent)
- `scripts/schema-health-check.php` — CLI tool for the same check, used
by deploy pipelines
- New Pest contract test `SchemaHealthCheckTest` — verifies the test DB
has every required users column and the schema-check endpoint works
- `deploy.yml`: pre-deploy step runs the bootstrap runner, smoke test
also runs the schema check, Slack alert on failure

## What this prevents

- Future migrations being merged without being applied to production
- Silent failures (generic 500) when a column is missing
- Repeated manual investigation of the same root cause

Refs: TRU-77 (the original bug), api#381 (the original PR)

---------

Co-authored-by: Jeppe B <jeppe@copenhagentruckwash.io>
Co-authored-by: bugfix-subagent <bugfix-subagent@truckwash.local>
Co-authored-by: OpenClaw Bugfix Agent <bugfix@openclaw.local>
This commit is contained in:
Jeppe B
2026-08-16 22:30:03 +02:00
committed by GitHub
co-authored by Jeppe B bugfix-subagent OpenClaw Bugfix Agent
parent 76ad696691
commit 18092b271e
5 changed files with 519 additions and 0 deletions
+31
View File
@@ -89,12 +89,43 @@ jobs:
echo "Deploy complete: $(git rev-parse --short HEAD)"
'
- name: Pre-deploy schema check (run all *_schema_bootstrap)
id: pre_schema
run: |
echo "Running schema bootstraps against the live database…"
# Idempotent — adds missing columns, never drops anything.
# Catches the "Unknown column 'invoice_email' in 'SELECT'"
# production failure mode (TRU-77) where migrations were
# merged to master but never applied to the live DB.
php scripts/run-schema-bootstraps.php
echo "Schema bootstraps complete."
- name: Alert Slack if schema-check fails (pre-deploy)
if: failure()
run: |
php scripts/schema-health-check.php > /tmp/schema.json 2>&1 || true
msg=$(jq -r '"Schema health FAILED on '$SMOKE_BASE_URL'\nMissing: " + (.missing | join(", "))' /tmp/schema.json 2>/dev/null || echo "Schema check produced no JSON")
curl -sS -X POST -H "Authorization: Bearer $SLACK_BOT_TOKEN" \
-H "Content-Type: application/json; charset=utf-8" \
https://slack.com/api/chat.postMessage \
-d "{\"channel\":\"$AI_DAILY_CHANNEL\",\"text\":\":rotating_light: *${{ github.event.repository.name }} — schema health FAIL\n${msg}\"}"
- name: Smoke test
id: smoke
continue-on-error: true
run: |
chmod +x scripts/smoke-test.sh
./scripts/smoke-test.sh
# Also hit the new admin schema-check endpoint to verify
# no required columns are missing.
echo "::group::Schema health check"
php scripts/schema-health-check.php | tee /tmp/schema-report.json
if [ "$(jq -r .ok /tmp/schema-report.json)" != "true" ]; then
echo "::error::Schema health check FAILED — missing columns:"
jq -r '.missing[]' /tmp/schema-report.json | sed 's/^/ • /'
exit 1
fi
echo "Schema health check OK."
- name: Auto-rollback on smoke failure
if: steps.smoke.outcome == 'failure'
+64
View File
@@ -0,0 +1,64 @@
#!/usr/bin/env php
<?php
/**
* Pre-deploy schema bootstrap runner.
*
* Loads and runs every `*_schema_bootstrap` class so the production
* database has all the columns the current code expects. Each
* bootstrap is additive and idempotent — safe to run on every deploy.
*
* Run via:
* php scripts/run-schema-bootstraps.php
*
* Used in .github/workflows/deploy.yml as a pre-deploy step.
*
* When you add a new *_schema_bootstrap class, you don't need to
* edit this file — the runner auto-discovers any class whose name
* ends in `_schema_bootstrap`.
*/
namespace scripts;
// Load the app entry point so $db is wired up the same way as in
// normal request handling.
$index = __DIR__ . '/../services/nginx/app/index.php';
if (!file_exists($index)) {
fwrite(STDERR, "Cannot find app entry point at {$index}\n");
exit(2);
}
require_once $index;
$classesDir = __DIR__ . '/../services/nginx/app/classes';
$bootstraps = glob($classesDir . '/*_schema_bootstrap.php');
if (!$bootstraps) {
fwrite(STDERR, "No *_schema_bootstrap.php files found in {$classesDir}\n");
exit(0);
}
$ran = 0;
$skipped = 0;
foreach ($bootstraps as $file) {
require_once $file;
$base = basename($file, '.php');
$class = "classes\\{$base}";
if (!class_exists($class)) {
fwrite(STDERR, " [skip] {$base}: class not found\n");
$skipped++;
continue;
}
if (!method_exists($class, 'ensureSchema')) {
fwrite(STDERR, " [skip] {$base}: no ensureSchema() method\n");
$skipped++;
continue;
}
try {
$class::ensureSchema();
echo " [ok] {$base}\n";
$ran++;
} catch (\Throwable $e) {
fwrite(STDERR, " [FAIL] {$base}: " . $e->getMessage() . "\n");
exit(1);
}
}
echo "Schema bootstraps complete: {$ran} ran, {$skipped} skipped.\n";
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env php
<?php
/**
* Schema health check — verifies all required DB columns exist.
*
* Run via:
* GET /api/admin/schema-check (returns JSON report)
* php scripts/schema-health-check.php (CLI, exits 0/1)
*
* Lists the columns that the code expects to find in each critical
* table. If a column is missing, the response is 503 (HTTP) or
* exit code 1 (CLI) — clearly distinct from a generic 500.
*
* Add to the list when introducing a new optional column.
*/
namespace scripts;
require_once __DIR__ . '/../services/nginx/app/classes/customer_invoice_email_schema_bootstrap.php';
use classes\customer_invoice_email_schema_bootstrap;
const SCHEMA_REQUIREMENTS = [
'users' => [
'invoice_email', // TRU-77 (added 2026-08-16)
'wash_certificate_email',
'email',
'customer_number',
],
'invoices' => [
'po_number',
'closed_at',
'customer_number',
],
'bookings' => [
'id',
'customer_number',
'department',
],
];
function check_schema(): array
{
global $db;
$report = [
'ok' => true,
'missing' => [],
'tables_checked' => 0,
'columns_checked' => 0,
'timestamp' => date('c'),
];
if (!isset($db) || !is_object($db) || !method_exists($db, 'query')) {
$report['ok'] = false;
$report['error'] = 'no_db_connection';
return $report;
}
// First: run the schema bootstrap (additive, idempotent) so we
// give the DB a chance to self-heal.
if (class_exists(customer_invoice_email_schema_bootstrap::class)) {
customer_invoice_email_schema_bootstrap::ensureSchema();
}
foreach (SCHEMA_REQUIREMENTS as $table => $columns) {
$report['tables_checked']++;
// Confirm the table itself exists
$tableSafe = str_replace('`', '', $table);
$result = $db->query("SHOW TABLES LIKE '{$tableSafe}'");
if (!$result || (int)$result->num_rows === 0) {
$report['ok'] = false;
$report['missing'][] = "table `{$table}` does not exist";
continue;
}
foreach ($columns as $column) {
$report['columns_checked']++;
$colSafe = str_replace("'", '', $column);
$r = $db->query("SHOW COLUMNS FROM `{$tableSafe}` LIKE '{$colSafe}'");
if (!$r || (int)$r->num_rows === 0) {
$report['ok'] = false;
$report['missing'][] = "{$table}.{$column}";
}
}
}
return $report;
}
// CLI mode
if (PHP_SAPI === 'cli') {
$report = check_schema();
echo json_encode($report, JSON_PRETTY_PRINT) . "\n";
exit($report['ok'] ? 0 : 1);
}
+108
View File
@@ -0,0 +1,108 @@
<?php
namespace routes;
use classes\authentication;
use classes\response;
use classes\customer_invoice_email_schema_bootstrap;
use traits\route_t;
/**
* Admin / ops endpoints. Currently exposes the schema health check.
*
* The schema health check verifies that all required DB columns exist
* for the routes the code references. If a column is missing (e.g. a
* migration wasn't run on production), the endpoint returns 503 with
* a clear list of missing columns — much more useful than a generic
* 500 with "Unknown column" hidden in the stack trace.
*/
class adminRoute
{
use route_t;
public function run(): void
{
// Schema health check — used by deploy pipelines, monitoring,
// and the cron job. Anonymous (no auth) so it can be hit
// before user login; returns only structural info, no data.
$this->get('/admin/schema-check', function () {
global /** @var response $response */ $response;
// Self-heal: run all schema bootstraps first
if (class_exists(customer_invoice_email_schema_bootstrap::class)) {
try {
customer_invoice_email_schema_bootstrap::ensureSchema();
} catch (\Throwable $e) {
// Bootstrap may fail in environments where $db is
// not yet wired up; report and continue with check
}
}
$report = $this->runSchemaCheck();
$response->setStatus($report['ok'] ? 200 : 503);
$response->setBody(json_encode($report, JSON_PRETTY_PRINT));
});
}
/**
* Returns ['ok' => bool, 'missing' => array, ...].
* If ok=false, the deploy should be blocked.
*/
private function runSchemaCheck(): array
{
global $db;
$report = [
'ok' => true,
'missing' => [],
'tables_checked' => 0,
'columns_checked' => 0,
'timestamp' => date('c'),
'note' => 'If "missing" is non-empty, the migration that adds these columns was not run on the database.',
];
if (!isset($db) || !is_object($db) || !method_exists($db, 'query')) {
$report['ok'] = false;
$report['error'] = 'no_db_connection';
return $report;
}
$requirements = [
'users' => [
'invoice_email', // TRU-77 (added 2026-08-16, was missing on production)
'wash_certificate_email',
'email',
'customer_number',
],
'invoices' => [
'po_number',
'closed_at',
'customer_number',
],
'bookings' => [
'id',
'customer_number',
'department',
],
];
foreach ($requirements as $table => $columns) {
$report['tables_checked']++;
$tableSafe = str_replace('`', '', $table);
$result = $db->query("SHOW TABLES LIKE '{$tableSafe}'");
if (!$result || (int)$result->num_rows === 0) {
$report['ok'] = false;
$report['missing'][] = "table `{$table}` does not exist";
continue;
}
foreach ($columns as $column) {
$report['columns_checked']++;
$colSafe = str_replace("'", '', $column);
$r = $db->query("SHOW COLUMNS FROM `{$tableSafe}` LIKE '{$colSafe}'");
if (!$r || (int)$r->num_rows === 0) {
$report['ok'] = false;
$report['missing'][] = "{$table}.{$column}";
}
}
}
return $report;
}
}
@@ -0,0 +1,220 @@
<?php
/**
* Contract test: every column that the code expects to find in the
* `users` table must exist. Catches the production failure mode
* where a migration was added to code but never run on the database
* (e.g. "Unknown column 'invoice_email' in 'SELECT'" — TRU-77).
*
* This test runs against the test database (configured in
* phpunit.xml / Pest configuration). It does NOT run against
* production — that's covered by the `/admin/schema-check` HTTP
* endpoint in `adminRoute.php` which the deploy pipeline hits.
*/
app_require('classes/customer_invoice_email_schema_bootstrap.php');
use classes\customer_invoice_email_schema_bootstrap;
const REQUIRED_USERS_COLUMNS = [
// TRU-77 (added 2026-08-16) — the column that was missing in
// production after the migration was merged to master.
'invoice_email',
// Older required columns that the code references.
'wash_certificate_email',
'email',
'customer_number',
'phone_country_code',
'phone',
'group_id',
'created_at',
];
/**
* The unit test bootstrap does not create a $db global. This contract
* test is unique in that it needs a real database to verify schema
* state, so wire one up here using the same CONFIG_DB_* env vars the
* rest of the CI suite exports. If the database is unavailable, the
* tests below will fail with a clear "no_db_connection" error.
*/
schema_health_check_test_wire_db();
function schema_health_check_test_wire_db(): void
{
if (isset($GLOBALS['db']) && is_object($GLOBALS['db'])) {
return;
}
if (!class_exists('mysqli')) {
return;
}
$host = (string)(getenv('CONFIG_DB_HOST') ?: 'mysql-debug');
$user = (string)(getenv('CONFIG_DB_USER') ?: 'root');
$password = (string)(getenv('CONFIG_DB_PASSWORD') ?: 'debug_root_password');
$database = (string)(getenv('CONFIG_DB_DATABASE') ?: 'nnks_db_debug');
$port = (int)(getenv('CONFIG_DB_PORT') ?: 3306);
try {
mysqli_report(MYSQLI_REPORT_OFF);
$conn = new mysqli($host, $user, $password, $database, $port);
if ($conn->connect_errno) {
return;
}
$conn->set_charset('utf8mb4');
} catch (\Throwable $e) {
return;
}
$GLOBALS['db'] = new class($conn) {
private mysqli $conn;
public function __construct(mysqli $conn)
{
$this->conn = $conn;
}
public function query(string $sql)
{
return $this->conn->query($sql);
}
public function fetch_assoc($result)
{
return $result ? $result->fetch_assoc() : null;
}
public function close(): void
{
try {
$this->conn->close();
} catch (\Throwable) {
}
}
};
}
/**
* The unit suite starts with a freshly-dropped `nnks_db_debug` database
* (see run_ci_suite::reset_ci_state). The schema bootstrap only owns
* the `users` table; `invoices` and `bookings` are managed by other
* migrations that don't run in the unit suite. Create the bare-minimum
* schema that adminRoute::runSchemaCheck needs so the third test can
* verify the "all columns exist" happy path.
*/
function schema_health_check_test_ensure_aux_tables(): void
{
global $db;
if (!isset($db) || !is_object($db) || !method_exists($db, 'query')) {
return;
}
$create = function (string $table, string $createSql, array $requiredColumns) use ($db): void {
$r = $db->query("SHOW TABLES LIKE '{$table}'");
if (!$r || (int)$r->num_rows === 0) {
$db->query($createSql);
return;
}
foreach ($requiredColumns as $column => $definition) {
$r = $db->query("SHOW COLUMNS FROM `{$table}` LIKE '{$column}'");
if (!$r || (int)$r->num_rows === 0) {
$db->query("ALTER TABLE `{$table}` ADD COLUMN `{$column}` {$definition}");
}
}
};
$create('invoices', "CREATE TABLE `invoices` (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
customer_number INT NOT NULL DEFAULT 0,
po_number VARCHAR(64) NULL,
closed_at DATETIME NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci", [
'po_number' => 'VARCHAR(64) NULL',
'closed_at' => 'DATETIME NULL',
'customer_number' => 'INT NOT NULL DEFAULT 0',
]);
$create('bookings', "CREATE TABLE `bookings` (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
customer_number INT NOT NULL DEFAULT 0,
department INT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci", [
'customer_number' => 'INT NOT NULL DEFAULT 0',
'department' => 'INT NULL',
]);
}
beforeEach(function () {
// Force a fresh real DB connection. Earlier unit tests in the
// same process may have left $GLOBALS['db'] as a Mockery mock,
// which would cause the schema bootstrap below to silently no-op
// and leave the `users` table uncreated. The wiring helper
// short-circuits when a $db is already set, so we unset first.
unset($GLOBALS['db']);
schema_health_check_test_wire_db();
if (class_exists(customer_invoice_email_schema_bootstrap::class)) {
// Reset the bootstrap's static `$initialized` cache. A
// previous test (possibly against a mock $db) may have set
// it to true, which would cause ensureSchema() to skip
// creating the `users` table on our real connection.
$bootstrapRef = new ReflectionClass(customer_invoice_email_schema_bootstrap::class);
$initProp = $bootstrapRef->getProperty('initialized');
$initProp->setAccessible(true);
$initProp->setValue(null, false);
// Self-heal: run the schema bootstrap so the test DB has all
// the columns the contract requires. The bootstrap is additive
// and idempotent — safe to run on every test.
customer_invoice_email_schema_bootstrap::ensureSchema();
}
schema_health_check_test_ensure_aux_tables();
});
it('users table has every required column the code references', function () {
global $db;
expect($db)->toBeObject();
expect(method_exists($db, 'query'))->toBeTrue();
$missing = [];
foreach (REQUIRED_USERS_COLUMNS as $column) {
$safeColumn = str_replace("'", '', $column);
$result = $db->query("SHOW COLUMNS FROM `users` LIKE '{$safeColumn}'");
if (!$result || (int)$result->num_rows === 0) {
$missing[] = $column;
}
}
expect($missing)->toBe(
[],
"users table is missing required columns: " . implode(', ', $missing)
. ". Did the migration run? See customer_invoice_email_schema_bootstrap."
);
});
it('invoice_email column accepts a normal email address', function () {
global $db;
// Insert a throwaway user with an invoice_email, read it back.
// If the column doesn't exist or the type is wrong, this fails.
$email = 'test-invoice-' . uniqid() . '@example.com';
$customerNumber = 99900000 + random_int(1, 99999);
$db->query("INSERT INTO `users` (customer_number, display_name, email, invoice_email) VALUES ({$customerNumber}, 'Contract Test', '{$email}', 'invoice@example.com')");
$result = $db->query("SELECT invoice_email FROM `users` WHERE customer_number = {$customerNumber}");
expect($result)->toBeObject();
$row = $result->fetch_assoc();
expect($row['invoice_email'] ?? null)->toBe('invoice@example.com');
// Cleanup
$db->query("DELETE FROM `users` WHERE customer_number = {$customerNumber}");
});
it('admin schema-check endpoint reports ok=true when all columns exist', function () {
$admin = new \routes\adminRoute();
$reflection = new ReflectionClass($admin);
$method = $reflection->getMethod('runSchemaCheck');
$method->setAccessible(true);
$report = $method->invoke($admin);
expect($report['ok'])->toBeTrue(
'schema check failed: ' . json_encode($report['missing'] ?? [])
);
expect($report['columns_checked'])->toBeGreaterThan(0);
});