237 lines
9.2 KiB
PHP
237 lines
9.2 KiB
PHP
<?php
|
|
|
|
app_require('classes/replication_secret_box.php');
|
|
app_require('classes/replication_bootstrap_config.php');
|
|
app_require('classes/replica_failover_manager.php');
|
|
|
|
use classes\replica_failover_manager;
|
|
use classes\replication_bootstrap_config;
|
|
|
|
beforeEach(function (): void {
|
|
$GLOBALS['ENCRYPTION_KEY'] = 'replica-failover-test-key';
|
|
});
|
|
|
|
function failover_test_host(string $kind, int $id, string $checkedAt, array $overrides = []): array
|
|
{
|
|
$base = [
|
|
'id' => $id,
|
|
'kind' => $kind,
|
|
'label' => $kind . ' replica ' . $id,
|
|
'host' => $kind . '-replica-' . $id,
|
|
'port' => match ($kind) {
|
|
'database' => 3306,
|
|
'redis' => 6379,
|
|
'minio' => 9000,
|
|
default => 1,
|
|
},
|
|
'database_name' => $kind === 'database' ? 'truckwash' : null,
|
|
'database_index' => $kind === 'redis' ? 0 : null,
|
|
'username' => $kind === 'minio' ? 'access-key' : 'app',
|
|
'password_secret' => 'secret',
|
|
'admin_username' => '',
|
|
'admin_password_secret' => '',
|
|
'role' => 'replica',
|
|
'status' => 'ok',
|
|
'options_json' => $kind === 'minio'
|
|
? json_encode(['endpoint' => 'http://minio-replica-' . $id . ':9000', 'buckets' => ['attachments']])
|
|
: null,
|
|
'last_status_json' => json_encode([
|
|
'status' => 'ok',
|
|
'replication_percent' => 100,
|
|
'blockers' => [],
|
|
'checked_at' => $checkedAt,
|
|
]),
|
|
'last_checked_at' => $checkedAt,
|
|
'deleted_at' => null,
|
|
];
|
|
|
|
return array_replace($base, $overrides);
|
|
}
|
|
|
|
it('normalizes failover config defaults and per-kind enablement', function (): void {
|
|
$defaults = replica_failover_manager::normalizeConfig([]);
|
|
|
|
expect($defaults)->toMatchArray([
|
|
'enabled' => false,
|
|
'database_enabled' => false,
|
|
'redis_enabled' => false,
|
|
'minio_enabled' => false,
|
|
'max_status_age_seconds' => 90,
|
|
]);
|
|
|
|
$config = replica_failover_manager::normalizeConfig([
|
|
'enabled' => 'true',
|
|
'database_enabled' => '1',
|
|
'redis_enabled' => false,
|
|
'minio_enabled' => 'yes',
|
|
'max_status_age_seconds' => '120',
|
|
]);
|
|
|
|
expect(replica_failover_manager::kindEnabled($config, 'database'))->toBeTrue();
|
|
expect(replica_failover_manager::kindEnabled($config, 'redis'))->toBeFalse();
|
|
expect(replica_failover_manager::kindEnabled($config, 'minio'))->toBeTrue();
|
|
expect($config['max_status_age_seconds'])->toBe(120);
|
|
});
|
|
|
|
it('requires strict fresh 100 percent replica status for candidates', function (): void {
|
|
$now = time();
|
|
$fresh = failover_test_host('database', 2, date('c', $now - 30));
|
|
$stale = failover_test_host('database', 3, date('c', $now - 120));
|
|
$notCaughtUp = failover_test_host('database', 4, date('c', $now - 10), [
|
|
'last_status_json' => json_encode([
|
|
'status' => 'ok',
|
|
'replication_percent' => 99.99,
|
|
'blockers' => [],
|
|
'checked_at' => date('c', $now - 10),
|
|
]),
|
|
]);
|
|
$blocked = failover_test_host('database', 5, date('c', $now - 10), [
|
|
'last_status_json' => json_encode([
|
|
'status' => 'degraded',
|
|
'replication_percent' => 100,
|
|
'blockers' => ['lagging'],
|
|
'checked_at' => date('c', $now - 10),
|
|
]),
|
|
]);
|
|
|
|
expect(replica_failover_manager::snapshotHostIsStrictlyFresh($fresh, 90, $now))->toBeTrue();
|
|
expect(replica_failover_manager::snapshotHostIsStrictlyFresh($stale, 90, $now))->toBeFalse();
|
|
expect(replica_failover_manager::snapshotHostIsStrictlyFresh($notCaughtUp, 90, $now))->toBeFalse();
|
|
expect(replica_failover_manager::snapshotHostIsStrictlyFresh($blocked, 90, $now))->toBeFalse();
|
|
});
|
|
|
|
it('selects the freshest eligible replica for failover', function (): void {
|
|
$now = time();
|
|
$older = failover_test_host('redis', 2, date('c', $now - 40));
|
|
$newer = failover_test_host('redis', 3, date('c', $now - 10));
|
|
$wrongKind = failover_test_host('minio', 4, date('c', $now - 5));
|
|
|
|
$candidate = replica_failover_manager::snapshotFailoverCandidate([$older, $newer, $wrongKind], 'redis', 90, $now);
|
|
|
|
expect($candidate['id'])->toBe(3);
|
|
});
|
|
|
|
it('promotes enabled startup dependencies from snapshot in dependency order', function (): void {
|
|
$path = tempnam(sys_get_temp_dir(), 'failover-snapshot-');
|
|
$events = [];
|
|
$now = time();
|
|
|
|
replication_bootstrap_config::writeSnapshot([
|
|
'active' => [
|
|
'database' => ['host' => 'db-primary', 'database' => 'truckwash', 'user' => 'app', 'password_secret' => 'secret'],
|
|
'redis' => ['host' => 'redis-primary', 'database' => 0, 'user' => '', 'password_secret' => 'secret'],
|
|
'minio' => ['endpoint' => 'http://minio-primary:9000', 'access_key' => 'access-key', 'secret_key_secret' => 'secret'],
|
|
],
|
|
'failover' => [
|
|
'config' => [
|
|
'enabled' => true,
|
|
'database_enabled' => true,
|
|
'redis_enabled' => true,
|
|
'minio_enabled' => true,
|
|
'max_status_age_seconds' => 90,
|
|
],
|
|
'hosts' => [
|
|
'database' => [failover_test_host('database', 11, date('c', $now - 10))],
|
|
'redis' => [failover_test_host('redis', 12, date('c', $now - 10))],
|
|
'minio' => [failover_test_host('minio', 13, date('c', $now - 10))],
|
|
],
|
|
],
|
|
], $path);
|
|
|
|
try {
|
|
$result = replica_failover_manager::applyStartupFailoverFromSnapshot($path, [
|
|
'primary_down' => function (string $kind) use (&$events): bool {
|
|
$events[] = 'primary:' . $kind;
|
|
return true;
|
|
},
|
|
'candidate_reachable' => function (string $kind) use (&$events): bool {
|
|
$events[] = 'reachable:' . $kind;
|
|
return true;
|
|
},
|
|
'promote_candidate' => function (string $kind) use (&$events): void {
|
|
$events[] = 'promote:' . $kind;
|
|
},
|
|
]);
|
|
|
|
$snapshot = replication_bootstrap_config::loadSnapshot($path);
|
|
|
|
expect($result['changed'])->toBeTrue();
|
|
expect($events)->toBe([
|
|
'primary:database',
|
|
'reachable:database',
|
|
'promote:database',
|
|
'primary:redis',
|
|
'reachable:redis',
|
|
'promote:redis',
|
|
'primary:minio',
|
|
'reachable:minio',
|
|
'promote:minio',
|
|
]);
|
|
expect($snapshot['active']['database']['id'])->toBe(11);
|
|
expect($snapshot['active']['redis']['id'])->toBe(12);
|
|
expect($snapshot['active']['minio']['id'])->toBe(13);
|
|
expect($snapshot['pending_failovers'])->toHaveCount(3);
|
|
} finally {
|
|
@unlink($path);
|
|
}
|
|
});
|
|
|
|
it('does not promote a disabled dependency during startup failover', function (): void {
|
|
$path = tempnam(sys_get_temp_dir(), 'failover-snapshot-');
|
|
$events = [];
|
|
|
|
replication_bootstrap_config::writeSnapshot([
|
|
'active' => [
|
|
'redis' => ['host' => 'redis-primary', 'database' => 0, 'user' => '', 'password_secret' => 'secret'],
|
|
],
|
|
'failover' => [
|
|
'config' => [
|
|
'enabled' => true,
|
|
'redis_enabled' => false,
|
|
'max_status_age_seconds' => 90,
|
|
],
|
|
'hosts' => [
|
|
'redis' => [failover_test_host('redis', 12, date('c'))],
|
|
],
|
|
],
|
|
], $path);
|
|
|
|
try {
|
|
$result = replica_failover_manager::applyStartupFailoverFromSnapshot($path, [
|
|
'primary_down' => function (string $kind) use (&$events): bool {
|
|
$events[] = $kind;
|
|
return true;
|
|
},
|
|
]);
|
|
|
|
expect($result['changed'])->toBeFalse();
|
|
expect($result['results']['redis']['reason'])->toBe('disabled');
|
|
expect($events)->toBe([]);
|
|
} finally {
|
|
@unlink($path);
|
|
}
|
|
});
|
|
|
|
it('wires the failover module config endpoint and promotion paths', function (): void {
|
|
$route = file_get_contents(app_path('routes/moduleConfigRoute.php'));
|
|
$module = file_get_contents(app_path('modules/failover/failover_c.php'));
|
|
$enabledConfig = file_get_contents(app_path('modules/failover/config/failover_enabled_c.php'));
|
|
$manager = file_get_contents(app_path('classes/replication_manager.php'));
|
|
$startup = file_get_contents(app_path('classes/replica_failover_manager.php'));
|
|
|
|
expect($route)->toContain("'/failover/config'");
|
|
expect($route)->toContain('modules_failover_config');
|
|
expect($enabledConfig)->toContain("'enabled'");
|
|
expect($module)->toContain('failover_database_enabled_c::class');
|
|
expect($module)->toContain('failover_redis_enabled_c::class');
|
|
expect($module)->toContain('failover_minio_enabled_c::class');
|
|
expect($module)->toContain('failover_max_status_age_seconds_c::class');
|
|
expect($manager)->toContain('promoteDatabaseHostForFailover');
|
|
expect($manager)->toContain('promoteRedisHostForFailover');
|
|
expect($manager)->toContain('promoteMinioHostForFailover');
|
|
expect($manager)->toContain("['REPLICAOF', 'NO', 'ONE']");
|
|
expect($manager)->toContain('runAutomaticFailoverMonitor');
|
|
expect($startup)->toContain('replication_bootstrap_config::loadSnapshot($path)');
|
|
expect($startup)->not->toContain('module_config');
|
|
});
|