522 lines
19 KiB
PHP
522 lines
19 KiB
PHP
<?php
|
|
|
|
namespace classes;
|
|
|
|
use Aws\S3\S3Client;
|
|
use mysqli;
|
|
use Predis\Client as PredisClient;
|
|
use Throwable;
|
|
|
|
class replica_failover_manager
|
|
{
|
|
public const KIND_DATABASE = 'database';
|
|
public const KIND_REDIS = 'redis';
|
|
public const KIND_MINIO = 'minio';
|
|
public const DEFAULT_MAX_STATUS_AGE_SECONDS = 90;
|
|
|
|
public static function configDefaults(): array
|
|
{
|
|
return [
|
|
'enabled' => false,
|
|
'database_enabled' => false,
|
|
'redis_enabled' => false,
|
|
'minio_enabled' => false,
|
|
'max_status_age_seconds' => self::DEFAULT_MAX_STATUS_AGE_SECONDS,
|
|
];
|
|
}
|
|
|
|
public static function normalizeConfig(array $config): array
|
|
{
|
|
$normalized = self::configDefaults();
|
|
foreach (['enabled', 'database_enabled', 'redis_enabled', 'minio_enabled'] as $key) {
|
|
if (array_key_exists($key, $config)) {
|
|
$normalized[$key] = self::boolValue($config[$key]);
|
|
}
|
|
}
|
|
|
|
if (array_key_exists('max_status_age_seconds', $config)) {
|
|
$normalized['max_status_age_seconds'] = max(1, (int)$config['max_status_age_seconds']);
|
|
}
|
|
|
|
return $normalized;
|
|
}
|
|
|
|
public static function kindEnabled(array $config, string $kind): bool
|
|
{
|
|
$config = self::normalizeConfig($config);
|
|
return $config['enabled'] && !empty($config[$kind . '_enabled']);
|
|
}
|
|
|
|
public static function snapshotHostIsStrictlyFresh(array $host, int $maxAgeSeconds, ?int $now = null): bool
|
|
{
|
|
if (($host['role'] ?? '') !== 'replica') {
|
|
return false;
|
|
}
|
|
|
|
if (!empty($host['deleted_at'])) {
|
|
return false;
|
|
}
|
|
|
|
$status = self::hostStatus($host);
|
|
if (($status['status'] ?? '') !== 'ok') {
|
|
return false;
|
|
}
|
|
|
|
if (round((float)($status['replication_percent'] ?? 0), 2) < 100.0) {
|
|
return false;
|
|
}
|
|
|
|
$blockers = $status['blockers'] ?? [];
|
|
if (is_array($blockers) && $blockers !== []) {
|
|
return false;
|
|
}
|
|
|
|
$checkedAt = self::hostCheckedAt($host, $status);
|
|
if ($checkedAt === null) {
|
|
return false;
|
|
}
|
|
|
|
return (($now ?? time()) - $checkedAt) <= max(1, $maxAgeSeconds);
|
|
}
|
|
|
|
public static function snapshotFailoverCandidate(array $hosts, string $kind, int $maxAgeSeconds, ?int $now = null): ?array
|
|
{
|
|
$eligible = array_values(array_filter(
|
|
$hosts,
|
|
static fn(array $host): bool => ($host['kind'] ?? '') === $kind
|
|
&& self::snapshotHostIsStrictlyFresh($host, $maxAgeSeconds, $now)
|
|
));
|
|
|
|
if ($eligible === []) {
|
|
return null;
|
|
}
|
|
|
|
usort($eligible, static function (array $a, array $b) use ($now): int {
|
|
$aChecked = self::hostCheckedAt($a, self::hostStatus($a)) ?? 0;
|
|
$bChecked = self::hostCheckedAt($b, self::hostStatus($b)) ?? 0;
|
|
if ($aChecked === $bChecked) {
|
|
return (int)($a['id'] ?? 0) <=> (int)($b['id'] ?? 0);
|
|
}
|
|
return $bChecked <=> $aChecked;
|
|
});
|
|
|
|
return $eligible[0];
|
|
}
|
|
|
|
public static function activeConfigFromHost(string $kind, array $host): ?array
|
|
{
|
|
if ($kind === self::KIND_DATABASE) {
|
|
$database = trim((string)($host['database_name'] ?? $host['database'] ?? ''));
|
|
$user = trim((string)($host['username'] ?? $host['user'] ?? ''));
|
|
if ($database === '' || $user === '') {
|
|
return null;
|
|
}
|
|
|
|
return [
|
|
'id' => isset($host['id']) ? (int)$host['id'] : null,
|
|
'host' => (string)($host['host'] ?? ''),
|
|
'port' => (int)($host['port'] ?? 3306) ?: 3306,
|
|
'database' => $database,
|
|
'user' => $user,
|
|
'password_secret' => (string)($host['password_secret'] ?? ''),
|
|
'ssl_mode' => (string)($host['ssl_mode'] ?? 'DISABLED'),
|
|
];
|
|
}
|
|
|
|
if ($kind === self::KIND_REDIS) {
|
|
return [
|
|
'id' => isset($host['id']) ? (int)$host['id'] : null,
|
|
'host' => (string)($host['host'] ?? ''),
|
|
'port' => (int)($host['port'] ?? 6379) ?: 6379,
|
|
'database' => (int)($host['database_index'] ?? $host['database'] ?? 0),
|
|
'user' => (string)($host['username'] ?? $host['user'] ?? ''),
|
|
'password_secret' => (string)($host['password_secret'] ?? ''),
|
|
];
|
|
}
|
|
|
|
if ($kind === self::KIND_MINIO) {
|
|
$options = self::jsonDecode($host['options_json'] ?? null);
|
|
return [
|
|
'id' => isset($host['id']) ? (int)$host['id'] : null,
|
|
'endpoint' => self::minioEndpoint($host, $options),
|
|
'access_key' => (string)($host['username'] ?? $host['access_key'] ?? ''),
|
|
'secret_key_secret' => (string)($host['password_secret'] ?? ''),
|
|
'buckets' => is_array($options['buckets'] ?? null) ? array_values($options['buckets']) : [],
|
|
];
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
public static function applyStartupFailoverFromSnapshot(?string $path = null, array $probes = []): array
|
|
{
|
|
$snapshot = replication_bootstrap_config::loadSnapshot($path);
|
|
$failover = is_array($snapshot['failover'] ?? null) ? $snapshot['failover'] : [];
|
|
$config = self::normalizeConfig(is_array($failover['config'] ?? null) ? $failover['config'] : $failover);
|
|
$active = is_array($snapshot['active'] ?? null) ? $snapshot['active'] : [];
|
|
$hostGroups = is_array($failover['hosts'] ?? null) ? $failover['hosts'] : [];
|
|
$maxAgeSeconds = (int)$config['max_status_age_seconds'];
|
|
$summary = [];
|
|
$changed = false;
|
|
|
|
$primaryDown = $probes['primary_down'] ?? [self::class, 'activePrimaryIsDown'];
|
|
$candidateReachable = $probes['candidate_reachable'] ?? [self::class, 'candidateReachable'];
|
|
$promoteCandidate = $probes['promote_candidate'] ?? [self::class, 'promoteCandidate'];
|
|
|
|
foreach ([self::KIND_DATABASE, self::KIND_REDIS, self::KIND_MINIO] as $kind) {
|
|
if (!self::kindEnabled($config, $kind)) {
|
|
$summary[$kind] = ['status' => 'skipped', 'reason' => 'disabled'];
|
|
continue;
|
|
}
|
|
|
|
if (!is_array($active[$kind] ?? null)) {
|
|
$summary[$kind] = ['status' => 'skipped', 'reason' => 'missing_active_primary'];
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
if (!call_user_func($primaryDown, $kind, $active[$kind], $snapshot)) {
|
|
$summary[$kind] = ['status' => 'skipped', 'reason' => 'primary_healthy'];
|
|
continue;
|
|
}
|
|
|
|
$hosts = is_array($hostGroups[$kind] ?? null) ? $hostGroups[$kind] : [];
|
|
$candidate = self::snapshotFailoverCandidate($hosts, $kind, $maxAgeSeconds);
|
|
if ($candidate === null) {
|
|
$summary[$kind] = ['status' => 'skipped', 'reason' => 'no_fresh_caught_up_replica'];
|
|
continue;
|
|
}
|
|
|
|
if (!call_user_func($candidateReachable, $kind, $candidate)) {
|
|
$summary[$kind] = [
|
|
'status' => 'skipped',
|
|
'reason' => 'candidate_unreachable',
|
|
'candidate_id' => (int)($candidate['id'] ?? 0),
|
|
];
|
|
continue;
|
|
}
|
|
|
|
call_user_func($promoteCandidate, $kind, $candidate);
|
|
$candidateActive = self::activeConfigFromHost($kind, $candidate);
|
|
if ($candidateActive === null) {
|
|
$summary[$kind] = [
|
|
'status' => 'skipped',
|
|
'reason' => 'candidate_missing_active_config',
|
|
'candidate_id' => (int)($candidate['id'] ?? 0),
|
|
];
|
|
continue;
|
|
}
|
|
|
|
$snapshot['active'][$kind] = $candidateActive;
|
|
$pending = is_array($snapshot['pending_failovers'] ?? null) ? $snapshot['pending_failovers'] : [];
|
|
$pending[] = [
|
|
'kind' => $kind,
|
|
'host_id' => (int)($candidate['id'] ?? 0),
|
|
'label' => (string)($candidate['label'] ?? ''),
|
|
'source' => 'startup_snapshot',
|
|
'promoted_at' => date('c'),
|
|
];
|
|
$snapshot['pending_failovers'] = $pending;
|
|
$summary[$kind] = [
|
|
'status' => 'promoted',
|
|
'candidate_id' => (int)($candidate['id'] ?? 0),
|
|
];
|
|
$changed = true;
|
|
} catch (Throwable $throwable) {
|
|
$summary[$kind] = [
|
|
'status' => 'failed',
|
|
'reason' => $throwable->getMessage(),
|
|
];
|
|
}
|
|
}
|
|
|
|
if ($changed) {
|
|
$snapshot['generated_at'] = date('c');
|
|
replication_bootstrap_config::writeSnapshot($snapshot, $path);
|
|
if ($path === null) {
|
|
replication_bootstrap_config::applyToGlobals($snapshot);
|
|
}
|
|
}
|
|
|
|
return [
|
|
'changed' => $changed,
|
|
'results' => $summary,
|
|
];
|
|
}
|
|
|
|
public static function activePrimaryIsDown(string $kind, array $activeConfig, array $snapshot = []): bool
|
|
{
|
|
try {
|
|
match ($kind) {
|
|
self::KIND_DATABASE => self::probeActiveDatabase($activeConfig),
|
|
self::KIND_REDIS => self::probeActiveRedis($activeConfig),
|
|
self::KIND_MINIO => self::probeActiveMinio($activeConfig),
|
|
default => null,
|
|
};
|
|
return false;
|
|
} catch (Throwable) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
public static function candidateReachable(string $kind, array $host): bool
|
|
{
|
|
try {
|
|
match ($kind) {
|
|
self::KIND_DATABASE => self::probeHostDatabase($host),
|
|
self::KIND_REDIS => self::probeHostRedis($host),
|
|
self::KIND_MINIO => self::probeHostMinio($host),
|
|
default => null,
|
|
};
|
|
return true;
|
|
} catch (Throwable) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public static function promoteCandidate(string $kind, array $host): void
|
|
{
|
|
match ($kind) {
|
|
self::KIND_DATABASE => self::promoteDatabaseCandidate($host),
|
|
self::KIND_REDIS => self::promoteRedisCandidate($host),
|
|
self::KIND_MINIO => self::probeHostMinio($host),
|
|
default => null,
|
|
};
|
|
}
|
|
|
|
private static function probeActiveDatabase(array $config): void
|
|
{
|
|
$host = (string)($config['host'] ?? '');
|
|
$user = (string)($config['user'] ?? '');
|
|
$database = (string)($config['database'] ?? '');
|
|
$password = self::activePassword($config, 'password_secret', 'password');
|
|
self::connectMysqli($host, $user, $password, $database, (int)($config['port'] ?? 3306))->close();
|
|
}
|
|
|
|
private static function probeHostDatabase(array $host): void
|
|
{
|
|
$credentials = self::hostCredentials($host);
|
|
$connection = self::connectMysqli(
|
|
(string)($host['host'] ?? ''),
|
|
$credentials['username'],
|
|
$credentials['password'],
|
|
(string)($host['database_name'] ?? ''),
|
|
(int)($host['port'] ?? 3306)
|
|
);
|
|
$connection->close();
|
|
}
|
|
|
|
private static function promoteDatabaseCandidate(array $host): void
|
|
{
|
|
$credentials = self::hostCredentials($host);
|
|
$user = $credentials['admin_username'] !== '' ? $credentials['admin_username'] : $credentials['username'];
|
|
$password = $credentials['admin_password'] !== '' ? $credentials['admin_password'] : $credentials['password'];
|
|
$connection = self::connectMysqli(
|
|
(string)($host['host'] ?? ''),
|
|
$user,
|
|
$password,
|
|
(string)($host['database_name'] ?? ''),
|
|
(int)($host['port'] ?? 3306)
|
|
);
|
|
|
|
try {
|
|
foreach (['STOP REPLICA', 'STOP SLAVE'] as $statement) {
|
|
try {
|
|
$connection->query($statement);
|
|
break;
|
|
} catch (Throwable) {
|
|
}
|
|
}
|
|
foreach (['SET GLOBAL super_read_only = OFF', 'SET GLOBAL read_only = OFF'] as $statement) {
|
|
try {
|
|
$connection->query($statement);
|
|
} catch (Throwable) {
|
|
}
|
|
}
|
|
} finally {
|
|
$connection->close();
|
|
}
|
|
}
|
|
|
|
private static function probeActiveRedis(array $config): void
|
|
{
|
|
self::redisClientFromConfig([
|
|
'host' => (string)($config['host'] ?? ''),
|
|
'port' => (int)($config['port'] ?? 6379),
|
|
'database' => (int)($config['database'] ?? 0),
|
|
'user' => (string)($config['user'] ?? ''),
|
|
'password' => self::activePassword($config, 'password_secret', 'password'),
|
|
])->ping();
|
|
}
|
|
|
|
private static function probeHostRedis(array $host): void
|
|
{
|
|
self::redisClientFromHost($host)->ping();
|
|
}
|
|
|
|
private static function promoteRedisCandidate(array $host): void
|
|
{
|
|
$client = self::redisClientFromHost($host);
|
|
$client->executeRaw(['REPLICAOF', 'NO', 'ONE']);
|
|
try {
|
|
$client->executeRaw(['CONFIG', 'REWRITE']);
|
|
} catch (Throwable) {
|
|
}
|
|
}
|
|
|
|
private static function probeActiveMinio(array $config): void
|
|
{
|
|
self::minioClientFromConfig([
|
|
'endpoint' => (string)($config['endpoint'] ?? ''),
|
|
'access_key' => (string)($config['access_key'] ?? $config['user'] ?? ''),
|
|
'secret_key' => self::activePassword($config, 'secret_key_secret', 'secret_key'),
|
|
])->listBuckets();
|
|
}
|
|
|
|
private static function probeHostMinio(array $host): void
|
|
{
|
|
self::minioClientFromHost($host)->listBuckets();
|
|
}
|
|
|
|
private static function connectMysqli(string $host, string $user, string $password, string $database, int $port): mysqli
|
|
{
|
|
mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
|
|
$connection = mysqli_init();
|
|
$connection->options(MYSQLI_OPT_CONNECT_TIMEOUT, 2);
|
|
$connection->real_connect($host, $user, $password, $database, $port ?: 3306);
|
|
$connection->set_charset('utf8mb4');
|
|
return $connection;
|
|
}
|
|
|
|
private static function redisClientFromHost(array $host): PredisClient
|
|
{
|
|
$credentials = self::hostCredentials($host);
|
|
return self::redisClientFromConfig([
|
|
'host' => (string)($host['host'] ?? ''),
|
|
'port' => (int)($host['port'] ?? 6379),
|
|
'database' => (int)($host['database_index'] ?? 0),
|
|
'user' => $credentials['username'],
|
|
'password' => $credentials['password'],
|
|
]);
|
|
}
|
|
|
|
private static function redisClientFromConfig(array $config): PredisClient
|
|
{
|
|
$params = [
|
|
'scheme' => 'tcp',
|
|
'host' => (string)$config['host'],
|
|
'port' => (int)$config['port'],
|
|
'database' => (int)$config['database'],
|
|
'password' => (string)$config['password'],
|
|
'timeout' => 2.0,
|
|
'read_write_timeout' => 2.0,
|
|
];
|
|
if (($config['user'] ?? '') !== '' && $config['user'] !== 'default') {
|
|
$params['username'] = (string)$config['user'];
|
|
}
|
|
return new PredisClient($params);
|
|
}
|
|
|
|
private static function minioClientFromHost(array $host): S3Client
|
|
{
|
|
$credentials = self::hostCredentials($host);
|
|
$options = self::jsonDecode($host['options_json'] ?? null);
|
|
return self::minioClientFromConfig([
|
|
'endpoint' => self::minioEndpoint($host, $options),
|
|
'access_key' => $credentials['username'],
|
|
'secret_key' => $credentials['password'],
|
|
]);
|
|
}
|
|
|
|
private static function minioClientFromConfig(array $config): S3Client
|
|
{
|
|
return new S3Client([
|
|
'version' => 'latest',
|
|
'region' => 'us-east-1',
|
|
'endpoint' => (string)$config['endpoint'],
|
|
'use_path_style_endpoint' => true,
|
|
'credentials' => [
|
|
'key' => (string)$config['access_key'],
|
|
'secret' => (string)$config['secret_key'],
|
|
],
|
|
'http' => [
|
|
'connect_timeout' => 2,
|
|
'timeout' => 2,
|
|
],
|
|
]);
|
|
}
|
|
|
|
private static function minioEndpoint(array $host, array $options): string
|
|
{
|
|
$endpoint = trim((string)($options['endpoint'] ?? ''));
|
|
if ($endpoint !== '') {
|
|
return $endpoint;
|
|
}
|
|
|
|
$scheme = strtolower(trim((string)($options['scheme'] ?? 'http')));
|
|
if ($scheme !== 'https') {
|
|
$scheme = 'http';
|
|
}
|
|
|
|
return $scheme . '://' . (string)($host['host'] ?? '') . ':' . ((int)($host['port'] ?? 9000) ?: 9000);
|
|
}
|
|
|
|
private static function hostCredentials(array $host): array
|
|
{
|
|
return [
|
|
'username' => (string)($host['username'] ?? ''),
|
|
'password' => replication_secret_box::decrypt($host['password_secret'] ?? ''),
|
|
'admin_username' => (string)($host['admin_username'] ?? ''),
|
|
'admin_password' => replication_secret_box::decrypt($host['admin_password_secret'] ?? ''),
|
|
];
|
|
}
|
|
|
|
private static function activePassword(array $config, string $secretKey, string $plainKey): string
|
|
{
|
|
if (!empty($config[$secretKey])) {
|
|
return replication_secret_box::decrypt((string)$config[$secretKey]);
|
|
}
|
|
|
|
return (string)($config[$plainKey] ?? '');
|
|
}
|
|
|
|
private static function hostStatus(array $host): array
|
|
{
|
|
if (isset($host['last_status']) && is_array($host['last_status'])) {
|
|
return $host['last_status'];
|
|
}
|
|
|
|
return self::jsonDecode($host['last_status_json'] ?? null);
|
|
}
|
|
|
|
private static function hostCheckedAt(array $host, array $status): ?int
|
|
{
|
|
$raw = $host['last_checked_at'] ?? $status['checked_at'] ?? null;
|
|
if (!is_string($raw) || trim($raw) === '') {
|
|
return null;
|
|
}
|
|
|
|
$timestamp = strtotime($raw);
|
|
return $timestamp === false ? null : $timestamp;
|
|
}
|
|
|
|
private static function boolValue(mixed $value): bool
|
|
{
|
|
if (is_bool($value)) {
|
|
return $value;
|
|
}
|
|
|
|
return in_array(strtolower(trim((string)$value)), ['1', 'true', 'yes', 'on'], true);
|
|
}
|
|
|
|
private static function jsonDecode(mixed $value): array
|
|
{
|
|
if (!is_string($value) || trim($value) === '') {
|
|
return [];
|
|
}
|
|
|
|
$decoded = json_decode($value, true);
|
|
return is_array($decoded) ? $decoded : [];
|
|
}
|
|
}
|