false, 'database_enabled' => false, 'redis_enabled' => false, 'minio_enabled' => false, 'max_status_age_seconds' => self::DEFAULT_MAX_STATUS_AGE_SECONDS, ]; } public static function normalizeConfig(array $config): array { $normalized = self::configDefaults(); foreach (['enabled', 'database_enabled', 'redis_enabled', 'minio_enabled'] as $key) { if (array_key_exists($key, $config)) { $normalized[$key] = self::boolValue($config[$key]); } } if (array_key_exists('max_status_age_seconds', $config)) { $normalized['max_status_age_seconds'] = max(1, (int)$config['max_status_age_seconds']); } return $normalized; } public static function kindEnabled(array $config, string $kind): bool { $config = self::normalizeConfig($config); return $config['enabled'] && !empty($config[$kind . '_enabled']); } public static function snapshotHostIsStrictlyFresh(array $host, int $maxAgeSeconds, ?int $now = null): bool { if (($host['role'] ?? '') !== 'replica') { return false; } if (!empty($host['deleted_at'])) { return false; } $status = self::hostStatus($host); if (($status['status'] ?? '') !== 'ok') { return false; } if (round((float)($status['replication_percent'] ?? 0), 2) < 100.0) { return false; } $blockers = $status['blockers'] ?? []; if (is_array($blockers) && $blockers !== []) { return false; } $checkedAt = self::hostCheckedAt($host, $status); if ($checkedAt === null) { return false; } return (($now ?? time()) - $checkedAt) <= max(1, $maxAgeSeconds); } public static function snapshotFailoverCandidate(array $hosts, string $kind, int $maxAgeSeconds, ?int $now = null): ?array { $eligible = array_values(array_filter( $hosts, static fn(array $host): bool => ($host['kind'] ?? '') === $kind && self::snapshotHostIsStrictlyFresh($host, $maxAgeSeconds, $now) )); if ($eligible === []) { return null; } usort($eligible, static function (array $a, array $b) use ($now): int { $aChecked = self::hostCheckedAt($a, self::hostStatus($a)) ?? 0; $bChecked = self::hostCheckedAt($b, self::hostStatus($b)) ?? 0; if ($aChecked === $bChecked) { return (int)($a['id'] ?? 0) <=> (int)($b['id'] ?? 0); } return $bChecked <=> $aChecked; }); return $eligible[0]; } public static function activeConfigFromHost(string $kind, array $host): ?array { if ($kind === self::KIND_DATABASE) { $database = trim((string)($host['database_name'] ?? $host['database'] ?? '')); $user = trim((string)($host['username'] ?? $host['user'] ?? '')); if ($database === '' || $user === '') { return null; } return [ 'id' => isset($host['id']) ? (int)$host['id'] : null, 'host' => (string)($host['host'] ?? ''), 'port' => (int)($host['port'] ?? 3306) ?: 3306, 'database' => $database, 'user' => $user, 'password_secret' => (string)($host['password_secret'] ?? ''), 'ssl_mode' => (string)($host['ssl_mode'] ?? 'DISABLED'), ]; } if ($kind === self::KIND_REDIS) { return [ 'id' => isset($host['id']) ? (int)$host['id'] : null, 'host' => (string)($host['host'] ?? ''), 'port' => (int)($host['port'] ?? 6379) ?: 6379, 'database' => (int)($host['database_index'] ?? $host['database'] ?? 0), 'user' => (string)($host['username'] ?? $host['user'] ?? ''), 'password_secret' => (string)($host['password_secret'] ?? ''), ]; } if ($kind === self::KIND_MINIO) { $options = self::jsonDecode($host['options_json'] ?? null); return [ 'id' => isset($host['id']) ? (int)$host['id'] : null, 'endpoint' => self::minioEndpoint($host, $options), 'access_key' => (string)($host['username'] ?? $host['access_key'] ?? ''), 'secret_key_secret' => (string)($host['password_secret'] ?? ''), 'buckets' => is_array($options['buckets'] ?? null) ? array_values($options['buckets']) : [], ]; } return null; } public static function applyStartupFailoverFromSnapshot(?string $path = null, array $probes = []): array { $snapshot = replication_bootstrap_config::loadSnapshot($path); $failover = is_array($snapshot['failover'] ?? null) ? $snapshot['failover'] : []; $config = self::normalizeConfig(is_array($failover['config'] ?? null) ? $failover['config'] : $failover); $active = is_array($snapshot['active'] ?? null) ? $snapshot['active'] : []; $hostGroups = is_array($failover['hosts'] ?? null) ? $failover['hosts'] : []; $maxAgeSeconds = (int)$config['max_status_age_seconds']; $summary = []; $changed = false; $primaryDown = $probes['primary_down'] ?? [self::class, 'activePrimaryIsDown']; $candidateReachable = $probes['candidate_reachable'] ?? [self::class, 'candidateReachable']; $promoteCandidate = $probes['promote_candidate'] ?? [self::class, 'promoteCandidate']; foreach ([self::KIND_DATABASE, self::KIND_REDIS, self::KIND_MINIO] as $kind) { if (!self::kindEnabled($config, $kind)) { $summary[$kind] = ['status' => 'skipped', 'reason' => 'disabled']; continue; } if (!is_array($active[$kind] ?? null)) { $summary[$kind] = ['status' => 'skipped', 'reason' => 'missing_active_primary']; continue; } try { if (!call_user_func($primaryDown, $kind, $active[$kind], $snapshot)) { $summary[$kind] = ['status' => 'skipped', 'reason' => 'primary_healthy']; continue; } $hosts = is_array($hostGroups[$kind] ?? null) ? $hostGroups[$kind] : []; $candidate = self::snapshotFailoverCandidate($hosts, $kind, $maxAgeSeconds); if ($candidate === null) { $summary[$kind] = ['status' => 'skipped', 'reason' => 'no_fresh_caught_up_replica']; continue; } if (!call_user_func($candidateReachable, $kind, $candidate)) { $summary[$kind] = [ 'status' => 'skipped', 'reason' => 'candidate_unreachable', 'candidate_id' => (int)($candidate['id'] ?? 0), ]; continue; } call_user_func($promoteCandidate, $kind, $candidate); $candidateActive = self::activeConfigFromHost($kind, $candidate); if ($candidateActive === null) { $summary[$kind] = [ 'status' => 'skipped', 'reason' => 'candidate_missing_active_config', 'candidate_id' => (int)($candidate['id'] ?? 0), ]; continue; } $snapshot['active'][$kind] = $candidateActive; $pending = is_array($snapshot['pending_failovers'] ?? null) ? $snapshot['pending_failovers'] : []; $pending[] = [ 'kind' => $kind, 'host_id' => (int)($candidate['id'] ?? 0), 'label' => (string)($candidate['label'] ?? ''), 'source' => 'startup_snapshot', 'promoted_at' => date('c'), ]; $snapshot['pending_failovers'] = $pending; $summary[$kind] = [ 'status' => 'promoted', 'candidate_id' => (int)($candidate['id'] ?? 0), ]; $changed = true; } catch (Throwable $throwable) { $summary[$kind] = [ 'status' => 'failed', 'reason' => $throwable->getMessage(), ]; } } if ($changed) { $snapshot['generated_at'] = date('c'); replication_bootstrap_config::writeSnapshot($snapshot, $path); if ($path === null) { replication_bootstrap_config::applyToGlobals($snapshot); } } return [ 'changed' => $changed, 'results' => $summary, ]; } public static function activePrimaryIsDown(string $kind, array $activeConfig, array $snapshot = []): bool { try { match ($kind) { self::KIND_DATABASE => self::probeActiveDatabase($activeConfig), self::KIND_REDIS => self::probeActiveRedis($activeConfig), self::KIND_MINIO => self::probeActiveMinio($activeConfig), default => null, }; return false; } catch (Throwable) { return true; } } public static function candidateReachable(string $kind, array $host): bool { try { match ($kind) { self::KIND_DATABASE => self::probeHostDatabase($host), self::KIND_REDIS => self::probeHostRedis($host), self::KIND_MINIO => self::probeHostMinio($host), default => null, }; return true; } catch (Throwable) { return false; } } public static function promoteCandidate(string $kind, array $host): void { match ($kind) { self::KIND_DATABASE => self::promoteDatabaseCandidate($host), self::KIND_REDIS => self::promoteRedisCandidate($host), self::KIND_MINIO => self::probeHostMinio($host), default => null, }; } private static function probeActiveDatabase(array $config): void { $host = (string)($config['host'] ?? ''); $user = (string)($config['user'] ?? ''); $database = (string)($config['database'] ?? ''); $password = self::activePassword($config, 'password_secret', 'password'); self::connectMysqli($host, $user, $password, $database, (int)($config['port'] ?? 3306))->close(); } private static function probeHostDatabase(array $host): void { $credentials = self::hostCredentials($host); $connection = self::connectMysqli( (string)($host['host'] ?? ''), $credentials['username'], $credentials['password'], (string)($host['database_name'] ?? ''), (int)($host['port'] ?? 3306) ); $connection->close(); } private static function promoteDatabaseCandidate(array $host): void { $credentials = self::hostCredentials($host); $user = $credentials['admin_username'] !== '' ? $credentials['admin_username'] : $credentials['username']; $password = $credentials['admin_password'] !== '' ? $credentials['admin_password'] : $credentials['password']; $connection = self::connectMysqli( (string)($host['host'] ?? ''), $user, $password, (string)($host['database_name'] ?? ''), (int)($host['port'] ?? 3306) ); try { foreach (['STOP REPLICA', 'STOP SLAVE'] as $statement) { try { $connection->query($statement); break; } catch (Throwable) { } } foreach (['SET GLOBAL super_read_only = OFF', 'SET GLOBAL read_only = OFF'] as $statement) { try { $connection->query($statement); } catch (Throwable) { } } } finally { $connection->close(); } } private static function probeActiveRedis(array $config): void { self::redisClientFromConfig([ 'host' => (string)($config['host'] ?? ''), 'port' => (int)($config['port'] ?? 6379), 'database' => (int)($config['database'] ?? 0), 'user' => (string)($config['user'] ?? ''), 'password' => self::activePassword($config, 'password_secret', 'password'), ])->ping(); } private static function probeHostRedis(array $host): void { self::redisClientFromHost($host)->ping(); } private static function promoteRedisCandidate(array $host): void { $client = self::redisClientFromHost($host); $client->executeRaw(['REPLICAOF', 'NO', 'ONE']); try { $client->executeRaw(['CONFIG', 'REWRITE']); } catch (Throwable) { } } private static function probeActiveMinio(array $config): void { self::minioClientFromConfig([ 'endpoint' => (string)($config['endpoint'] ?? ''), 'access_key' => (string)($config['access_key'] ?? $config['user'] ?? ''), 'secret_key' => self::activePassword($config, 'secret_key_secret', 'secret_key'), ])->listBuckets(); } private static function probeHostMinio(array $host): void { self::minioClientFromHost($host)->listBuckets(); } private static function connectMysqli(string $host, string $user, string $password, string $database, int $port): mysqli { mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT); $connection = mysqli_init(); $connection->options(MYSQLI_OPT_CONNECT_TIMEOUT, 2); $connection->real_connect($host, $user, $password, $database, $port ?: 3306); $connection->set_charset('utf8mb4'); return $connection; } private static function redisClientFromHost(array $host): PredisClient { $credentials = self::hostCredentials($host); return self::redisClientFromConfig([ 'host' => (string)($host['host'] ?? ''), 'port' => (int)($host['port'] ?? 6379), 'database' => (int)($host['database_index'] ?? 0), 'user' => $credentials['username'], 'password' => $credentials['password'], ]); } private static function redisClientFromConfig(array $config): PredisClient { $params = [ 'scheme' => 'tcp', 'host' => (string)$config['host'], 'port' => (int)$config['port'], 'database' => (int)$config['database'], 'password' => (string)$config['password'], 'timeout' => 2.0, 'read_write_timeout' => 2.0, ]; if (($config['user'] ?? '') !== '' && $config['user'] !== 'default') { $params['username'] = (string)$config['user']; } return new PredisClient($params); } private static function minioClientFromHost(array $host): S3Client { $credentials = self::hostCredentials($host); $options = self::jsonDecode($host['options_json'] ?? null); return self::minioClientFromConfig([ 'endpoint' => self::minioEndpoint($host, $options), 'access_key' => $credentials['username'], 'secret_key' => $credentials['password'], ]); } private static function minioClientFromConfig(array $config): S3Client { return new S3Client([ 'version' => 'latest', 'region' => 'us-east-1', 'endpoint' => (string)$config['endpoint'], 'use_path_style_endpoint' => true, 'credentials' => [ 'key' => (string)$config['access_key'], 'secret' => (string)$config['secret_key'], ], 'http' => [ 'connect_timeout' => 2, 'timeout' => 2, ], ]); } private static function minioEndpoint(array $host, array $options): string { $endpoint = trim((string)($options['endpoint'] ?? '')); if ($endpoint !== '') { return $endpoint; } $scheme = strtolower(trim((string)($options['scheme'] ?? 'http'))); if ($scheme !== 'https') { $scheme = 'http'; } return $scheme . '://' . (string)($host['host'] ?? '') . ':' . ((int)($host['port'] ?? 9000) ?: 9000); } private static function hostCredentials(array $host): array { return [ 'username' => (string)($host['username'] ?? ''), 'password' => replication_secret_box::decrypt($host['password_secret'] ?? ''), 'admin_username' => (string)($host['admin_username'] ?? ''), 'admin_password' => replication_secret_box::decrypt($host['admin_password_secret'] ?? ''), ]; } private static function activePassword(array $config, string $secretKey, string $plainKey): string { if (!empty($config[$secretKey])) { return replication_secret_box::decrypt((string)$config[$secretKey]); } return (string)($config[$plainKey] ?? ''); } private static function hostStatus(array $host): array { if (isset($host['last_status']) && is_array($host['last_status'])) { return $host['last_status']; } return self::jsonDecode($host['last_status_json'] ?? null); } private static function hostCheckedAt(array $host, array $status): ?int { $raw = $host['last_checked_at'] ?? $status['checked_at'] ?? null; if (!is_string($raw) || trim($raw) === '') { return null; } $timestamp = strtotime($raw); return $timestamp === false ? null : $timestamp; } private static function boolValue(mixed $value): bool { if (is_bool($value)) { return $value; } return in_array(strtolower(trim((string)$value)), ['1', 'true', 'yes', 'on'], true); } private static function jsonDecode(mixed $value): array { if (!is_string($value) || trim($value) === '') { return []; } $decoded = json_decode($value, true); return is_array($decoded) ? $decoded : []; } }