6523 lines
258 KiB
PHP
6523 lines
258 KiB
PHP
<?php
|
|
|
|
namespace classes;
|
|
|
|
use Aws\S3\S3Client;
|
|
use mysqli;
|
|
use Predis\Client as PredisClient;
|
|
use RuntimeException;
|
|
use Throwable;
|
|
|
|
class replication_manager
|
|
{
|
|
private const KIND_DATABASE = 'database';
|
|
private const KIND_REDIS = 'redis';
|
|
private const KIND_MINIO = 'minio';
|
|
private const MINIO_DEFAULT_BUCKETS = ['attachments', 'backups', 'invoices', 'pdfs', 'uploads', 'truckwashdev'];
|
|
private const MINIO_BACKUP_BUCKET = 'backups';
|
|
private const MINIO_BACKUP_REPLICA_RETENTION_DAYS = 30;
|
|
private const MINIO_BACKUP_REPLICA_RETENTION_RULE_ID = 'truckwash-replica-backup-retention-30-days';
|
|
private const MINIO_DEFAULT_REPLICATION_TRANSFER_LIMIT = '25Mi';
|
|
private const MINIO_PROGRESS_SCAN_INTERVAL_SECONDS = 30;
|
|
private const MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER = 'MinIO replica has not caught up.';
|
|
private const MINIO_CATCH_UP_PENDING_BYTES_TOLERANCE = 26214400;
|
|
private const MINIO_CATCH_UP_PENDING_OBJECTS_TOLERANCE = 100;
|
|
private const MINIO_INCOMPLETE_PROGRESS_MAX_PERCENT = 99.9;
|
|
private const MINIO_S3_CONNECT_TIMEOUT_SECONDS = 2;
|
|
private const MINIO_S3_REQUEST_TIMEOUT_SECONDS = 5;
|
|
private const MINIO_MC_COMMAND_TIMEOUT_SECONDS = 8;
|
|
private const MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS = 8;
|
|
private const MINIO_SPACE_HEADROOM_PERCENT = 20.0;
|
|
private const MINIO_MC_DOWNLOAD_BASE_URL = 'https://dl.min.io/client/mc/release';
|
|
private const MARIADB_SEED_BATCH_ROWS = 500;
|
|
private const MARIADB_SEED_STEP_SECONDS = 3;
|
|
private const MARIADB_SEED_FREEZE_TTL_SECONDS = 900;
|
|
private const MARIADB_SCHEMA_ONLY_TABLES = [
|
|
'logs',
|
|
'edge_gateway_log_entries',
|
|
'edge_gateway_audit_logs',
|
|
'edge_gateway_shell_sessions',
|
|
'replication_status_snapshots',
|
|
'replication_operations',
|
|
'replication_audit_logs',
|
|
'system_search_documents',
|
|
];
|
|
|
|
public function __construct()
|
|
{
|
|
replication_schema_bootstrap::ensureTables();
|
|
}
|
|
|
|
public function summary(bool $refresh = false): array
|
|
{
|
|
$this->ensureEnvironmentPrimaryRows();
|
|
if ($refresh) {
|
|
$this->refreshStatuses();
|
|
}
|
|
|
|
$databaseHosts = $this->listHosts(self::KIND_DATABASE);
|
|
$redisHosts = $this->listHosts(self::KIND_REDIS);
|
|
$minioHosts = $this->listHosts(self::KIND_MINIO);
|
|
|
|
return [
|
|
'generated_at' => date('c'),
|
|
'database' => [
|
|
'primary' => $this->publicHost($this->primaryHost(self::KIND_DATABASE)),
|
|
'hosts' => array_map(fn(array $host): array => $this->publicHost($host), $databaseHosts),
|
|
'replication' => $this->buildReplicationSummary(self::KIND_DATABASE, $databaseHosts),
|
|
],
|
|
'redis' => [
|
|
'primary' => $this->publicHost($this->primaryHost(self::KIND_REDIS)),
|
|
'hosts' => array_map(fn(array $host): array => $this->publicHost($host), $redisHosts),
|
|
'replication' => $this->buildReplicationSummary(self::KIND_REDIS, $redisHosts),
|
|
],
|
|
'minio' => [
|
|
'primary' => $this->publicHost($this->primaryHost(self::KIND_MINIO)),
|
|
'hosts' => array_map(fn(array $host): array => $this->publicHost($host), $minioHosts),
|
|
'replication' => $this->buildReplicationSummary(self::KIND_MINIO, $minioHosts),
|
|
],
|
|
'write_freeze' => application_write_freeze::state(),
|
|
];
|
|
}
|
|
|
|
public function dependencyReplication(string $kind): array
|
|
{
|
|
$kind = self::normalizeKind($kind);
|
|
$this->ensureEnvironmentPrimaryRows();
|
|
return $this->buildReplicationSummary($kind, $this->listHosts($kind));
|
|
}
|
|
|
|
public function addHost(string $kind, array $input, ?int $actorUserId = null): array
|
|
{
|
|
$kind = self::normalizeKind($kind);
|
|
$host = $this->normalizeHostInput($kind, $input);
|
|
|
|
$this->execute(
|
|
"INSERT INTO replication_hosts (
|
|
kind, label, host, port, database_name, database_index, username, password_secret,
|
|
admin_username, admin_password_secret, replication_username, replication_password_secret,
|
|
role, status, ssl_mode, options_json
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'replica', 'unknown', ?, ?)",
|
|
'sssissssssssss',
|
|
[
|
|
$kind,
|
|
$host['label'],
|
|
$host['host'],
|
|
$host['port'],
|
|
$host['database_name'],
|
|
$host['database_index'],
|
|
$host['username'],
|
|
$host['password_secret'],
|
|
$host['admin_username'],
|
|
$host['admin_password_secret'],
|
|
$host['replication_username'],
|
|
$host['replication_password_secret'],
|
|
$host['ssl_mode'],
|
|
self::jsonEncode($host['options']),
|
|
]
|
|
);
|
|
|
|
$id = $this->insertId();
|
|
$this->audit($kind, $id, 'host_added', $actorUserId, 'info', [
|
|
'label' => $host['label'],
|
|
'host' => $host['host'],
|
|
'port' => $host['port'],
|
|
]);
|
|
$this->writeBootstrapSnapshot();
|
|
|
|
return $this->publicHost($this->getHost($kind, $id));
|
|
}
|
|
|
|
public function testHost(string $kind, int $id, ?int $actorUserId = null): array
|
|
{
|
|
$kind = self::normalizeKind($kind);
|
|
$host = $this->getHost($kind, $id);
|
|
|
|
$status = match ($kind) {
|
|
self::KIND_DATABASE => $this->testDatabaseHost($host),
|
|
self::KIND_REDIS => $this->testRedisHost($host),
|
|
self::KIND_MINIO => $this->testMinioHost($host),
|
|
};
|
|
|
|
$this->storeStatus($host, $status);
|
|
$this->writeBootstrapSnapshot();
|
|
$this->audit($kind, $id, 'host_tested', $actorUserId, $status['blockers'] === [] ? 'info' : 'warning', [
|
|
'status' => $status['status'],
|
|
'replication_percent' => $status['replication_percent'],
|
|
'blockers' => $status['blockers'],
|
|
]);
|
|
|
|
return [
|
|
'host' => $this->publicHost($this->getHost($kind, $id)),
|
|
'status' => $status,
|
|
];
|
|
}
|
|
|
|
public function testCredentials(string $kind, array $input): array
|
|
{
|
|
$kind = self::normalizeKind($kind);
|
|
$host = $this->transientHost($kind, $input);
|
|
$options = $this->decodeOptions($host);
|
|
if ($kind === self::KIND_DATABASE && !empty($options['allow_preseeded_replica'])) {
|
|
$host['connect_without_database'] = true;
|
|
}
|
|
$status = match ($kind) {
|
|
self::KIND_DATABASE => $this->testDatabaseHost($host),
|
|
self::KIND_REDIS => $this->testRedisHost($host),
|
|
self::KIND_MINIO => $this->testMinioHost(array_merge($host, [
|
|
'test_connectivity_only' => true,
|
|
'skip_storage_scan' => true,
|
|
])),
|
|
};
|
|
|
|
if ($kind === self::KIND_DATABASE
|
|
&& ($host['role'] ?? '') !== 'primary'
|
|
&& !empty($options['allow_preseeded_replica'])
|
|
&& $status['status'] !== 'down') {
|
|
try {
|
|
$target = $this->databaseConnection($host, true);
|
|
try {
|
|
$seedBlockers = $this->databaseReplicaSeedBlockers($this->primaryHost(self::KIND_DATABASE), $host, $target);
|
|
} finally {
|
|
$target->close();
|
|
}
|
|
} catch (Throwable $throwable) {
|
|
$seedBlockers = [$throwable->getMessage()];
|
|
}
|
|
|
|
if ($seedBlockers !== []) {
|
|
$status['blockers'] = array_values(array_unique(array_merge($status['blockers'], $seedBlockers)));
|
|
$status['status'] = 'degraded';
|
|
if ((float)$status['replication_percent'] >= 100.0) {
|
|
$status['replication_percent'] = 99.99;
|
|
}
|
|
}
|
|
}
|
|
|
|
return [
|
|
'ok' => $status['status'] === 'ok',
|
|
'host' => $this->publicHost($host),
|
|
'status' => $status,
|
|
];
|
|
}
|
|
|
|
public function provisionHost(
|
|
string $kind,
|
|
int $id,
|
|
?int $actorUserId = null,
|
|
bool $deferCoolifyManagedMinio = false
|
|
): array
|
|
{
|
|
$kind = self::normalizeKind($kind);
|
|
$host = $this->getHost($kind, $id);
|
|
$operationId = $this->activeOperationId($kind, $id, 'provision')
|
|
?? $this->startOperation($kind, $id, 'provision', $actorUserId);
|
|
|
|
if ($deferCoolifyManagedMinio && $this->shouldDeferCoolifyManagedMinioProvision($kind, $host)) {
|
|
return $this->deferCoolifyManagedMinioProvision($host, $operationId);
|
|
}
|
|
|
|
try {
|
|
$result = match ($kind) {
|
|
self::KIND_DATABASE => $this->provisionDatabaseHost($host, $operationId),
|
|
self::KIND_REDIS => $this->provisionRedisHost($host, $operationId),
|
|
self::KIND_MINIO => $this->provisionMinioHost($host, $operationId),
|
|
};
|
|
|
|
if (($result['operation']['status'] ?? null) === 'running') {
|
|
$this->audit($kind, $id, 'host_provision_progress', $actorUserId, 'info', $result);
|
|
return $result;
|
|
}
|
|
|
|
$status = $result['ok'] ? 'completed' : 'blocked';
|
|
$this->finishOperation($operationId, $status, (float)($result['replication_percent'] ?? 0), $result['message'] ?? null, $result['blockers'] ?? []);
|
|
$this->audit($kind, $id, 'host_provisioned', $actorUserId, $result['ok'] ? 'info' : 'warning', $result);
|
|
return $result;
|
|
} catch (Throwable $throwable) {
|
|
$this->finishOperation($operationId, 'failed', 0, null, [$throwable->getMessage()]);
|
|
$this->audit($kind, $id, 'host_provision_failed', $actorUserId, 'error', ['error' => $throwable->getMessage()]);
|
|
throw $throwable;
|
|
}
|
|
}
|
|
|
|
private function shouldDeferCoolifyManagedMinioProvision(string $kind, array $host): bool
|
|
{
|
|
if ($kind !== self::KIND_MINIO) {
|
|
return false;
|
|
}
|
|
|
|
$options = $this->decodeOptions($host);
|
|
return (string)($options['deployment_provider'] ?? '') === 'coolify'
|
|
|| isset($options['coolify_target_id'])
|
|
|| isset($options['coolify_instance_id']);
|
|
}
|
|
|
|
private function deferCoolifyManagedMinioProvision(array $host, int $operationId): array
|
|
{
|
|
$lastStatus = self::sanitizePublicLastStatus($host, self::jsonDecode($host['last_status_json'] ?? null));
|
|
$progress = 45.0;
|
|
if (is_array($lastStatus) && is_numeric($lastStatus['replication_percent'] ?? null)) {
|
|
$progress = max($progress, self::minioIncompleteProgress((float)$lastStatus['replication_percent']));
|
|
}
|
|
|
|
$message = 'MinIO provisioning was queued for Coolify background maintenance.';
|
|
$this->updateOperationProgress($operationId, $progress, $message, [
|
|
'phase' => 'coolify_deferred',
|
|
'queued_at' => date('c'),
|
|
]);
|
|
$this->execute(
|
|
"UPDATE replication_hosts SET status = 'provisioning' WHERE id = ? AND kind = ?",
|
|
'is',
|
|
[(int)$host['id'], self::KIND_MINIO]
|
|
);
|
|
|
|
$status = [
|
|
'status' => 'provisioning',
|
|
'replication_percent' => $progress,
|
|
'lag_seconds' => null,
|
|
'blockers' => [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER],
|
|
'raw' => [
|
|
'progress_source' => 'coolify_deferred',
|
|
'previous_status' => is_array($lastStatus) ? [
|
|
'status' => $lastStatus['status'] ?? null,
|
|
'replication_percent' => $lastStatus['replication_percent'] ?? null,
|
|
'checked_at' => $lastStatus['checked_at'] ?? null,
|
|
] : null,
|
|
],
|
|
'checked_at' => date('c'),
|
|
];
|
|
$this->storeStatus($this->getHost(self::KIND_MINIO, (int)$host['id']), $status);
|
|
|
|
return [
|
|
'ok' => true,
|
|
'message' => $message,
|
|
'blockers' => $status['blockers'],
|
|
'replication_percent' => $progress,
|
|
'operation' => [
|
|
'id' => $operationId,
|
|
'status' => 'running',
|
|
'progress_percent' => $progress,
|
|
'message' => $message,
|
|
],
|
|
'host' => $this->publicHost($this->getHost(self::KIND_MINIO, (int)$host['id'])),
|
|
];
|
|
}
|
|
|
|
public function promoteHost(string $kind, int $id, ?int $actorUserId = null): array
|
|
{
|
|
$kind = self::normalizeKind($kind);
|
|
$host = $this->getHost($kind, $id);
|
|
if (($host['role'] ?? '') === 'primary') {
|
|
return [
|
|
'ok' => true,
|
|
'message' => 'Host is already primary.',
|
|
'host' => $this->publicHost($host),
|
|
'blockers' => [],
|
|
];
|
|
}
|
|
|
|
$operationId = $this->startOperation($kind, $id, 'promote', $actorUserId);
|
|
$owner = 'replication-promote-' . $kind . '-' . $id . '-' . bin2hex(random_bytes(4));
|
|
$lockHandle = $this->acquirePromotionLock();
|
|
|
|
try {
|
|
application_write_freeze::freeze('Replication promotion in progress.', $owner, 600);
|
|
$result = match ($kind) {
|
|
self::KIND_DATABASE => $this->promoteDatabaseHost($host),
|
|
self::KIND_REDIS => $this->promoteRedisHost($host),
|
|
self::KIND_MINIO => $this->promoteMinioHost($host),
|
|
};
|
|
|
|
$this->finishOperation($operationId, 'completed', 100, $result['message'] ?? null, []);
|
|
$this->audit($kind, $id, 'host_promoted', $actorUserId, 'critical', $result);
|
|
return $result;
|
|
} catch (Throwable $throwable) {
|
|
$this->finishOperation($operationId, 'failed', 0, null, [$throwable->getMessage()]);
|
|
$this->audit($kind, $id, 'host_promotion_failed', $actorUserId, 'error', ['error' => $throwable->getMessage()]);
|
|
throw $throwable;
|
|
} finally {
|
|
application_write_freeze::unfreeze($owner);
|
|
$this->releasePromotionLock($lockHandle);
|
|
}
|
|
}
|
|
|
|
public function runAutomaticFailoverMonitor(?int $actorUserId = null): array
|
|
{
|
|
$this->ensureEnvironmentPrimaryRows();
|
|
$config = $this->failoverConfigForSnapshot();
|
|
$results = [];
|
|
|
|
foreach ([self::KIND_DATABASE, self::KIND_REDIS, self::KIND_MINIO] as $kind) {
|
|
$results[$kind] = $this->runAutomaticFailoverForKind($kind, $config, $actorUserId);
|
|
}
|
|
|
|
try {
|
|
$this->refreshStatuses();
|
|
} catch (Throwable $throwable) {
|
|
$this->audit(self::KIND_DATABASE, null, 'automatic_failover_status_refresh_failed', $actorUserId, 'warning', [
|
|
'error' => $throwable->getMessage(),
|
|
]);
|
|
}
|
|
|
|
$this->writeBootstrapSnapshot();
|
|
|
|
return [
|
|
'ok' => true,
|
|
'config' => $config,
|
|
'results' => $results,
|
|
];
|
|
}
|
|
|
|
public function syncStartupFailoversFromSnapshot(?int $actorUserId = null): array
|
|
{
|
|
$snapshot = replication_bootstrap_config::loadSnapshot();
|
|
$pending = is_array($snapshot['pending_failovers'] ?? null) ? $snapshot['pending_failovers'] : [];
|
|
$synced = [];
|
|
|
|
foreach ($pending as $entry) {
|
|
if (!is_array($entry)) {
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
$kind = self::normalizeKind((string)($entry['kind'] ?? ''));
|
|
$hostId = (int)($entry['host_id'] ?? 0);
|
|
if ($hostId <= 0) {
|
|
continue;
|
|
}
|
|
|
|
$currentPrimary = $this->primaryHost($kind);
|
|
if ($currentPrimary !== null && (int)$currentPrimary['id'] !== $hostId) {
|
|
$this->switchPrimary($kind, $hostId, (int)$currentPrimary['id']);
|
|
}
|
|
|
|
$this->audit($kind, $hostId, 'startup_failover_synced', $actorUserId, 'critical', $entry);
|
|
$synced[] = [
|
|
'kind' => $kind,
|
|
'host_id' => $hostId,
|
|
];
|
|
} catch (Throwable $throwable) {
|
|
$this->audit((string)($entry['kind'] ?? self::KIND_DATABASE), null, 'startup_failover_sync_failed', $actorUserId, 'error', [
|
|
'entry' => $entry,
|
|
'error' => $throwable->getMessage(),
|
|
]);
|
|
}
|
|
}
|
|
|
|
if ($pending !== []) {
|
|
$snapshot['pending_failovers'] = [];
|
|
replication_bootstrap_config::writeSnapshot($snapshot);
|
|
$this->writeBootstrapSnapshot();
|
|
}
|
|
|
|
return $synced;
|
|
}
|
|
|
|
public function removeHost(string $kind, int $id, ?int $actorUserId = null, bool $removeLinkedCoolifyTargets = true): array
|
|
{
|
|
$kind = self::normalizeKind($kind);
|
|
$host = $this->getHost($kind, $id);
|
|
$canRemove = self::replicationHostCanBeRemoved($host);
|
|
if (!$canRemove && class_exists(coolify_manager::class)) {
|
|
$canRemove = coolify_manager::replicationHostCanBeRemoved($host);
|
|
}
|
|
if (!$canRemove) {
|
|
if (($host['role'] ?? '') === 'primary') {
|
|
throw new RuntimeException('Primary hosts cannot be removed. Promote a healthy replica first.');
|
|
}
|
|
throw new RuntimeException('Only inactive prior hosts or unhealthy replicas can be removed.');
|
|
}
|
|
$this->execute(
|
|
"UPDATE replication_hosts SET deleted_at = NOW(), status = 'removed' WHERE id = ? AND kind = ?",
|
|
'is',
|
|
[$id, $kind]
|
|
);
|
|
$this->audit($kind, $id, 'host_removed', $actorUserId, 'warning', [
|
|
'label' => $host['label'] ?? '',
|
|
'host' => $host['host'] ?? '',
|
|
]);
|
|
if ($removeLinkedCoolifyTargets && class_exists(coolify_manager::class)) {
|
|
coolify_manager::markTargetsRemovedForReplicationHost($id, $actorUserId);
|
|
}
|
|
$this->writeBootstrapSnapshot();
|
|
|
|
return [
|
|
'ok' => true,
|
|
'message' => 'Replication host removed.',
|
|
'id' => $id,
|
|
'kind' => $kind,
|
|
];
|
|
}
|
|
|
|
public function renameHost(string $kind, int $id, array $input, ?int $actorUserId = null): array
|
|
{
|
|
$kind = self::normalizeKind($kind);
|
|
$host = $this->getHost($kind, $id);
|
|
$label = trim((string)($input['label'] ?? $input['name'] ?? ''));
|
|
if ($label === '') {
|
|
throw new RuntimeException('Replication host label is required.');
|
|
}
|
|
if (mb_strlen($label) > 128) {
|
|
throw new RuntimeException('Replication host label must be 128 characters or fewer.');
|
|
}
|
|
|
|
$oldLabel = (string)($host['label'] ?? '');
|
|
if ($label !== $oldLabel) {
|
|
$this->execute(
|
|
"UPDATE replication_hosts SET label = ? WHERE id = ? AND kind = ? AND deleted_at IS NULL",
|
|
'sis',
|
|
[$label, $id, $kind]
|
|
);
|
|
if (class_exists(coolify_manager::class)) {
|
|
coolify_manager::syncLabelForReplicationHost($id, $label);
|
|
}
|
|
$this->audit($kind, $id, 'host_renamed', $actorUserId, 'info', [
|
|
'old_label' => $oldLabel,
|
|
'new_label' => $label,
|
|
]);
|
|
$this->writeBootstrapSnapshot();
|
|
}
|
|
|
|
return $this->publicHost($this->getHost($kind, $id));
|
|
}
|
|
|
|
public static function replicationHostCanBeRemoved(array $host): bool
|
|
{
|
|
$role = (string)($host['role'] ?? '');
|
|
if ($role === 'primary') {
|
|
return false;
|
|
}
|
|
if ($role === 'inactive') {
|
|
return true;
|
|
}
|
|
|
|
$status = (string)($host['status'] ?? 'unknown');
|
|
return $role === 'replica' && in_array($status, ['degraded', 'down', 'unknown', 'not_configured'], true);
|
|
}
|
|
|
|
public static function composeTemplate(array $input): array
|
|
{
|
|
$kind = self::normalizeKind((string)($input['kind'] ?? self::KIND_DATABASE));
|
|
$role = self::normalizeComposeRole((string)($input['role'] ?? 'replica'));
|
|
|
|
return match ($kind) {
|
|
self::KIND_DATABASE => self::databaseComposeTemplate($input, $role),
|
|
self::KIND_REDIS => self::redisComposeTemplate($input, $role),
|
|
self::KIND_MINIO => self::minioComposeTemplate($input, $role),
|
|
};
|
|
}
|
|
|
|
public static function normalizeKind(string $kind): string
|
|
{
|
|
$kind = strtolower(trim($kind));
|
|
if (in_array($kind, ['database', 'databases', 'mysql', 'db'], true)) {
|
|
return self::KIND_DATABASE;
|
|
}
|
|
if ($kind === self::KIND_REDIS) {
|
|
return self::KIND_REDIS;
|
|
}
|
|
if (in_array($kind, ['minio', 's3', 'object-storage', 'object_storage'], true)) {
|
|
return self::KIND_MINIO;
|
|
}
|
|
throw new RuntimeException('Unsupported replication kind.');
|
|
}
|
|
|
|
private static function databaseComposeTemplate(array $input, string $role): array
|
|
{
|
|
$serviceName = self::composeIdentifier($input['service_name'] ?? null, 'mariadb-' . $role);
|
|
$volumeName = self::composeIdentifier($input['volume_name'] ?? null, $serviceName . '-data');
|
|
$database = self::composeScalar($input['database'] ?? null, 'nnks_db');
|
|
$username = self::composeScalar($input['username'] ?? null, 'nnks_db_user');
|
|
$image = self::composeImage($input['image'] ?? null, 'mariadb:11');
|
|
$hostPort = self::boundedInt($input['host_port'] ?? null, $role === 'primary' ? 3306 : 3307, 1, 65535);
|
|
$serverId = self::boundedInt($input['server_id'] ?? null, $role === 'primary' ? 1 : 2, 1, 4294967295);
|
|
$rootPassword = self::composePassword($input['admin_password'] ?? null);
|
|
$applicationPassword = self::composePassword($input['password'] ?? null);
|
|
$replicationUsername = self::composeScalar($input['replication_username'] ?? null, 'replication');
|
|
$replicationPassword = self::composePassword($input['replication_password'] ?? null);
|
|
$primaryHost = self::composeScalar($input['primary_host'] ?? null, '<primary-host>');
|
|
$primaryPort = self::boundedInt($input['primary_port'] ?? null, 3306, 1, 65535);
|
|
$primaryAdminUsername = self::composeScalar($input['primary_admin_username'] ?? null, 'root');
|
|
$primaryAdminPassword = trim((string)($input['primary_admin_password'] ?? ''));
|
|
|
|
$command = [
|
|
'mariadbd',
|
|
'--server-id=' . $serverId,
|
|
'--log-bin=/var/lib/mysql/mariadb-bin',
|
|
'--binlog-format=ROW',
|
|
'--gtid-strict-mode=ON',
|
|
'--expire-logs-days=7',
|
|
];
|
|
if ($role === 'replica') {
|
|
$command[] = '--read-only=ON';
|
|
foreach (self::MARIADB_SCHEMA_ONLY_TABLES as $tableName) {
|
|
$command[] = '--replicate-ignore-table=' . $database . '.' . $tableName;
|
|
}
|
|
}
|
|
|
|
$lines = [
|
|
'services:',
|
|
' ' . $serviceName . ':',
|
|
' image: ' . self::yamlQuote($image),
|
|
' restart: unless-stopped',
|
|
' environment:',
|
|
' MARIADB_ROOT_PASSWORD: "${MARIADB_ROOT_PASSWORD:?set MARIADB_ROOT_PASSWORD}"',
|
|
' MARIADB_DATABASE: ' . self::yamlQuote($database),
|
|
' MARIADB_USER: ' . self::yamlQuote($username),
|
|
' MARIADB_PASSWORD: "${MARIADB_PASSWORD:?set MARIADB_PASSWORD}"',
|
|
' command:',
|
|
];
|
|
|
|
foreach ($command as $argument) {
|
|
$lines[] = ' - ' . self::yamlQuote($argument);
|
|
}
|
|
|
|
$lines = array_merge($lines, [
|
|
' volumes:',
|
|
' - ' . $volumeName . ':/var/lib/mysql',
|
|
' ports:',
|
|
' - ' . self::yamlQuote($hostPort . ':3306'),
|
|
' healthcheck:',
|
|
' test:',
|
|
' - "CMD-SHELL"',
|
|
' - "mariadb-admin ping -h 127.0.0.1 -uroot -p$${MARIADB_ROOT_PASSWORD} --silent"',
|
|
' interval: 10s',
|
|
' timeout: 5s',
|
|
' retries: 12',
|
|
]);
|
|
|
|
if ($role === 'replica') {
|
|
$seedServiceName = self::composeIdentifier($serviceName . '-seed', 'mariadb-replica-seed');
|
|
$seedScript = [
|
|
'marker="/var/lib/mysql/.truckwash-replica-seeded"',
|
|
'if [ -f "$${marker}" ]; then',
|
|
' echo "Replica already seeded."',
|
|
' exit 0',
|
|
'fi',
|
|
'echo "Waiting for local replica..."',
|
|
'until mariadb-admin ping -h ' . self::shellArg($serviceName) . ' -uroot -p"$${MARIADB_ROOT_PASSWORD}" --silent; do sleep 2; done',
|
|
'echo "Importing seed from primary..."',
|
|
'mariadb-dump --host="$${MARIADB_PRIMARY_HOST}" --port="$${MARIADB_PRIMARY_PORT}" --user="$${MARIADB_PRIMARY_ADMIN_USER}" --password="$${MARIADB_PRIMARY_ADMIN_PASSWORD}" --single-transaction --quick --routines --triggers --events --gtid --master-data=2 ' . self::mariaDbSchemaOnlyDumpIgnoreArgs('$${MARIADB_SEED_DATABASE}') . ' --databases "$${MARIADB_SEED_DATABASE}" | mariadb --host=' . self::shellArg($serviceName) . ' --user=root --password="$${MARIADB_ROOT_PASSWORD}"',
|
|
'for table in ' . implode(' ', self::MARIADB_SCHEMA_ONLY_TABLES) . '; do',
|
|
' mariadb-dump --host="$${MARIADB_PRIMARY_HOST}" --port="$${MARIADB_PRIMARY_PORT}" --user="$${MARIADB_PRIMARY_ADMIN_USER}" --password="$${MARIADB_PRIMARY_ADMIN_PASSWORD}" --single-transaction --quick --no-data "$${MARIADB_SEED_DATABASE}" "$${table}" | mariadb --host=' . self::shellArg($serviceName) . ' --user=root --password="$${MARIADB_ROOT_PASSWORD}" "$${MARIADB_SEED_DATABASE}" || true',
|
|
'done',
|
|
'touch "$${marker}"',
|
|
'echo "Replica seed completed."',
|
|
];
|
|
|
|
$lines = array_merge($lines, [
|
|
' ' . $seedServiceName . ':',
|
|
' image: ' . self::yamlQuote($image),
|
|
' restart: "no"',
|
|
' depends_on:',
|
|
' ' . $serviceName . ':',
|
|
' condition: service_healthy',
|
|
' environment:',
|
|
' MARIADB_ROOT_PASSWORD: "${MARIADB_ROOT_PASSWORD:?set MARIADB_ROOT_PASSWORD}"',
|
|
' MARIADB_PRIMARY_HOST: "${MARIADB_PRIMARY_HOST:?set MARIADB_PRIMARY_HOST}"',
|
|
' MARIADB_PRIMARY_PORT: "${MARIADB_PRIMARY_PORT:-3306}"',
|
|
' MARIADB_PRIMARY_ADMIN_USER: "${MARIADB_PRIMARY_ADMIN_USER:-root}"',
|
|
' MARIADB_PRIMARY_ADMIN_PASSWORD: "${MARIADB_PRIMARY_ADMIN_PASSWORD:?set MARIADB_PRIMARY_ADMIN_PASSWORD}"',
|
|
' MARIADB_SEED_DATABASE: ' . self::yamlQuote($database),
|
|
' volumes:',
|
|
' - ' . $volumeName . ':/var/lib/mysql',
|
|
' entrypoint:',
|
|
' - /bin/sh',
|
|
' - -ec',
|
|
' - |',
|
|
]);
|
|
foreach ($seedScript as $scriptLine) {
|
|
$lines[] = ' ' . $scriptLine;
|
|
}
|
|
}
|
|
|
|
$lines = array_merge($lines, [
|
|
'volumes:',
|
|
' ' . $volumeName . ':',
|
|
]);
|
|
|
|
$steps = [
|
|
'Deploy this compose file as a normal Docker Compose or Coolify compose service.',
|
|
'Keep server-id unique across the MariaDB primary and every replica.',
|
|
'Create or store a replication user on the primary with REPLICATION SLAVE privileges.',
|
|
];
|
|
if ($role === 'replica') {
|
|
$steps[] = 'Fill MARIADB_PRIMARY_ADMIN_PASSWORD in the generated .env file.';
|
|
$steps[] = 'Deploy the compose file and wait for the seed service to complete successfully.';
|
|
$steps[] = 'Test the connection, then save and provision the replica.';
|
|
} else {
|
|
$steps[] = 'Add the primary credentials in the superuser UI after the service is reachable.';
|
|
}
|
|
|
|
$seedCommand = implode(' ', [
|
|
'mariadb-dump',
|
|
'--host=' . self::shellArg($primaryHost),
|
|
'--port=' . $primaryPort,
|
|
'--user=<primary-admin-user>',
|
|
'--password',
|
|
'--single-transaction',
|
|
'--quick',
|
|
'--routines',
|
|
'--triggers',
|
|
'--events',
|
|
'--gtid',
|
|
'--master-data=2',
|
|
...self::mariaDbSchemaOnlySeedCommandIgnoreArgs($database),
|
|
'--databases',
|
|
self::shellArg($database),
|
|
'|',
|
|
'mariadb',
|
|
'--host=<replica-host>',
|
|
'--port=' . $hostPort,
|
|
'--user=root',
|
|
'--password',
|
|
]);
|
|
$envLines = [
|
|
'MARIADB_ROOT_PASSWORD=' . $rootPassword,
|
|
'MARIADB_PASSWORD=' . $applicationPassword,
|
|
];
|
|
if ($role === 'replica') {
|
|
$envLines[] = 'MARIADB_PRIMARY_HOST=' . $primaryHost;
|
|
$envLines[] = 'MARIADB_PRIMARY_PORT=' . $primaryPort;
|
|
$envLines[] = 'MARIADB_PRIMARY_ADMIN_USER=' . $primaryAdminUsername;
|
|
$envLines[] = 'MARIADB_PRIMARY_ADMIN_PASSWORD=' . $primaryAdminPassword;
|
|
}
|
|
|
|
return [
|
|
'kind' => self::KIND_DATABASE,
|
|
'engine' => 'mariadb',
|
|
'role' => $role,
|
|
'service_name' => $serviceName,
|
|
'host_port' => $hostPort,
|
|
'server_id' => $serverId,
|
|
'compose' => implode("\n", $lines) . "\n",
|
|
'env' => implode("\n", $envLines) . "\n",
|
|
'seed_command' => $role === 'replica' ? $seedCommand : '',
|
|
'credentials' => [
|
|
'label' => $serviceName,
|
|
'host' => '',
|
|
'port' => $hostPort,
|
|
'database' => $database,
|
|
'username' => $username,
|
|
'password' => $applicationPassword,
|
|
'admin_username' => 'root',
|
|
'admin_password' => $rootPassword,
|
|
'replication_username' => $replicationUsername,
|
|
'replication_password' => $replicationPassword,
|
|
'ssl_mode' => 'DISABLED',
|
|
'allow_preseeded_replica' => $role === 'replica',
|
|
],
|
|
'steps' => $steps,
|
|
];
|
|
}
|
|
|
|
private static function redisComposeTemplate(array $input, string $role): array
|
|
{
|
|
$serviceName = self::composeIdentifier($input['service_name'] ?? null, 'redis-' . $role);
|
|
$volumeName = self::composeIdentifier($input['volume_name'] ?? null, $serviceName . '-data');
|
|
$image = self::composeImage($input['image'] ?? null, 'redis:7');
|
|
$hostPort = self::boundedInt($input['host_port'] ?? null, $role === 'primary' ? 6379 : 6380, 1, 65535);
|
|
$primaryHost = self::composeScalar($input['primary_host'] ?? null, 'redis-primary');
|
|
$primaryPort = self::boundedInt($input['primary_port'] ?? null, 6379, 1, 65535);
|
|
$redisPassword = self::composePassword($input['password'] ?? null);
|
|
$primaryPassword = self::composeScalar($input['primary_password'] ?? null, '');
|
|
$primaryUsername = self::composeScalar($input['primary_username'] ?? null, '');
|
|
|
|
$script = [
|
|
'if [ ! -f /data/redis.conf ]; then',
|
|
' {',
|
|
' echo "appendonly yes"',
|
|
' echo "requirepass $$REDIS_PASSWORD"',
|
|
];
|
|
if ($role === 'replica') {
|
|
$script[] = ' echo "replicaof $$REDIS_PRIMARY_HOST $${REDIS_PRIMARY_PORT:-6379}"';
|
|
$script[] = ' echo "masterauth $$REDIS_PRIMARY_PASSWORD"';
|
|
$script[] = ' if [ -n "$${REDIS_PRIMARY_USERNAME:-}" ] && [ "$${REDIS_PRIMARY_USERNAME}" != "default" ]; then';
|
|
$script[] = ' echo "masteruser $$REDIS_PRIMARY_USERNAME"';
|
|
$script[] = ' fi';
|
|
}
|
|
$script = array_merge($script, [
|
|
' } > /data/redis.conf',
|
|
'fi',
|
|
'exec redis-server /data/redis.conf',
|
|
]);
|
|
|
|
$lines = [
|
|
'services:',
|
|
' ' . $serviceName . ':',
|
|
' image: ' . self::yamlQuote($image),
|
|
' restart: unless-stopped',
|
|
' environment:',
|
|
' REDIS_PASSWORD: "${REDIS_PASSWORD:?set REDIS_PASSWORD}"',
|
|
];
|
|
if ($role === 'replica') {
|
|
$lines[] = ' REDIS_PRIMARY_HOST: "${REDIS_PRIMARY_HOST:?set REDIS_PRIMARY_HOST}"';
|
|
$lines[] = ' REDIS_PRIMARY_PORT: "${REDIS_PRIMARY_PORT:-6379}"';
|
|
$lines[] = ' REDIS_PRIMARY_PASSWORD: "${REDIS_PRIMARY_PASSWORD:?set REDIS_PRIMARY_PASSWORD}"';
|
|
$lines[] = ' REDIS_PRIMARY_USERNAME: "${REDIS_PRIMARY_USERNAME:-}"';
|
|
}
|
|
$lines[] = ' command:';
|
|
$lines[] = ' - /bin/sh';
|
|
$lines[] = ' - -ec';
|
|
$lines[] = ' - |';
|
|
foreach ($script as $scriptLine) {
|
|
$lines[] = ' ' . $scriptLine;
|
|
}
|
|
|
|
$lines = array_merge($lines, [
|
|
' volumes:',
|
|
' - ' . $volumeName . ':/data',
|
|
' ports:',
|
|
' - ' . self::yamlQuote($hostPort . ':6379'),
|
|
' healthcheck:',
|
|
' test:',
|
|
' - "CMD-SHELL"',
|
|
' - "redis-cli --no-auth-warning -a \"$${REDIS_PASSWORD}\" ping | grep PONG"',
|
|
' interval: 10s',
|
|
' timeout: 5s',
|
|
' retries: 12',
|
|
'volumes:',
|
|
' ' . $volumeName . ':',
|
|
]);
|
|
|
|
$steps = [
|
|
'Deploy this compose file as a normal Docker Compose or Coolify compose service.',
|
|
'Add the Redis credentials in the superuser UI after the service is reachable.',
|
|
];
|
|
if ($role === 'replica') {
|
|
$steps[] = 'Use the current Redis primary host and password for REDIS_PRIMARY_HOST and REDIS_PRIMARY_PASSWORD, then run Test in the superuser UI.';
|
|
}
|
|
|
|
$envLines = [
|
|
'REDIS_PASSWORD=' . $redisPassword,
|
|
];
|
|
if ($role === 'replica') {
|
|
$envLines[] = 'REDIS_PRIMARY_HOST=' . $primaryHost;
|
|
$envLines[] = 'REDIS_PRIMARY_PORT=' . $primaryPort;
|
|
$envLines[] = 'REDIS_PRIMARY_PASSWORD=' . $primaryPassword;
|
|
$envLines[] = 'REDIS_PRIMARY_USERNAME=' . $primaryUsername;
|
|
}
|
|
|
|
return [
|
|
'kind' => self::KIND_REDIS,
|
|
'engine' => 'redis',
|
|
'role' => $role,
|
|
'service_name' => $serviceName,
|
|
'host_port' => $hostPort,
|
|
'compose' => implode("\n", $lines) . "\n",
|
|
'env' => implode("\n", $envLines) . "\n",
|
|
'credentials' => [
|
|
'label' => $serviceName,
|
|
'host' => '',
|
|
'port' => $hostPort,
|
|
'database' => 0,
|
|
'username' => '',
|
|
'password' => $redisPassword,
|
|
],
|
|
'steps' => $steps,
|
|
];
|
|
}
|
|
|
|
private static function minioComposeTemplate(array $input, string $role): array
|
|
{
|
|
$serviceName = self::composeIdentifier($input['service_name'] ?? null, 'minio-' . $role);
|
|
$volumeName = self::composeIdentifier($input['volume_name'] ?? null, $serviceName . '-data');
|
|
$image = self::composeImage($input['image'] ?? null, 'minio/minio:latest');
|
|
$mcImage = self::composeImage($input['mc_image'] ?? null, 'minio/mc:latest');
|
|
$hostPort = self::boundedInt($input['host_port'] ?? null, $role === 'primary' ? 9000 : 9010, 1, 65535);
|
|
$consolePort = self::boundedInt($input['console_port'] ?? null, $role === 'primary' ? 9001 : 9011, 1, 65535);
|
|
$rootUser = self::composeAccessKey($input['username'] ?? $input['access_key'] ?? null);
|
|
$rootPassword = self::composePassword($input['password'] ?? $input['secret_key'] ?? null);
|
|
$buckets = self::normalizeMinioBuckets($input['buckets'] ?? self::MINIO_DEFAULT_BUCKETS);
|
|
$transferLimit = self::normalizeMinioTransferLimit(
|
|
$input['replication_transfer_limit'] ?? self::MINIO_DEFAULT_REPLICATION_TRANSFER_LIMIT,
|
|
true
|
|
);
|
|
$primaryEndpoint = self::minioPrimaryComposeValue($input, 'endpoint');
|
|
$primaryAccessKey = self::minioPrimaryComposeValue($input, 'access_key');
|
|
$primarySecretKey = self::minioPrimaryComposeValue($input, 'secret_key');
|
|
[$serverUrl, $browserRedirectUrl] = self::minioComposePublicUrls($input, $hostPort, $consolePort);
|
|
|
|
$setupScript = [
|
|
'until mc alias set local http://' . self::shellArg($serviceName) . ':9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"; do sleep 2; done',
|
|
];
|
|
foreach ($buckets as $bucket) {
|
|
$bucketArg = self::shellArg('local/' . $bucket);
|
|
$setupScript[] = 'mc mb --with-lock --ignore-existing ' . $bucketArg;
|
|
$setupScript[] = 'mc version enable ' . $bucketArg . ' || true';
|
|
if ($role === 'replica' && self::minioBucketUsesBoundedReplicaRetention($bucket)) {
|
|
$setupScript[] = 'mc ilm rule add --expire-days "' . self::MINIO_BACKUP_REPLICA_RETENTION_DAYS . '" --noncurrent-expire-days "' . self::MINIO_BACKUP_REPLICA_RETENTION_DAYS . '" ' . $bucketArg . ' || true';
|
|
}
|
|
}
|
|
|
|
$lines = [
|
|
'services:',
|
|
' ' . $serviceName . ':',
|
|
' image: ' . self::yamlQuote($image),
|
|
' restart: unless-stopped',
|
|
' command:',
|
|
' - server',
|
|
' - /data',
|
|
' - --console-address',
|
|
' - ":9001"',
|
|
' environment:',
|
|
' MINIO_ROOT_USER: "${MINIO_ROOT_USER:?set MINIO_ROOT_USER}"',
|
|
' MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:?set MINIO_ROOT_PASSWORD}"',
|
|
' MINIO_SERVER_URL: "${MINIO_SERVER_URL:-}"',
|
|
' MINIO_BROWSER_REDIRECT_URL: "${MINIO_BROWSER_REDIRECT_URL:-}"',
|
|
' volumes:',
|
|
' - ' . $volumeName . ':/data',
|
|
' ports:',
|
|
' - ' . self::yamlQuote($hostPort . ':9000'),
|
|
' - ' . self::yamlQuote($consolePort . ':9001'),
|
|
' healthcheck:',
|
|
' test:',
|
|
' - "CMD"',
|
|
' - "curl"',
|
|
' - "-f"',
|
|
' - "http://127.0.0.1:9000/minio/health/live"',
|
|
' interval: 10s',
|
|
' timeout: 5s',
|
|
' retries: 12',
|
|
' ' . $serviceName . '-setup:',
|
|
' image: ' . self::yamlQuote($mcImage),
|
|
' restart: "no"',
|
|
' depends_on:',
|
|
' ' . $serviceName . ':',
|
|
' condition: service_healthy',
|
|
' environment:',
|
|
' MINIO_ROOT_USER: "${MINIO_ROOT_USER:?set MINIO_ROOT_USER}"',
|
|
' MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:?set MINIO_ROOT_PASSWORD}"',
|
|
];
|
|
if ($role === 'replica') {
|
|
$lines[] = ' MINIO_PRIMARY_ENDPOINT: "${MINIO_PRIMARY_ENDPOINT:-}"';
|
|
$lines[] = ' MINIO_PRIMARY_ACCESS_KEY: "${MINIO_PRIMARY_ACCESS_KEY:-}"';
|
|
$lines[] = ' MINIO_PRIMARY_SECRET_KEY: "${MINIO_PRIMARY_SECRET_KEY:-}"';
|
|
}
|
|
$lines = array_merge($lines, [
|
|
' entrypoint:',
|
|
' - /bin/sh',
|
|
' - -ec',
|
|
' - |',
|
|
]);
|
|
foreach ($setupScript as $scriptLine) {
|
|
$lines[] = ' ' . $scriptLine;
|
|
}
|
|
$lines = array_merge($lines, [
|
|
'volumes:',
|
|
' ' . $volumeName . ':',
|
|
]);
|
|
|
|
$envLines = [
|
|
'MINIO_ROOT_USER=' . $rootUser,
|
|
'MINIO_ROOT_PASSWORD=' . $rootPassword,
|
|
'MINIO_SERVER_URL=' . $serverUrl,
|
|
'MINIO_BROWSER_REDIRECT_URL=' . $browserRedirectUrl,
|
|
'MINIO_BUCKETS=' . implode(',', $buckets),
|
|
];
|
|
if ($role === 'replica') {
|
|
$envLines[] = 'MINIO_BACKUP_REPLICA_RETENTION_DAYS=' . self::MINIO_BACKUP_REPLICA_RETENTION_DAYS;
|
|
$envLines[] = 'MINIO_REPLICATION_TRANSFER_LIMIT=' . $transferLimit;
|
|
$envLines[] = 'MINIO_PRIMARY_ENDPOINT=' . $primaryEndpoint;
|
|
$envLines[] = 'MINIO_PRIMARY_ACCESS_KEY=' . $primaryAccessKey;
|
|
$envLines[] = 'MINIO_PRIMARY_SECRET_KEY=' . $primarySecretKey;
|
|
}
|
|
|
|
$steps = [
|
|
'Deploy this compose file as a normal Docker Compose or Coolify compose service.',
|
|
'The setup service creates required buckets and enables bucket versioning.',
|
|
'Add the MinIO credentials in the superuser UI after the API endpoint is reachable.',
|
|
];
|
|
if ($role === 'replica') {
|
|
$steps[] = 'Fill the MINIO_PRIMARY_* .env values for reference; managed bucket replication is configured from the superuser UI.';
|
|
$steps[] = 'The backups bucket is retained on replicas for ' . self::MINIO_BACKUP_REPLICA_RETENTION_DAYS . ' days; other buckets are fully replicated.';
|
|
$steps[] = 'Replica seeding and bucket replication are bandwidth-limited to ' . ($transferLimit !== '' ? $transferLimit : 'unlimited') . '.';
|
|
$steps[] = 'Test the connection, then save and provision the replica.';
|
|
}
|
|
|
|
return [
|
|
'kind' => self::KIND_MINIO,
|
|
'engine' => 'minio',
|
|
'role' => $role,
|
|
'service_name' => $serviceName,
|
|
'host_port' => $hostPort,
|
|
'console_port' => $consolePort,
|
|
'compose' => implode("\n", $lines) . "\n",
|
|
'env' => implode("\n", $envLines) . "\n",
|
|
'credentials' => [
|
|
'label' => $serviceName,
|
|
'host' => '',
|
|
'port' => $hostPort,
|
|
'scheme' => 'http',
|
|
'endpoint' => '',
|
|
'buckets' => $buckets,
|
|
'console_port' => $consolePort,
|
|
'username' => $rootUser,
|
|
'password' => $rootPassword,
|
|
'replication_transfer_limit' => $transferLimit,
|
|
'space_headroom_percent' => self::MINIO_SPACE_HEADROOM_PERCENT,
|
|
],
|
|
'steps' => $steps,
|
|
];
|
|
}
|
|
|
|
private static function normalizeComposeRole(string $role): string
|
|
{
|
|
$role = strtolower(trim($role));
|
|
if (in_array($role, ['primary', 'replica'], true)) {
|
|
return $role;
|
|
}
|
|
throw new RuntimeException('Unsupported compose role.');
|
|
}
|
|
|
|
private static function composeIdentifier(mixed $value, string $fallback): string
|
|
{
|
|
$identifier = strtolower(trim((string)$value));
|
|
$identifier = (string)preg_replace('/[^a-z0-9_.-]+/', '-', $identifier);
|
|
$identifier = trim($identifier, '-_.');
|
|
return $identifier !== '' ? $identifier : $fallback;
|
|
}
|
|
|
|
private static function composeScalar(mixed $value, string $fallback): string
|
|
{
|
|
$scalar = trim((string)$value);
|
|
return $scalar !== '' ? $scalar : $fallback;
|
|
}
|
|
|
|
private static function composePassword(mixed $value): string
|
|
{
|
|
$password = trim((string)$value);
|
|
if ($password !== '') {
|
|
return $password;
|
|
}
|
|
|
|
return self::generateSecret(24);
|
|
}
|
|
|
|
private static function composeAccessKey(mixed $value): string
|
|
{
|
|
$accessKey = trim((string)$value);
|
|
if ($accessKey !== '') {
|
|
return $accessKey;
|
|
}
|
|
|
|
return 'twminio' . bin2hex(random_bytes(12));
|
|
}
|
|
|
|
private static function minioPrimaryComposeValue(array $input, string $field): string
|
|
{
|
|
if ($field === 'endpoint') {
|
|
foreach (['primary_endpoint', 'minio_primary_endpoint'] as $key) {
|
|
$value = trim((string)($input[$key] ?? ''));
|
|
if ($value !== '') {
|
|
return $value;
|
|
}
|
|
}
|
|
|
|
$primaryHost = trim((string)($input['primary_host'] ?? ''));
|
|
if ($primaryHost !== '') {
|
|
if (preg_match('/^https?:\/\//i', $primaryHost) === 1) {
|
|
return $primaryHost;
|
|
}
|
|
|
|
$primaryScheme = trim((string)($input['primary_scheme'] ?? 'http')) ?: 'http';
|
|
$primaryPort = self::boundedInt($input['primary_port'] ?? null, 9000, 1, 65535);
|
|
return self::minioEndpointFromParts($primaryScheme, $primaryHost, $primaryPort);
|
|
}
|
|
}
|
|
|
|
$inputKeys = match ($field) {
|
|
'endpoint' => [],
|
|
'access_key' => ['primary_access_key', 'minio_primary_access_key', 'primary_username'],
|
|
'secret_key' => ['primary_secret_key', 'minio_primary_secret_key', 'primary_password'],
|
|
default => [],
|
|
};
|
|
|
|
foreach ($inputKeys as $key) {
|
|
$value = trim((string)($input[$key] ?? ''));
|
|
if ($value !== '') {
|
|
return $value;
|
|
}
|
|
}
|
|
|
|
$minioConfig = $GLOBALS['MINIO'] ?? null;
|
|
if (!is_array($minioConfig)) {
|
|
return '';
|
|
}
|
|
|
|
return trim((string)($minioConfig[$field] ?? ''));
|
|
}
|
|
|
|
/**
|
|
* Public MinIO URLs keep browser redirects on the externally mapped ports.
|
|
*/
|
|
private static function minioComposePublicUrls(array $input, int $hostPort, int $consolePort): array
|
|
{
|
|
$rawHost = trim((string)($input['public_host'] ?? $input['host'] ?? $input['endpoint'] ?? ''));
|
|
if ($rawHost === '') {
|
|
return ['', ''];
|
|
}
|
|
|
|
try {
|
|
[$host, , $scheme] = self::normalizeMinioAddress($rawHost, null, $input['scheme'] ?? null);
|
|
} catch (Throwable) {
|
|
return ['', ''];
|
|
}
|
|
|
|
return [
|
|
self::minioEndpointFromParts($scheme, $host, $hostPort),
|
|
self::minioEndpointFromParts($scheme, $host, $consolePort),
|
|
];
|
|
}
|
|
|
|
private static function generateSecret(int $bytes): string
|
|
{
|
|
return rtrim(strtr(base64_encode(random_bytes($bytes)), '+/', '-_'), '=');
|
|
}
|
|
|
|
private static function composeImage(mixed $value, string $fallback): string
|
|
{
|
|
$image = trim((string)$value);
|
|
if ($image === '' || preg_match('/^[a-zA-Z0-9._:\/-]+$/', $image) !== 1) {
|
|
return $fallback;
|
|
}
|
|
return $image;
|
|
}
|
|
|
|
private static function boundedInt(mixed $value, int $fallback, int $min, int $max): int
|
|
{
|
|
if (filter_var($value, FILTER_VALIDATE_INT) === false) {
|
|
return $fallback;
|
|
}
|
|
return max($min, min($max, (int)$value));
|
|
}
|
|
|
|
private static function yamlQuote(mixed $value): string
|
|
{
|
|
$encoded = json_encode((string)$value, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
|
|
return is_string($encoded) ? $encoded : '""';
|
|
}
|
|
|
|
private static function shellArg(string $value): string
|
|
{
|
|
return "'" . str_replace("'", "'\"'\"'", $value) . "'";
|
|
}
|
|
|
|
private static function mariaDbSchemaOnlyDumpIgnoreArgs(string $databaseExpression): string
|
|
{
|
|
return implode(' ', array_map(
|
|
static fn(string $tableName): string => '--ignore-table="' . $databaseExpression . '.' . $tableName . '"',
|
|
self::MARIADB_SCHEMA_ONLY_TABLES
|
|
));
|
|
}
|
|
|
|
private static function mariaDbSchemaOnlySeedCommandIgnoreArgs(string $database): array
|
|
{
|
|
return array_map(
|
|
static fn(string $tableName): string => '--ignore-table=' . self::shellArg($database . '.' . $tableName),
|
|
self::MARIADB_SCHEMA_ONLY_TABLES
|
|
);
|
|
}
|
|
|
|
private static function quoteIdentifier(string $identifier): string
|
|
{
|
|
return '`' . str_replace('`', '``', $identifier) . '`';
|
|
}
|
|
|
|
private static function sqlString(mysqli $connection, string $value): string
|
|
{
|
|
return "'" . $connection->real_escape_string($value) . "'";
|
|
}
|
|
|
|
public static function mysqlGtidIntervalCount(string $gtidSet): int
|
|
{
|
|
$count = 0;
|
|
foreach (self::parseMysqlGtidSet($gtidSet) as $intervals) {
|
|
foreach ($intervals as [$start, $end]) {
|
|
$count += max(0, $end - $start + 1);
|
|
}
|
|
}
|
|
return $count;
|
|
}
|
|
|
|
public static function mysqlGtidCoveragePercent(string $sourceSet, string $executedSet): float
|
|
{
|
|
$source = self::parseMysqlGtidSet($sourceSet);
|
|
$executed = self::parseMysqlGtidSet($executedSet);
|
|
$total = 0;
|
|
$covered = 0;
|
|
|
|
foreach ($source as $uuid => $sourceIntervals) {
|
|
foreach ($sourceIntervals as [$sourceStart, $sourceEnd]) {
|
|
$total += max(0, $sourceEnd - $sourceStart + 1);
|
|
foreach ($executed[$uuid] ?? [] as [$executedStart, $executedEnd]) {
|
|
$start = max($sourceStart, $executedStart);
|
|
$end = min($sourceEnd, $executedEnd);
|
|
if ($end >= $start) {
|
|
$covered += $end - $start + 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($total === 0) {
|
|
return 100.0;
|
|
}
|
|
|
|
return round(min(100, max(0, ($covered / $total) * 100)), 2);
|
|
}
|
|
|
|
public static function redisOffsetPercent(int $primaryOffset, int $replicaOffset): float
|
|
{
|
|
if ($primaryOffset <= 0) {
|
|
return 100.0;
|
|
}
|
|
|
|
return round(min(100, max(0, ($replicaOffset / $primaryOffset) * 100)), 2);
|
|
}
|
|
|
|
public static function redisReplicationPercentFromInfo(array $primaryInfo, array $replicaInfo): float
|
|
{
|
|
$syncInProgress = (string)($replicaInfo['master_sync_in_progress'] ?? '0') === '1';
|
|
if ($syncInProgress) {
|
|
$totalBytes = (int)($replicaInfo['master_sync_total_bytes'] ?? 0);
|
|
$leftBytes = (int)($replicaInfo['master_sync_left_bytes'] ?? 0);
|
|
if ($totalBytes <= 0) {
|
|
return 5.0;
|
|
}
|
|
|
|
$copiedBytes = max(0, $totalBytes - max(0, $leftBytes));
|
|
return round(min(99.99, max(5.0, ($copiedBytes / $totalBytes) * 100)), 2);
|
|
}
|
|
|
|
$primaryOffset = (int)($primaryInfo['master_repl_offset'] ?? 0);
|
|
$replicaOffset = (int)($replicaInfo['slave_repl_offset'] ?? $replicaInfo['master_repl_offset'] ?? 0);
|
|
return self::redisOffsetPercent($primaryOffset, $replicaOffset);
|
|
}
|
|
|
|
public static function redisProvisionProgress(float $replicationPercent, array $syncBlockers = []): float
|
|
{
|
|
if ($replicationPercent >= 100.0 && $syncBlockers === []) {
|
|
return 100.0;
|
|
}
|
|
|
|
return round(min(99.99, max(5.0, $replicationPercent)), 2);
|
|
}
|
|
|
|
public static function replicationHealthStatus(
|
|
bool $reachable,
|
|
string $role,
|
|
float $replicationPercent,
|
|
array $blockers,
|
|
bool $replicationChecked = true
|
|
): string {
|
|
if (!$reachable) {
|
|
return 'down';
|
|
}
|
|
|
|
if ($blockers !== []) {
|
|
return 'degraded';
|
|
}
|
|
|
|
if ($replicationChecked && $role !== 'primary' && $replicationPercent < 100.0) {
|
|
return 'degraded';
|
|
}
|
|
|
|
return 'ok';
|
|
}
|
|
|
|
public static function minioRequiredFreeBytes(int $sourceBytes, float $headroomPercent = self::MINIO_SPACE_HEADROOM_PERCENT): int
|
|
{
|
|
return (int)ceil(max(0, $sourceBytes) * (1 + max(0.0, $headroomPercent) / 100));
|
|
}
|
|
|
|
public static function minioByteReplicationPercent(int $sourceBytes, int $replicaBytes): float
|
|
{
|
|
if ($sourceBytes <= 0) {
|
|
return 100.0;
|
|
}
|
|
|
|
return round(min(100, max(0, ($replicaBytes / $sourceBytes) * 100)), 2);
|
|
}
|
|
|
|
public static function minioProvisionProgress(array $status): float
|
|
{
|
|
$percent = round((float)($status['replication_percent'] ?? 0), 2);
|
|
$blockers = array_values(array_filter($status['blockers'] ?? []));
|
|
if ($percent >= 100.0 && $blockers === []) {
|
|
return 100.0;
|
|
}
|
|
|
|
$measured = !empty($status['raw']['storage']['measured'])
|
|
|| (string)($status['raw']['progress_source'] ?? '') === 'minio_replicate_status';
|
|
if ($measured) {
|
|
return min(self::MINIO_INCOMPLETE_PROGRESS_MAX_PERCENT, max(0.0, $percent));
|
|
}
|
|
|
|
return self::minioIncompleteProgress($percent);
|
|
}
|
|
|
|
private static function minioIncompleteProgress(float $percent, float $minimum = 5.0): float
|
|
{
|
|
return round(min(self::MINIO_INCOMPLETE_PROGRESS_MAX_PERCENT, max($minimum, $percent)), 2);
|
|
}
|
|
|
|
public static function minioReplicationProgressFromStatusOutput(mixed $value): ?array
|
|
{
|
|
if (is_string($value)) {
|
|
$textProgress = self::minioReplicationProgressFromText($value);
|
|
if ($textProgress !== null) {
|
|
return $textProgress;
|
|
}
|
|
|
|
$decoded = self::decodeMinioJsonOutput($value);
|
|
if ($decoded !== null && $decoded !== $value) {
|
|
return self::minioReplicationProgressFromStatusOutput($decoded);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
$stats = [
|
|
'completed_bytes' => 0.0,
|
|
'pending_bytes' => 0.0,
|
|
'failed_bytes' => 0.0,
|
|
'total_bytes' => 0.0,
|
|
'completed_count' => 0.0,
|
|
'pending_count' => 0.0,
|
|
'failed_count' => 0.0,
|
|
'total_count' => 0.0,
|
|
'complete_signals' => 0,
|
|
'incomplete_signals' => 0,
|
|
];
|
|
self::collectMinioReplicationProgress($value, $stats);
|
|
|
|
$completedBytes = (float)$stats['completed_bytes'];
|
|
$remainingBytes = (float)$stats['pending_bytes'] + (float)$stats['failed_bytes'];
|
|
$totalBytes = (float)$stats['total_bytes'];
|
|
$completedCount = (float)$stats['completed_count'];
|
|
$remainingCount = (float)$stats['pending_count'] + (float)$stats['failed_count'];
|
|
$totalCount = (float)$stats['total_count'];
|
|
$basis = null;
|
|
$percent = null;
|
|
|
|
if ($totalBytes > 0.0) {
|
|
$percent = ($completedBytes / $totalBytes) * 100;
|
|
$basis = 'total_bytes';
|
|
} elseif (($completedBytes + $remainingBytes) > 0.0) {
|
|
$percent = ($completedBytes / ($completedBytes + $remainingBytes)) * 100;
|
|
$basis = 'byte_balance';
|
|
} elseif ($totalCount > 0.0) {
|
|
$percent = ($completedCount / $totalCount) * 100;
|
|
$basis = 'total_count';
|
|
} elseif (($completedCount + $remainingCount) > 0.0) {
|
|
$percent = ($completedCount / ($completedCount + $remainingCount)) * 100;
|
|
$basis = 'count_balance';
|
|
} elseif ((int)$stats['complete_signals'] > 0 && (int)$stats['incomplete_signals'] === 0) {
|
|
$percent = 100.0;
|
|
$basis = 'status_signal';
|
|
} elseif ((int)$stats['incomplete_signals'] > 0) {
|
|
$percent = 5.0;
|
|
$basis = 'status_signal';
|
|
}
|
|
|
|
if ($percent === null) {
|
|
return null;
|
|
}
|
|
|
|
$percent = round(min(100.0, max(0.0, $percent)), 2);
|
|
|
|
return [
|
|
'replication_percent' => $percent,
|
|
'blockers' => $percent < 100.0 ? [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER] : [],
|
|
'basis' => $basis,
|
|
'stats' => $stats,
|
|
];
|
|
}
|
|
|
|
public static function minioBackupReplicaRetentionDays(): int
|
|
{
|
|
return self::MINIO_BACKUP_REPLICA_RETENTION_DAYS;
|
|
}
|
|
|
|
public static function minioBackupRetentionBlockers(array $stats): array
|
|
{
|
|
foreach ($stats['buckets'] ?? [] as $bucket) {
|
|
if (!is_array($bucket) || (string)($bucket['name'] ?? '') !== self::MINIO_BACKUP_BUCKET) {
|
|
continue;
|
|
}
|
|
|
|
$expiredObjects = (int)($bucket['expired_objects'] ?? 0);
|
|
if ($expiredObjects <= 0) {
|
|
return [];
|
|
}
|
|
|
|
return [
|
|
'MinIO backup replica contains ' . $expiredObjects . ' backup object'
|
|
. ($expiredObjects === 1 ? '' : 's') . ' older than '
|
|
. self::MINIO_BACKUP_REPLICA_RETENTION_DAYS . ' days. Run provisioning to prune retained backups.',
|
|
];
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
public static function minioSpaceBlockers(?int $availableBytes, int $requiredBytes): array
|
|
{
|
|
if ($availableBytes === null) {
|
|
return [];
|
|
}
|
|
if ($availableBytes < $requiredBytes) {
|
|
return ['MinIO target does not have enough free space. Required ' . $requiredBytes . ' bytes, available ' . $availableBytes . ' bytes.'];
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
public static function normalizeMinioBuckets(mixed $value): array
|
|
{
|
|
if (is_string($value)) {
|
|
$value = preg_split('/[\s,]+/', $value);
|
|
}
|
|
if (!is_array($value)) {
|
|
$value = self::MINIO_DEFAULT_BUCKETS;
|
|
}
|
|
|
|
$buckets = [];
|
|
foreach ($value as $bucket) {
|
|
$bucket = strtolower(trim((string)$bucket));
|
|
if ($bucket === '' || preg_match('/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/', $bucket) !== 1) {
|
|
continue;
|
|
}
|
|
$buckets[] = $bucket;
|
|
}
|
|
|
|
$buckets = array_values(array_unique($buckets));
|
|
return $buckets !== [] ? $buckets : self::MINIO_DEFAULT_BUCKETS;
|
|
}
|
|
|
|
private static function minioBucketUsesBoundedReplicaRetention(string $bucket): bool
|
|
{
|
|
return strtolower(trim($bucket)) === self::MINIO_BACKUP_BUCKET;
|
|
}
|
|
|
|
public static function minioBucketCountsTowardCatchUp(string $bucket): bool
|
|
{
|
|
return !self::minioBucketUsesBoundedReplicaRetention($bucket);
|
|
}
|
|
|
|
private static function minioReplicaRetentionDaysByBucket(array $buckets): array
|
|
{
|
|
$retention = [];
|
|
foreach ($buckets as $bucket) {
|
|
if (self::minioBucketUsesBoundedReplicaRetention((string)$bucket)) {
|
|
$retention[(string)$bucket] = self::MINIO_BACKUP_REPLICA_RETENTION_DAYS;
|
|
}
|
|
}
|
|
|
|
return $retention;
|
|
}
|
|
|
|
public static function minioDefaultReplicationTransferLimit(): string
|
|
{
|
|
return self::MINIO_DEFAULT_REPLICATION_TRANSFER_LIMIT;
|
|
}
|
|
|
|
public static function normalizeMinioTransferLimit(mixed $value, bool $defaultWhenEmpty = true): string
|
|
{
|
|
$raw = trim((string)$value);
|
|
if ($raw === '') {
|
|
return $defaultWhenEmpty ? self::MINIO_DEFAULT_REPLICATION_TRANSFER_LIMIT : '';
|
|
}
|
|
|
|
$normalized = preg_replace('/\s+/', '', $raw) ?? $raw;
|
|
$lower = strtolower($normalized);
|
|
if (in_array($lower, ['0', 'none', 'off', 'unlimited', 'disabled'], true)) {
|
|
return '';
|
|
}
|
|
|
|
$normalized = preg_replace('/\/s$/i', '', $normalized) ?? $normalized;
|
|
if (preg_match('/^(\d+(?:\.\d+)?)([a-zA-Z]*)$/', $normalized, $matches) !== 1) {
|
|
throw new RuntimeException('MinIO transfer limit must be empty, 0, or a rate like 25Mi, 100M, or 1G.');
|
|
}
|
|
|
|
$amount = $matches[1];
|
|
if (str_contains($amount, '.')) {
|
|
$amount = rtrim(rtrim($amount, '0'), '.');
|
|
}
|
|
if ($amount === '' || (float)$amount <= 0) {
|
|
return '';
|
|
}
|
|
|
|
$unit = $matches[2];
|
|
$unitMap = [
|
|
'' => '',
|
|
'b' => 'B',
|
|
'k' => 'K',
|
|
'kb' => 'K',
|
|
'm' => 'M',
|
|
'mb' => 'M',
|
|
'g' => 'G',
|
|
'gb' => 'G',
|
|
't' => 'T',
|
|
'tb' => 'T',
|
|
'ki' => 'Ki',
|
|
'kib' => 'Ki',
|
|
'mi' => 'Mi',
|
|
'mib' => 'Mi',
|
|
'gi' => 'Gi',
|
|
'gib' => 'Gi',
|
|
'ti' => 'Ti',
|
|
'tib' => 'Ti',
|
|
];
|
|
$unitKey = strtolower($unit);
|
|
if (!array_key_exists($unitKey, $unitMap)) {
|
|
throw new RuntimeException('MinIO transfer limit must use B, K, M, G, T, Ki, Mi, Gi, or Ti units.');
|
|
}
|
|
|
|
return $amount . $unitMap[$unitKey];
|
|
}
|
|
|
|
private static function minioReplicationTransferLimitFromOptions(array $options): string
|
|
{
|
|
foreach (['replication_transfer_limit', 'transfer_limit', 'bandwidth_limit'] as $key) {
|
|
if (array_key_exists($key, $options)) {
|
|
return self::normalizeMinioTransferLimit($options[$key], false);
|
|
}
|
|
}
|
|
|
|
return self::MINIO_DEFAULT_REPLICATION_TRANSFER_LIMIT;
|
|
}
|
|
|
|
private static function minioReplicationTransferLimitArgs(string $transferLimit): array
|
|
{
|
|
$transferLimit = self::normalizeMinioTransferLimit($transferLimit, false);
|
|
if ($transferLimit === '') {
|
|
return [];
|
|
}
|
|
|
|
return ['--limit-upload', $transferLimit, '--limit-download', $transferLimit];
|
|
}
|
|
|
|
private static function minioReplicationProgressFromText(string $output): ?array
|
|
{
|
|
if (preg_match_all('/(?<![\d.])(\d+(?:\.\d+)?)\s*%/', $output, $matches) < 1) {
|
|
return null;
|
|
}
|
|
|
|
$percentages = array_map('floatval', $matches[1]);
|
|
if ($percentages === []) {
|
|
return null;
|
|
}
|
|
|
|
$nonZero = array_values(array_filter($percentages, static fn(float $percent): bool => $percent > 0.0));
|
|
$percent = $nonZero !== [] ? min($nonZero) : 0.0;
|
|
$percent = round(min(100.0, max(0.0, $percent)), 2);
|
|
|
|
return [
|
|
'replication_percent' => $percent,
|
|
'blockers' => $percent < 100.0 ? [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER] : [],
|
|
'basis' => 'text_percent',
|
|
'stats' => [
|
|
'percent_values' => $percentages,
|
|
],
|
|
];
|
|
}
|
|
|
|
private static function collectMinioReplicationProgress(mixed $value, array &$stats, array $path = []): void
|
|
{
|
|
if (is_object($value)) {
|
|
$value = get_object_vars($value);
|
|
}
|
|
if (!is_array($value)) {
|
|
return;
|
|
}
|
|
|
|
foreach ($value as $key => $entry) {
|
|
$normalizedKey = self::normalizeMinioProgressKey((string)$key);
|
|
$nextPath = array_values(array_filter(array_merge($path, [$normalizedKey]), static fn(string $part): bool => $part !== ''));
|
|
|
|
if (is_numeric($entry)) {
|
|
self::collectMinioReplicationProgressNumber($nextPath, (float)$entry, $stats);
|
|
continue;
|
|
}
|
|
|
|
if (is_string($entry)) {
|
|
self::collectMinioReplicationProgressString($entry, $stats);
|
|
$textProgress = self::minioReplicationProgressFromText($entry);
|
|
if ($textProgress !== null) {
|
|
$stats['completed_count'] += (float)$textProgress['replication_percent'];
|
|
$stats['total_count'] += 100.0;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
self::collectMinioReplicationProgress($entry, $stats, $nextPath);
|
|
}
|
|
}
|
|
|
|
private static function collectMinioReplicationProgressNumber(array $path, float $value, array &$stats): void
|
|
{
|
|
if ($value < 0.0) {
|
|
return;
|
|
}
|
|
|
|
$pathText = implode('', $path);
|
|
foreach ([
|
|
'percent',
|
|
'percentage',
|
|
'duration',
|
|
'elapsed',
|
|
'timestamp',
|
|
'time',
|
|
'priority',
|
|
'port',
|
|
'versionid',
|
|
'avg',
|
|
'average',
|
|
'peak',
|
|
'rate',
|
|
'latency',
|
|
'uptime',
|
|
'downtime',
|
|
'lastminute',
|
|
'lasthour',
|
|
'last1hr',
|
|
'last1m',
|
|
'last5min',
|
|
'sinceuptime',
|
|
] as $ignored) {
|
|
if (str_contains($pathText, $ignored)) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
$category = null;
|
|
foreach (['failed', 'failure', 'failures', 'error', 'errors'] as $needle) {
|
|
if (str_contains($pathText, $needle)) {
|
|
$category = 'failed';
|
|
break;
|
|
}
|
|
}
|
|
if ($category === null) {
|
|
foreach (['pending', 'queued', 'queue', 'backlog', 'remaining', 'unreplicated', 'inprogress', 'missing'] as $needle) {
|
|
if (str_contains($pathText, $needle)) {
|
|
$category = 'pending';
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if ($category === null) {
|
|
foreach (['completed', 'complete', 'replicated', 'replicate', 'replica', 'success', 'synced'] as $needle) {
|
|
if (str_contains($pathText, $needle)) {
|
|
$category = 'completed';
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if ($category === null && str_contains($pathText, 'total')) {
|
|
$category = 'total';
|
|
}
|
|
if ($category === null) {
|
|
return;
|
|
}
|
|
|
|
$isBytes = str_contains($pathText, 'byte')
|
|
|| str_contains($pathText, 'bytes')
|
|
|| str_contains($pathText, 'size');
|
|
$suffix = $isBytes ? 'bytes' : 'count';
|
|
$stats[$category . '_' . $suffix] += $value;
|
|
}
|
|
|
|
private static function collectMinioReplicationProgressString(string $value, array &$stats): void
|
|
{
|
|
$normalized = self::normalizeMinioProgressKey($value);
|
|
if ($normalized === '') {
|
|
return;
|
|
}
|
|
|
|
foreach (['pending', 'queued', 'backlog', 'replicating', 'syncing', 'inprogress', 'failed', 'failure', 'error'] as $needle) {
|
|
if (str_contains($normalized, $needle)) {
|
|
$stats['incomplete_signals']++;
|
|
return;
|
|
}
|
|
}
|
|
|
|
foreach (['completed', 'complete', 'replicated', 'synced', 'success', 'healthy', 'ok'] as $needle) {
|
|
if (str_contains($normalized, $needle)) {
|
|
$stats['complete_signals']++;
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
private static function normalizeMinioProgressKey(string $value): string
|
|
{
|
|
return strtolower((string)preg_replace('/[^a-zA-Z0-9]+/', '', $value));
|
|
}
|
|
|
|
public static function mariadbGtidCoveragePercent(string $sourceSet, string $replicaSet): float
|
|
{
|
|
$source = self::parseMariaDbGtidSet($sourceSet);
|
|
$replica = self::parseMariaDbGtidSet($replicaSet);
|
|
$total = array_sum($source);
|
|
if ($total <= 0) {
|
|
return 100.0;
|
|
}
|
|
|
|
$covered = 0;
|
|
foreach ($source as $domain => $sourceSequence) {
|
|
$covered += min($sourceSequence, $replica[$domain] ?? 0);
|
|
}
|
|
|
|
return round(min(100, max(0, ($covered / $total) * 100)), 2);
|
|
}
|
|
|
|
private static function parseMariaDbGtidSet(string $gtidSet): array
|
|
{
|
|
$positions = [];
|
|
foreach (explode(',', trim($gtidSet)) as $gtid) {
|
|
$gtid = trim($gtid);
|
|
if ($gtid === '') {
|
|
continue;
|
|
}
|
|
|
|
$parts = explode('-', $gtid);
|
|
if (count($parts) !== 3) {
|
|
continue;
|
|
}
|
|
|
|
[$domain, , $sequence] = array_map('intval', $parts);
|
|
if ($sequence <= 0) {
|
|
continue;
|
|
}
|
|
|
|
$positions[$domain] = max($positions[$domain] ?? 0, $sequence);
|
|
}
|
|
|
|
return $positions;
|
|
}
|
|
|
|
private static function parseMysqlGtidSet(string $gtidSet): array
|
|
{
|
|
$parsed = [];
|
|
foreach (explode(',', trim($gtidSet)) as $uuidSet) {
|
|
$uuidSet = trim($uuidSet);
|
|
if ($uuidSet === '') {
|
|
continue;
|
|
}
|
|
|
|
$parts = explode(':', $uuidSet);
|
|
if (count($parts) < 2) {
|
|
continue;
|
|
}
|
|
|
|
$uuid = strtolower(array_shift($parts));
|
|
foreach ($parts as $interval) {
|
|
if (str_contains($interval, '-')) {
|
|
[$start, $end] = array_map('intval', explode('-', $interval, 2));
|
|
} else {
|
|
$start = $end = (int)$interval;
|
|
}
|
|
if ($start <= 0 || $end <= 0) {
|
|
continue;
|
|
}
|
|
if ($end < $start) {
|
|
[$start, $end] = [$end, $start];
|
|
}
|
|
$parsed[$uuid][] = [$start, $end];
|
|
}
|
|
}
|
|
|
|
foreach ($parsed as $uuid => $intervals) {
|
|
usort($intervals, static fn(array $a, array $b): int => $a[0] <=> $b[0]);
|
|
$merged = [];
|
|
foreach ($intervals as [$start, $end]) {
|
|
$lastIndex = count($merged) - 1;
|
|
if ($lastIndex >= 0 && $start <= $merged[$lastIndex][1] + 1) {
|
|
$merged[$lastIndex][1] = max($merged[$lastIndex][1], $end);
|
|
continue;
|
|
}
|
|
$merged[] = [$start, $end];
|
|
}
|
|
$parsed[$uuid] = $merged;
|
|
}
|
|
|
|
return $parsed;
|
|
}
|
|
|
|
private function provisionDatabaseHost(array $host, int $operationId): array
|
|
{
|
|
$primary = $this->primaryHost(self::KIND_DATABASE);
|
|
if ($primary === null) {
|
|
throw new RuntimeException('No database primary is registered.');
|
|
}
|
|
|
|
$options = $this->decodeOptions($host);
|
|
$usePreseededReplica = !empty($options['allow_preseeded_replica']);
|
|
$targetStatus = $this->testDatabaseHost(array_merge($host, [
|
|
'test_connectivity_only' => true,
|
|
'connect_without_database' => $usePreseededReplica,
|
|
]));
|
|
$primaryStatus = $this->testDatabaseHost($primary);
|
|
$blockers = array_merge($targetStatus['blockers'], $primaryStatus['blockers']);
|
|
$targetEngine = self::databaseEngine($targetStatus['raw'] ?? []);
|
|
$primaryEngine = self::databaseEngine($primaryStatus['raw'] ?? []);
|
|
$targetEngineKnown = self::databaseEngineKnown($targetStatus['raw'] ?? []);
|
|
$primaryEngineKnown = self::databaseEngineKnown($primaryStatus['raw'] ?? []);
|
|
|
|
if (($targetStatus['raw']['server_id'] ?? null) !== null
|
|
&& ($primaryStatus['raw']['server_id'] ?? null) !== null
|
|
&& (int)$targetStatus['raw']['server_id'] === (int)$primaryStatus['raw']['server_id']) {
|
|
$blockers[] = 'Database replica must have a unique server_id.';
|
|
}
|
|
if ($targetEngineKnown && $primaryEngineKnown && $targetEngine !== $primaryEngine) {
|
|
$blockers[] = 'Database primary and replica must use the same engine family.';
|
|
}
|
|
|
|
$cloneReady = (bool)($targetStatus['raw']['clone_plugin_active'] ?? false);
|
|
$primaryCloneReady = (bool)($primaryStatus['raw']['clone_plugin_active'] ?? false);
|
|
if ($targetEngineKnown && $targetEngine === 'mariadb' && !$usePreseededReplica) {
|
|
$blockers[] = 'MariaDB replicas must be safely seeded before managed replication can be configured.';
|
|
}
|
|
if ($targetEngineKnown && $targetEngine === 'mysql' && !$usePreseededReplica) {
|
|
if (!$cloneReady) {
|
|
$blockers[] = 'MySQL Clone plugin is not active on the target. Set allow_preseeded_replica only after the target has been safely seeded.';
|
|
}
|
|
if (!$primaryCloneReady) {
|
|
$blockers[] = 'MySQL Clone plugin is not active on the primary donor.';
|
|
}
|
|
}
|
|
|
|
if ($blockers !== []) {
|
|
$this->storeStatus($host, array_replace($targetStatus, ['blockers' => array_values(array_unique($blockers))]));
|
|
return [
|
|
'ok' => false,
|
|
'message' => 'Database replica provisioning is blocked.',
|
|
'blockers' => array_values(array_unique($blockers)),
|
|
'replication_percent' => $targetStatus['replication_percent'],
|
|
'host' => $this->publicHost($host),
|
|
];
|
|
}
|
|
|
|
$target = $this->databaseConnection($host, true, $usePreseededReplica);
|
|
try {
|
|
$primaryCredentials = $this->credentials($primary);
|
|
$hostCredentials = $this->credentials($host);
|
|
$replicationUser = $hostCredentials['replication_username']
|
|
?: ($primaryCredentials['replication_username'] ?: $primaryCredentials['username']);
|
|
$replicationPassword = $hostCredentials['replication_password']
|
|
?: ($primaryCredentials['replication_password'] ?: $primaryCredentials['password']);
|
|
$shouldManageReplicationUser = ($hostCredentials['replication_username'] ?? '') !== ''
|
|
&& ($hostCredentials['replication_password'] ?? '') !== '';
|
|
|
|
if ($usePreseededReplica && $targetEngine === 'mariadb') {
|
|
$seedContext = $this->operationContext($operationId);
|
|
$seedInProgress = ($seedContext['phase'] ?? '') !== '' && ($seedContext['phase'] ?? '') !== 'complete';
|
|
$seedBlockers = $this->databaseReplicaSeedBlockers($primary, $host, $target);
|
|
if ($seedInProgress || $seedBlockers !== []) {
|
|
$seedResult = $this->advanceMariaDbReplicaSeed($operationId, $primary, $host, $target);
|
|
$targetStatus = $seedResult['status'];
|
|
|
|
if (($seedResult['running'] ?? false) === true) {
|
|
$this->storeStatus($host, $targetStatus);
|
|
return [
|
|
'ok' => true,
|
|
'message' => $seedResult['message'],
|
|
'blockers' => [],
|
|
'replication_percent' => $targetStatus['replication_percent'],
|
|
'operation' => [
|
|
'id' => $operationId,
|
|
'status' => 'running',
|
|
'progress_percent' => $targetStatus['replication_percent'],
|
|
'message' => $seedResult['message'],
|
|
],
|
|
'host' => $this->publicHost($host),
|
|
];
|
|
}
|
|
}
|
|
} elseif ($usePreseededReplica) {
|
|
$seedBlockers = $this->databaseReplicaSeedBlockers($primary, $host, $target);
|
|
if ($seedBlockers !== []) {
|
|
$combinedBlockers = array_values(array_unique(array_merge($targetStatus['blockers'], $seedBlockers)));
|
|
$this->storeStatus($host, array_replace($targetStatus, [
|
|
'status' => 'degraded',
|
|
'replication_percent' => min(99.99, (float)$targetStatus['replication_percent']),
|
|
'blockers' => $combinedBlockers,
|
|
]));
|
|
return [
|
|
'ok' => false,
|
|
'message' => 'Database replica provisioning is blocked.',
|
|
'blockers' => $combinedBlockers,
|
|
'replication_percent' => min(99.99, (float)$targetStatus['replication_percent']),
|
|
'host' => $this->publicHost($host),
|
|
];
|
|
}
|
|
}
|
|
|
|
if ($shouldManageReplicationUser) {
|
|
$grantHosts = $this->databaseReplicationGrantHosts($host, $targetStatus);
|
|
$this->ensureDatabaseReplicationUser($primary, $replicationUser, $replicationPassword, $grantHosts);
|
|
}
|
|
|
|
if ($targetEngine === 'mysql' && !$usePreseededReplica) {
|
|
$this->runMysqlClone($target, $primary, $replicationUser, $replicationPassword);
|
|
$target->close();
|
|
$target = $this->waitForDatabaseConnection($host, true, 120);
|
|
}
|
|
|
|
if ($targetEngine === 'mariadb') {
|
|
$this->configureMariaDbReplication($target, $primary, $host, $replicationUser, $replicationPassword);
|
|
} else {
|
|
$this->configureMySqlReplication($target, $primary, $replicationUser, $replicationPassword);
|
|
}
|
|
} finally {
|
|
$target->close();
|
|
}
|
|
|
|
$this->execute(
|
|
"UPDATE replication_hosts SET replication_source_id = ?, status = 'provisioning' WHERE id = ?",
|
|
'ii',
|
|
[(int)$primary['id'], (int)$host['id']]
|
|
);
|
|
|
|
$status = $this->testDatabaseHost($this->getHost(self::KIND_DATABASE, (int)$host['id']));
|
|
$this->storeStatus($host, $status);
|
|
|
|
return [
|
|
'ok' => true,
|
|
'healthy' => $status['blockers'] === [],
|
|
'message' => $targetEngine === 'mariadb'
|
|
? 'MariaDB replication was configured with GTID slave_pos.'
|
|
: 'Database replication was configured with GTID auto-positioning.',
|
|
'blockers' => $status['blockers'],
|
|
'replication_percent' => $status['replication_percent'],
|
|
'host' => $this->publicHost($this->getHost(self::KIND_DATABASE, (int)$host['id'])),
|
|
];
|
|
}
|
|
|
|
private function databaseReplicationGrantHosts(array $host, array $targetStatus = []): array
|
|
{
|
|
$grantHosts = ['%'];
|
|
if (isset($host['host'])) {
|
|
$grantHosts[] = (string)$host['host'];
|
|
}
|
|
|
|
$lastStatus = self::jsonDecode($host['last_status_json'] ?? null);
|
|
foreach ([$targetStatus, $lastStatus] as $status) {
|
|
if (!is_array($status)) {
|
|
continue;
|
|
}
|
|
|
|
foreach ($this->databaseDeniedAccountHostsFromStatus($status) as $deniedHost) {
|
|
$grantHosts[] = $deniedHost;
|
|
}
|
|
}
|
|
|
|
$normalized = [];
|
|
foreach ($grantHosts as $grantHost) {
|
|
foreach (self::databaseAccountHostGrantCandidates((string)$grantHost) as $candidate) {
|
|
if (!in_array($candidate, $normalized, true)) {
|
|
$normalized[] = $candidate;
|
|
}
|
|
}
|
|
}
|
|
|
|
return $normalized === [] ? ['%'] : $normalized;
|
|
}
|
|
|
|
private function databaseDeniedAccountHostsFromStatus(array $status): array
|
|
{
|
|
$hosts = [];
|
|
|
|
foreach ($status['blockers'] ?? [] as $blocker) {
|
|
if (is_scalar($blocker)) {
|
|
$hosts = array_merge($hosts, self::databaseDeniedAccountHostsFromText((string)$blocker));
|
|
}
|
|
}
|
|
|
|
$replicaStatus = $status['raw']['replica_status'] ?? [];
|
|
if (is_array($replicaStatus)) {
|
|
foreach (['Last_IO_Error', 'Last_SQL_Error', 'Last_Error'] as $errorKey) {
|
|
$error = trim((string)($replicaStatus[$errorKey] ?? ''));
|
|
if ($error !== '') {
|
|
$hosts = array_merge($hosts, self::databaseDeniedAccountHostsFromText($error));
|
|
}
|
|
}
|
|
}
|
|
|
|
return array_values(array_unique($hosts));
|
|
}
|
|
|
|
private static function databaseDeniedAccountHostsFromText(string $text): array
|
|
{
|
|
preg_match_all('/Access denied for user\s+[\'"][^\'"]+[\'"]@[\'"]([^\'"]+)[\'"]/i', $text, $matches);
|
|
return array_values(array_unique(array_filter($matches[1] ?? [])));
|
|
}
|
|
|
|
private static function normalizeDatabaseAccountHost(string $host): ?string
|
|
{
|
|
$host = trim($host);
|
|
if ($host === '') {
|
|
return null;
|
|
}
|
|
|
|
if ($host !== '%') {
|
|
$host = trim($host, '[]');
|
|
}
|
|
|
|
if ($host === '' || strlen($host) > 255) {
|
|
return null;
|
|
}
|
|
|
|
if (preg_match('/[\s\'"`;\\\\]/', $host)) {
|
|
return null;
|
|
}
|
|
|
|
return preg_match('/^[A-Za-z0-9_.:%-]+$/', $host) === 1 ? $host : null;
|
|
}
|
|
|
|
private static function databaseAccountHostGrantCandidates(string $host): array
|
|
{
|
|
$host = self::normalizeDatabaseAccountHost($host);
|
|
if ($host === null) {
|
|
return [];
|
|
}
|
|
|
|
$candidates = [$host];
|
|
if ($host !== '%' && !str_contains($host, '%')) {
|
|
if (filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) !== false) {
|
|
$lastDot = strrpos($host, '.');
|
|
if ($lastDot !== false) {
|
|
$candidates[] = substr($host, 0, $lastDot + 1) . '%';
|
|
}
|
|
} elseif (str_contains($host, ':')) {
|
|
$lastColon = strrpos($host, ':');
|
|
if ($lastColon !== false) {
|
|
$candidates[] = substr($host, 0, $lastColon + 1) . '%';
|
|
}
|
|
}
|
|
}
|
|
|
|
return array_values(array_unique(array_filter(array_map(
|
|
static fn(string $candidate): ?string => self::normalizeDatabaseAccountHost($candidate),
|
|
$candidates
|
|
))));
|
|
}
|
|
|
|
private function ensureDatabaseReplicationUser(array $primary, string $replicationUser, string $replicationPassword, array $grantHosts = []): void
|
|
{
|
|
if (trim($replicationUser) === '' || trim($replicationPassword) === '') {
|
|
throw new RuntimeException('Replication username and password are required.');
|
|
}
|
|
|
|
$connection = $this->databaseConnection($primary, true);
|
|
try {
|
|
$grantHosts = $grantHosts === [] ? ['%'] : $grantHosts;
|
|
$user = $connection->real_escape_string($replicationUser);
|
|
$password = $connection->real_escape_string($replicationPassword);
|
|
|
|
foreach ($grantHosts as $grantHost) {
|
|
$grantHost = self::normalizeDatabaseAccountHost((string)$grantHost);
|
|
if ($grantHost === null) {
|
|
continue;
|
|
}
|
|
|
|
$account = sprintf(
|
|
"'%s'@'%s'",
|
|
$user,
|
|
$connection->real_escape_string($grantHost)
|
|
);
|
|
|
|
$this->mysqliExec($connection, "CREATE USER IF NOT EXISTS " . $account . " IDENTIFIED BY '" . $password . "'");
|
|
$this->mysqliExec($connection, "ALTER USER " . $account . " IDENTIFIED BY '" . $password . "'");
|
|
$this->mysqliExec($connection, "GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " . $account);
|
|
}
|
|
$this->mysqliExec($connection, 'FLUSH PRIVILEGES');
|
|
} catch (Throwable $throwable) {
|
|
throw new RuntimeException(
|
|
'Could not create or update the replication user on the primary database. Add primary admin credentials or create the replication user manually: ' . $throwable->getMessage(),
|
|
0,
|
|
$throwable
|
|
);
|
|
} finally {
|
|
$connection->close();
|
|
}
|
|
}
|
|
|
|
private function configureMySqlReplication(mysqli $target, array $primary, string $replicationUser, string $replicationPassword): void
|
|
{
|
|
try {
|
|
$this->mysqliExec($target, 'STOP REPLICA');
|
|
} catch (Throwable) {
|
|
}
|
|
$sql = sprintf(
|
|
"CHANGE REPLICATION SOURCE TO SOURCE_HOST = '%s', SOURCE_PORT = %d, SOURCE_USER = '%s', SOURCE_PASSWORD = '%s', SOURCE_AUTO_POSITION = 1",
|
|
$target->real_escape_string((string)$primary['host']),
|
|
(int)$primary['port'],
|
|
$target->real_escape_string($replicationUser),
|
|
$target->real_escape_string($replicationPassword)
|
|
);
|
|
$this->mysqliExec($target, $sql);
|
|
$this->mysqliExec($target, 'START REPLICA');
|
|
}
|
|
|
|
private function configureMariaDbReplication(mysqli $target, array $primary, array $host, string $replicationUser, string $replicationPassword): void
|
|
{
|
|
foreach (['STOP SLAVE', 'RESET SLAVE ALL'] as $statement) {
|
|
try {
|
|
$this->mysqliExec($target, $statement);
|
|
} catch (Throwable) {
|
|
}
|
|
}
|
|
$this->configureMariaDbReplicationFilters($target, $primary, $host);
|
|
$sql = sprintf(
|
|
"CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, MASTER_USER = '%s', MASTER_PASSWORD = '%s', MASTER_USE_GTID = slave_pos",
|
|
$target->real_escape_string((string)$primary['host']),
|
|
(int)$primary['port'],
|
|
$target->real_escape_string($replicationUser),
|
|
$target->real_escape_string($replicationPassword)
|
|
);
|
|
$this->mysqliExec($target, $sql);
|
|
$this->mysqliExec($target, 'START SLAVE');
|
|
}
|
|
|
|
private function configureMariaDbReplicationFilters(mysqli $target, array $primary, array $host): void
|
|
{
|
|
$existing = $this->mysqliSelectOne($target, "SHOW GLOBAL VARIABLES LIKE 'replicate_ignore_table'");
|
|
$filters = array_values(array_filter(array_map(
|
|
static fn(string $filter): string => trim($filter),
|
|
explode(',', (string)($existing['Value'] ?? ''))
|
|
)));
|
|
|
|
foreach ([(string)($primary['database_name'] ?? ''), (string)($host['database_name'] ?? '')] as $database) {
|
|
$database = trim($database);
|
|
if ($database !== '') {
|
|
foreach (self::MARIADB_SCHEMA_ONLY_TABLES as $tableName) {
|
|
$filters[] = $database . '.' . $tableName;
|
|
}
|
|
}
|
|
}
|
|
|
|
$filters = array_values(array_unique($filters));
|
|
if ($filters === []) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
$this->mysqliExec($target, 'SET GLOBAL replicate_ignore_table = ' . self::sqlString($target, implode(',', $filters)));
|
|
} catch (Throwable $throwable) {
|
|
throw new RuntimeException(
|
|
'Could not configure MariaDB replica schema-only table filters: ' . $throwable->getMessage(),
|
|
0,
|
|
$throwable
|
|
);
|
|
}
|
|
}
|
|
|
|
private function runMysqlClone(mysqli $target, array $primary, string $cloneUser, string $clonePassword): void
|
|
{
|
|
$donor = $target->real_escape_string((string)$primary['host'] . ':' . (int)$primary['port']);
|
|
$this->mysqliExec($target, "SET GLOBAL clone_valid_donor_list = '" . $donor . "'");
|
|
|
|
$sql = sprintf(
|
|
"CLONE INSTANCE FROM '%s'@'%s':%d IDENTIFIED BY '%s'",
|
|
$target->real_escape_string($cloneUser),
|
|
$target->real_escape_string((string)$primary['host']),
|
|
(int)$primary['port'],
|
|
$target->real_escape_string($clonePassword)
|
|
);
|
|
|
|
try {
|
|
$this->mysqliExec($target, $sql);
|
|
} catch (Throwable $throwable) {
|
|
$message = strtolower($throwable->getMessage());
|
|
if (!str_contains($message, 'lost connection') && !str_contains($message, 'server has gone away')) {
|
|
throw $throwable;
|
|
}
|
|
}
|
|
}
|
|
|
|
private function waitForDatabaseConnection(array $host, bool $admin, int $timeoutSeconds): mysqli
|
|
{
|
|
$deadline = time() + max(1, $timeoutSeconds);
|
|
$lastError = null;
|
|
|
|
do {
|
|
try {
|
|
return $this->databaseConnection($host, $admin);
|
|
} catch (Throwable $throwable) {
|
|
$lastError = $throwable;
|
|
sleep(2);
|
|
}
|
|
} while (time() < $deadline);
|
|
|
|
throw new RuntimeException('Database target did not reconnect after MySQL Clone: ' . ($lastError?->getMessage() ?? 'timeout'));
|
|
}
|
|
|
|
private function provisionRedisHost(array $host, int $operationId): array
|
|
{
|
|
$primary = $this->primaryHost(self::KIND_REDIS);
|
|
if ($primary === null) {
|
|
throw new RuntimeException('No Redis primary is registered.');
|
|
}
|
|
|
|
$targetStatus = $this->testRedisHost(array_merge($host, ['test_connectivity_only' => true]));
|
|
$primaryStatus = $this->testRedisHost($primary);
|
|
$blockers = array_merge($targetStatus['blockers'], $primaryStatus['blockers']);
|
|
if ($blockers !== []) {
|
|
$this->storeStatus($host, array_replace($targetStatus, ['blockers' => array_values(array_unique($blockers))]));
|
|
return [
|
|
'ok' => false,
|
|
'message' => 'Redis replica provisioning is blocked.',
|
|
'blockers' => array_values(array_unique($blockers)),
|
|
'replication_percent' => $targetStatus['replication_percent'],
|
|
'host' => $this->publicHost($host),
|
|
];
|
|
}
|
|
|
|
$context = $this->operationContext($operationId);
|
|
if (($context['phase'] ?? '') !== 'configured') {
|
|
try {
|
|
$client = $this->redisClient($host);
|
|
$primaryCredentials = $this->credentials($primary);
|
|
if (($primaryCredentials['username'] ?? '') !== '' && ($primaryCredentials['username'] ?? '') !== 'default') {
|
|
$client->executeRaw(['CONFIG', 'SET', 'masteruser', (string)$primaryCredentials['username']]);
|
|
}
|
|
$client->executeRaw(['CONFIG', 'SET', 'masterauth', (string)($primaryCredentials['password'] ?? '')]);
|
|
$client->executeRaw(['REPLICAOF', (string)$primary['host'], (string)$primary['port']]);
|
|
$client->executeRaw(['CONFIG', 'REWRITE']);
|
|
} catch (Throwable $throwable) {
|
|
$blockers = [$throwable->getMessage()];
|
|
$this->storeStatus($host, array_replace($targetStatus, [
|
|
'status' => 'degraded',
|
|
'replication_percent' => 0,
|
|
'blockers' => $blockers,
|
|
]));
|
|
return [
|
|
'ok' => false,
|
|
'message' => 'Redis replica provisioning is blocked.',
|
|
'blockers' => $blockers,
|
|
'replication_percent' => 0,
|
|
'host' => $this->publicHost($host),
|
|
];
|
|
}
|
|
|
|
$context = [
|
|
'phase' => 'configured',
|
|
'configured_at' => date('c'),
|
|
'primary_host' => (string)$primary['host'],
|
|
'primary_port' => (int)$primary['port'],
|
|
];
|
|
$this->updateOperationProgress(
|
|
$operationId,
|
|
5.0,
|
|
'Redis replication was configured; waiting for the replica to catch up.',
|
|
$context
|
|
);
|
|
}
|
|
|
|
$this->execute(
|
|
"UPDATE replication_hosts SET replication_source_id = ?, status = 'provisioning' WHERE id = ?",
|
|
'ii',
|
|
[(int)$primary['id'], (int)$host['id']]
|
|
);
|
|
|
|
$host = $this->getHost(self::KIND_REDIS, (int)$host['id']);
|
|
$status = $this->testRedisHost($host);
|
|
$syncBlockers = array_values(array_intersect($status['blockers'], [
|
|
'Redis host is not currently a replica.',
|
|
'Redis replica link to primary is not up.',
|
|
]));
|
|
$onlySyncBlockers = $status['blockers'] === []
|
|
|| ($syncBlockers !== [] && count($syncBlockers) === count($status['blockers']));
|
|
$progress = self::redisProvisionProgress((float)$status['replication_percent'], $syncBlockers);
|
|
|
|
if ($onlySyncBlockers && ((float)$status['replication_percent'] < 100.0 || $syncBlockers !== [])) {
|
|
$message = $syncBlockers !== []
|
|
? 'Redis replication is configured, but the replica is waiting for the primary link.'
|
|
: 'Redis replication is configured and syncing in the background.';
|
|
$this->storeStatus($host, array_replace($status, ['replication_percent' => $progress]));
|
|
$this->updateOperationProgress($operationId, $progress, $message, $context);
|
|
return [
|
|
'ok' => true,
|
|
'healthy' => false,
|
|
'message' => $message,
|
|
'blockers' => $status['blockers'],
|
|
'replication_percent' => $progress,
|
|
'host' => $this->publicHost($this->getHost(self::KIND_REDIS, (int)$host['id'])),
|
|
];
|
|
}
|
|
|
|
$this->storeStatus($host, $status);
|
|
if ($status['blockers'] !== []) {
|
|
return [
|
|
'ok' => false,
|
|
'message' => 'Redis replica provisioning is blocked.',
|
|
'blockers' => $status['blockers'],
|
|
'replication_percent' => $status['replication_percent'],
|
|
'host' => $this->publicHost($host),
|
|
];
|
|
}
|
|
|
|
return [
|
|
'ok' => true,
|
|
'healthy' => $status['blockers'] === [],
|
|
'message' => 'Redis replication was configured.',
|
|
'blockers' => $status['blockers'],
|
|
'replication_percent' => $status['replication_percent'],
|
|
'host' => $this->publicHost($this->getHost(self::KIND_REDIS, (int)$host['id'])),
|
|
];
|
|
}
|
|
|
|
private function provisionMinioHost(array $host, int $operationId): array
|
|
{
|
|
$primary = $this->primaryHost(self::KIND_MINIO);
|
|
if ($primary === null) {
|
|
throw new RuntimeException('No MinIO primary is registered.');
|
|
}
|
|
|
|
$targetStatus = $this->testMinioHost(array_merge($host, [
|
|
'test_connectivity_only' => true,
|
|
'skip_storage_scan' => true,
|
|
]));
|
|
$primaryStatus = $this->testMinioHost(array_merge($primary, [
|
|
'test_connectivity_only' => true,
|
|
'skip_storage_scan' => true,
|
|
]));
|
|
$blockers = array_values(array_unique(array_merge($targetStatus['blockers'], $primaryStatus['blockers'])));
|
|
if ($blockers !== []) {
|
|
$this->storeStatus($host, array_replace($targetStatus, ['blockers' => $blockers]));
|
|
return [
|
|
'ok' => false,
|
|
'message' => 'MinIO replica provisioning is blocked.',
|
|
'blockers' => $blockers,
|
|
'replication_percent' => $targetStatus['replication_percent'],
|
|
'host' => $this->publicHost($host),
|
|
];
|
|
}
|
|
|
|
$context = $this->operationContext($operationId);
|
|
$replicationConfigured = $this->minioReplicationConfiguredForHosts($primary, $host);
|
|
if (!$replicationConfigured) {
|
|
try {
|
|
$this->configureMinioReplication($primary, $host);
|
|
} catch (Throwable $throwable) {
|
|
$blockers = [$throwable->getMessage()];
|
|
$this->storeStatus($host, array_replace($targetStatus, [
|
|
'status' => 'degraded',
|
|
'replication_percent' => 0,
|
|
'blockers' => $blockers,
|
|
]));
|
|
return [
|
|
'ok' => false,
|
|
'message' => 'MinIO replica provisioning is blocked.',
|
|
'blockers' => $blockers,
|
|
'replication_percent' => 0,
|
|
'host' => $this->publicHost($host),
|
|
];
|
|
}
|
|
|
|
$context = [
|
|
'phase' => 'configured',
|
|
'configured_at' => date('c'),
|
|
'primary_endpoint' => self::minioEndpoint($primary),
|
|
];
|
|
$this->updateOperationProgress(
|
|
$operationId,
|
|
5.0,
|
|
'MinIO bucket replication was configured; waiting for buckets to catch up.',
|
|
$context
|
|
);
|
|
}
|
|
|
|
$this->execute(
|
|
"UPDATE replication_hosts SET replication_source_id = ?, status = 'provisioning' WHERE id = ?",
|
|
'ii',
|
|
[(int)$primary['id'], (int)$host['id']]
|
|
);
|
|
|
|
$host = $this->getHost(self::KIND_MINIO, (int)$host['id']);
|
|
$status = $this->minioProvisionStatus($primary, $host);
|
|
$this->storeStatus($host, $status);
|
|
|
|
$progress = self::minioProvisionProgress($status);
|
|
$syncInProgress = ((float)$status['replication_percent'] < 100.0 || $status['blockers'] !== [])
|
|
&& self::minioOnlyProgressBlockers($status['blockers']);
|
|
if ($syncInProgress) {
|
|
$message = self::minioProvisionProgressMessage($status);
|
|
$this->updateOperationProgress($operationId, $progress, $message, $context);
|
|
return [
|
|
'ok' => true,
|
|
'message' => $message,
|
|
'blockers' => $status['blockers'],
|
|
'replication_percent' => $progress,
|
|
'operation' => [
|
|
'id' => $operationId,
|
|
'status' => 'running',
|
|
'progress_percent' => $progress,
|
|
'message' => $message,
|
|
],
|
|
'host' => $this->publicHost($this->getHost(self::KIND_MINIO, (int)$host['id'])),
|
|
];
|
|
}
|
|
|
|
if ($status['blockers'] !== []) {
|
|
return [
|
|
'ok' => false,
|
|
'message' => 'MinIO replica provisioning is blocked.',
|
|
'blockers' => $status['blockers'],
|
|
'replication_percent' => $status['replication_percent'],
|
|
'host' => $this->publicHost($host),
|
|
];
|
|
}
|
|
|
|
return [
|
|
'ok' => true,
|
|
'healthy' => $status['blockers'] === [],
|
|
'message' => 'MinIO bucket replication was configured.',
|
|
'blockers' => $status['blockers'],
|
|
'replication_percent' => $status['replication_percent'],
|
|
'host' => $this->publicHost($this->getHost(self::KIND_MINIO, (int)$host['id'])),
|
|
];
|
|
}
|
|
|
|
private function promoteDatabaseHost(array $host): array
|
|
{
|
|
$status = $this->testDatabaseHost($host);
|
|
if ($status['blockers'] !== [] || (float)$status['replication_percent'] < 100.0) {
|
|
throw new RuntimeException('Database promotion blocked: ' . implode(' ', $status['blockers'] ?: ['Replica is not caught up.']));
|
|
}
|
|
|
|
$oldPrimary = $this->primaryHost(self::KIND_DATABASE);
|
|
if ($oldPrimary === null) {
|
|
throw new RuntimeException('No current database primary is registered.');
|
|
}
|
|
|
|
$oldPrimaryConn = null;
|
|
$targetConn = null;
|
|
$metadataSwitched = false;
|
|
|
|
try {
|
|
$oldPrimaryConn = $this->databaseConnection($oldPrimary, true);
|
|
$oldPrimaryStatus = $this->databaseServerStatus($oldPrimaryConn);
|
|
$this->setDatabaseReadOnly($oldPrimaryConn, $oldPrimaryStatus, true);
|
|
|
|
$targetConn = $this->databaseConnection($host, true);
|
|
$targetStatus = $this->databaseServerStatus($targetConn);
|
|
$this->stopDatabaseReplication($targetConn, $targetStatus);
|
|
$this->setDatabaseReadOnly($targetConn, $targetStatus, false);
|
|
|
|
$this->switchPrimary(self::KIND_DATABASE, (int)$host['id'], (int)$oldPrimary['id']);
|
|
$metadataSwitched = true;
|
|
$this->writeBootstrapSnapshot();
|
|
} catch (Throwable $throwable) {
|
|
if (!$metadataSwitched && $oldPrimaryConn instanceof mysqli) {
|
|
try {
|
|
$this->setDatabaseReadOnly($oldPrimaryConn, $oldPrimaryStatus ?? [], false);
|
|
} catch (Throwable) {
|
|
}
|
|
}
|
|
throw $throwable;
|
|
} finally {
|
|
if ($targetConn instanceof mysqli) {
|
|
$targetConn->close();
|
|
}
|
|
if ($oldPrimaryConn instanceof mysqli) {
|
|
$oldPrimaryConn->close();
|
|
}
|
|
}
|
|
|
|
return [
|
|
'ok' => true,
|
|
'message' => 'Database replica promoted to primary.',
|
|
'primary' => $this->publicHost($this->getHost(self::KIND_DATABASE, (int)$host['id'])),
|
|
'prior_primary' => $this->publicHost($this->getHost(self::KIND_DATABASE, (int)$oldPrimary['id'], true)),
|
|
'blockers' => [],
|
|
];
|
|
}
|
|
|
|
private function setDatabaseReadOnly(mysqli $connection, array $status, bool $readOnly): void
|
|
{
|
|
$value = $readOnly ? 'ON' : 'OFF';
|
|
if (array_key_exists('super_read_only', $status)) {
|
|
try {
|
|
$this->mysqliExec($connection, 'SET GLOBAL super_read_only = ' . $value);
|
|
} catch (Throwable) {
|
|
}
|
|
}
|
|
$this->mysqliExec($connection, 'SET GLOBAL read_only = ' . $value);
|
|
}
|
|
|
|
private function stopDatabaseReplication(mysqli $connection, array $status): void
|
|
{
|
|
if (self::databaseEngine($status) === 'mariadb') {
|
|
$this->mysqliExec($connection, 'STOP SLAVE');
|
|
return;
|
|
}
|
|
|
|
$this->mysqliExec($connection, 'STOP REPLICA');
|
|
}
|
|
|
|
private function promoteRedisHost(array $host): array
|
|
{
|
|
$status = $this->testRedisHost($host);
|
|
if ($status['blockers'] !== [] || (float)$status['replication_percent'] < 100.0) {
|
|
throw new RuntimeException('Redis promotion blocked: ' . implode(' ', $status['blockers'] ?: ['Replica is not caught up.']));
|
|
}
|
|
|
|
$oldPrimary = $this->primaryHost(self::KIND_REDIS);
|
|
if ($oldPrimary === null) {
|
|
throw new RuntimeException('No current Redis primary is registered.');
|
|
}
|
|
|
|
$client = $this->redisClient($host);
|
|
$client->executeRaw(['REPLICAOF', 'NO', 'ONE']);
|
|
try {
|
|
$client->executeRaw(['CONFIG', 'REWRITE']);
|
|
} catch (Throwable) {
|
|
}
|
|
|
|
$this->switchPrimary(self::KIND_REDIS, (int)$host['id'], (int)$oldPrimary['id']);
|
|
$this->writeBootstrapSnapshot();
|
|
|
|
return [
|
|
'ok' => true,
|
|
'message' => 'Redis replica promoted to primary.',
|
|
'primary' => $this->publicHost($this->getHost(self::KIND_REDIS, (int)$host['id'])),
|
|
'prior_primary' => $this->publicHost($this->getHost(self::KIND_REDIS, (int)$oldPrimary['id'], true)),
|
|
'blockers' => [],
|
|
];
|
|
}
|
|
|
|
private function promoteMinioHost(array $host): array
|
|
{
|
|
$status = $this->testMinioHost($host);
|
|
if ($status['blockers'] !== [] || (float)$status['replication_percent'] < 100.0) {
|
|
throw new RuntimeException('MinIO promotion blocked: ' . implode(' ', $status['blockers'] ?: ['Replica is not caught up.']));
|
|
}
|
|
|
|
$oldPrimary = $this->primaryHost(self::KIND_MINIO);
|
|
if ($oldPrimary === null) {
|
|
throw new RuntimeException('No current MinIO primary is registered.');
|
|
}
|
|
|
|
$this->switchPrimary(self::KIND_MINIO, (int)$host['id'], (int)$oldPrimary['id']);
|
|
$this->writeBootstrapSnapshot();
|
|
|
|
return [
|
|
'ok' => true,
|
|
'message' => 'MinIO replica promoted to primary.',
|
|
'primary' => $this->publicHost($this->getHost(self::KIND_MINIO, (int)$host['id'])),
|
|
'prior_primary' => $this->publicHost($this->getHost(self::KIND_MINIO, (int)$oldPrimary['id'], true)),
|
|
'blockers' => [],
|
|
];
|
|
}
|
|
|
|
private function runAutomaticFailoverForKind(string $kind, array $config, ?int $actorUserId): array
|
|
{
|
|
if (!replica_failover_manager::kindEnabled($config, $kind)) {
|
|
return [
|
|
'ok' => true,
|
|
'status' => 'skipped',
|
|
'reason' => 'disabled',
|
|
];
|
|
}
|
|
|
|
$primary = $this->primaryHost($kind);
|
|
if ($primary === null) {
|
|
return [
|
|
'ok' => false,
|
|
'status' => 'skipped',
|
|
'reason' => 'missing_primary',
|
|
];
|
|
}
|
|
|
|
if (!$this->primaryHostDown($kind, $primary)) {
|
|
return [
|
|
'ok' => true,
|
|
'status' => 'skipped',
|
|
'reason' => 'primary_healthy',
|
|
'primary' => $this->publicHost($primary),
|
|
];
|
|
}
|
|
|
|
$maxAgeSeconds = (int)$config['max_status_age_seconds'];
|
|
$candidate = replica_failover_manager::snapshotFailoverCandidate($this->listHosts($kind), $kind, $maxAgeSeconds);
|
|
if ($candidate === null) {
|
|
$result = [
|
|
'ok' => false,
|
|
'status' => 'blocked',
|
|
'reason' => 'no_fresh_caught_up_replica',
|
|
'primary' => $this->publicHost($primary),
|
|
];
|
|
$this->audit($kind, (int)$primary['id'], 'automatic_failover_blocked', $actorUserId, 'warning', $result);
|
|
return $result;
|
|
}
|
|
|
|
if (!replica_failover_manager::candidateReachable($kind, $candidate)) {
|
|
$result = [
|
|
'ok' => false,
|
|
'status' => 'blocked',
|
|
'reason' => 'candidate_unreachable',
|
|
'primary' => $this->publicHost($primary),
|
|
'candidate' => $this->publicHost($candidate),
|
|
];
|
|
$this->audit($kind, (int)$candidate['id'], 'automatic_failover_blocked', $actorUserId, 'warning', $result);
|
|
return $result;
|
|
}
|
|
|
|
$operationId = $this->startOperation($kind, (int)$candidate['id'], 'automatic_failover', $actorUserId);
|
|
$owner = 'replication-auto-failover-' . $kind . '-' . (int)$candidate['id'] . '-' . bin2hex(random_bytes(4));
|
|
$lockHandle = $this->acquirePromotionLock();
|
|
|
|
try {
|
|
application_write_freeze::freeze('Automatic replica failover in progress.', $owner, 600);
|
|
$result = match ($kind) {
|
|
self::KIND_DATABASE => $this->promoteDatabaseHostForFailover($candidate, $primary, $maxAgeSeconds),
|
|
self::KIND_REDIS => $this->promoteRedisHostForFailover($candidate, $primary, $maxAgeSeconds),
|
|
self::KIND_MINIO => $this->promoteMinioHostForFailover($candidate, $primary, $maxAgeSeconds),
|
|
};
|
|
|
|
$this->finishOperation($operationId, 'completed', 100, $result['message'] ?? null, []);
|
|
$this->audit($kind, (int)$candidate['id'], 'automatic_failover_promoted', $actorUserId, 'critical', $result);
|
|
return array_merge($result, [
|
|
'status' => 'promoted',
|
|
'candidate' => $this->publicHost($this->getHost($kind, (int)$candidate['id'])),
|
|
]);
|
|
} catch (Throwable $throwable) {
|
|
$this->finishOperation($operationId, 'failed', 0, null, [$throwable->getMessage()]);
|
|
$this->audit($kind, (int)$candidate['id'], 'automatic_failover_failed', $actorUserId, 'error', [
|
|
'error' => $throwable->getMessage(),
|
|
]);
|
|
return [
|
|
'ok' => false,
|
|
'status' => 'failed',
|
|
'reason' => $throwable->getMessage(),
|
|
'candidate' => $this->publicHost($candidate),
|
|
];
|
|
} finally {
|
|
application_write_freeze::unfreeze($owner);
|
|
$this->releasePromotionLock($lockHandle);
|
|
}
|
|
}
|
|
|
|
private function primaryHostDown(string $kind, array $primary): bool
|
|
{
|
|
$activeConfig = replica_failover_manager::activeConfigFromHost($kind, $primary);
|
|
if ($activeConfig === null) {
|
|
return false;
|
|
}
|
|
|
|
return replica_failover_manager::activePrimaryIsDown($kind, $activeConfig);
|
|
}
|
|
|
|
private function promoteDatabaseHostForFailover(array $host, array $oldPrimary, int $maxAgeSeconds): array
|
|
{
|
|
if (!replica_failover_manager::snapshotHostIsStrictlyFresh($host, $maxAgeSeconds)) {
|
|
throw new RuntimeException('Database failover blocked: replica status is not fresh and caught up.');
|
|
}
|
|
|
|
$targetConn = $this->databaseConnection($host, true);
|
|
try {
|
|
$targetStatus = $this->databaseServerStatus($targetConn);
|
|
$this->stopDatabaseReplication($targetConn, $targetStatus);
|
|
$this->setDatabaseReadOnly($targetConn, $targetStatus, false);
|
|
$this->switchPrimary(self::KIND_DATABASE, (int)$host['id'], (int)$oldPrimary['id']);
|
|
$this->writeBootstrapSnapshot();
|
|
replication_bootstrap_config::applyToGlobals(replication_bootstrap_config::loadSnapshot());
|
|
} finally {
|
|
$targetConn->close();
|
|
}
|
|
|
|
return [
|
|
'ok' => true,
|
|
'message' => 'Database replica promoted to primary after primary health check failed.',
|
|
'primary' => $this->publicHost($this->getHost(self::KIND_DATABASE, (int)$host['id'])),
|
|
'prior_primary' => $this->publicHost($this->getHost(self::KIND_DATABASE, (int)$oldPrimary['id'], true)),
|
|
'blockers' => [],
|
|
];
|
|
}
|
|
|
|
private function promoteRedisHostForFailover(array $host, array $oldPrimary, int $maxAgeSeconds): array
|
|
{
|
|
if (!replica_failover_manager::snapshotHostIsStrictlyFresh($host, $maxAgeSeconds)) {
|
|
throw new RuntimeException('Redis failover blocked: replica status is not fresh and caught up.');
|
|
}
|
|
|
|
$client = $this->redisClient($host);
|
|
$client->ping();
|
|
$client->executeRaw(['REPLICAOF', 'NO', 'ONE']);
|
|
try {
|
|
$client->executeRaw(['CONFIG', 'REWRITE']);
|
|
} catch (Throwable) {
|
|
}
|
|
|
|
$this->switchPrimary(self::KIND_REDIS, (int)$host['id'], (int)$oldPrimary['id']);
|
|
$this->writeBootstrapSnapshot();
|
|
replication_bootstrap_config::applyToGlobals(replication_bootstrap_config::loadSnapshot());
|
|
|
|
return [
|
|
'ok' => true,
|
|
'message' => 'Redis replica promoted to primary after primary health check failed.',
|
|
'primary' => $this->publicHost($this->getHost(self::KIND_REDIS, (int)$host['id'])),
|
|
'prior_primary' => $this->publicHost($this->getHost(self::KIND_REDIS, (int)$oldPrimary['id'], true)),
|
|
'blockers' => [],
|
|
];
|
|
}
|
|
|
|
private function promoteMinioHostForFailover(array $host, array $oldPrimary, int $maxAgeSeconds): array
|
|
{
|
|
if (!replica_failover_manager::snapshotHostIsStrictlyFresh($host, $maxAgeSeconds)) {
|
|
throw new RuntimeException('MinIO failover blocked: replica status is not fresh and caught up.');
|
|
}
|
|
|
|
$this->minioS3Client($host)->listBuckets();
|
|
$this->switchPrimary(self::KIND_MINIO, (int)$host['id'], (int)$oldPrimary['id']);
|
|
$this->writeBootstrapSnapshot();
|
|
replication_bootstrap_config::applyToGlobals(replication_bootstrap_config::loadSnapshot());
|
|
|
|
return [
|
|
'ok' => true,
|
|
'message' => 'MinIO replica endpoint selected after primary health check failed.',
|
|
'primary' => $this->publicHost($this->getHost(self::KIND_MINIO, (int)$host['id'])),
|
|
'prior_primary' => $this->publicHost($this->getHost(self::KIND_MINIO, (int)$oldPrimary['id'], true)),
|
|
'blockers' => [],
|
|
];
|
|
}
|
|
|
|
private function testDatabaseHost(array $host): array
|
|
{
|
|
$blockers = [];
|
|
$raw = [];
|
|
$percent = (float)(($host['role'] ?? '') === 'primary' ? 100 : 0);
|
|
$lagSeconds = null;
|
|
$reachable = true;
|
|
|
|
try {
|
|
$connection = $this->databaseConnection($host, true, !empty($host['connect_without_database']));
|
|
try {
|
|
$raw = $this->databaseServerStatus($connection);
|
|
$blockers = array_merge($blockers, self::databasePrerequisiteBlockers($raw));
|
|
|
|
if (($host['role'] ?? '') !== 'primary' && empty($host['test_connectivity_only'])) {
|
|
$primary = $this->primaryHost(self::KIND_DATABASE);
|
|
if ($primary === null) {
|
|
$blockers[] = 'No database primary is registered.';
|
|
} else {
|
|
$sourceConnection = $this->databaseConnection($primary, true);
|
|
try {
|
|
$source = $this->databaseServerStatus($sourceConnection);
|
|
$replica = $this->showReplicaStatus($connection);
|
|
$sourceEngine = self::databaseEngine($source);
|
|
$replicaEngine = self::databaseEngine($raw);
|
|
$raw['source_gtid_executed'] = self::databaseGtidPosition($source);
|
|
$raw['replica_status'] = $replica;
|
|
|
|
if ($sourceEngine !== $replicaEngine) {
|
|
$blockers[] = 'Database primary and replica must use the same engine family.';
|
|
}
|
|
$percent = $sourceEngine === 'mariadb'
|
|
? self::mariadbGtidCoveragePercent(
|
|
self::databaseGtidPosition($source),
|
|
(string)($replica['Gtid_IO_Pos'] ?? $raw['gtid_slave_pos'] ?? $raw['gtid_current_pos'] ?? '')
|
|
)
|
|
: self::mysqlGtidCoveragePercent(
|
|
(string)($source['gtid_executed'] ?? ''),
|
|
(string)($replica['Executed_Gtid_Set'] ?? $raw['gtid_executed'] ?? '')
|
|
);
|
|
$lagSeconds = isset($replica['Seconds_Behind_Source'])
|
|
? (int)$replica['Seconds_Behind_Source']
|
|
: (isset($replica['Seconds_Behind_Master']) ? (int)$replica['Seconds_Behind_Master'] : null);
|
|
|
|
$ioRunning = false;
|
|
$sqlRunning = false;
|
|
if ($replica === []) {
|
|
$blockers[] = 'Database replica status is not configured.';
|
|
} else {
|
|
$ioRunning = strtoupper((string)($replica['Replica_IO_Running'] ?? $replica['Slave_IO_Running'] ?? '')) === 'YES';
|
|
$sqlRunning = strtoupper((string)($replica['Replica_SQL_Running'] ?? $replica['Slave_SQL_Running'] ?? '')) === 'YES';
|
|
if (!$ioRunning || !$sqlRunning) {
|
|
$blockers[] = 'Database replication IO and SQL threads must both be running.';
|
|
}
|
|
if (!$ioRunning) {
|
|
$blockers[] = 'Database replication IO thread is not running.';
|
|
}
|
|
if (!$sqlRunning) {
|
|
$blockers[] = 'Database replication SQL thread is not running.';
|
|
}
|
|
if ($sourceEngine === 'mariadb' && isset($replica['Using_Gtid']) && strtoupper((string)$replica['Using_Gtid']) === 'NO') {
|
|
$blockers[] = 'MariaDB replication must use GTID mode.';
|
|
}
|
|
foreach (['Last_IO_Error', 'Last_SQL_Error', 'Last_Error'] as $errorKey) {
|
|
$error = trim((string)($replica[$errorKey] ?? ''));
|
|
if ($error !== '') {
|
|
$blockers[] = $error;
|
|
}
|
|
}
|
|
}
|
|
if (($blockers !== [] || !$ioRunning || !$sqlRunning) && $percent >= 100.0) {
|
|
$percent = 99.99;
|
|
}
|
|
} finally {
|
|
$sourceConnection->close();
|
|
}
|
|
}
|
|
}
|
|
} finally {
|
|
$connection->close();
|
|
}
|
|
} catch (Throwable $throwable) {
|
|
$reachable = false;
|
|
$blockers[] = $throwable->getMessage();
|
|
}
|
|
|
|
$blockers = array_values(array_unique(array_filter($blockers)));
|
|
$status = [
|
|
'status' => self::replicationHealthStatus(
|
|
$reachable,
|
|
(string)($host['role'] ?? ''),
|
|
(float)$percent,
|
|
$blockers,
|
|
empty($host['test_connectivity_only'])
|
|
),
|
|
'replication_percent' => round($percent, 2),
|
|
'lag_seconds' => $lagSeconds,
|
|
'blockers' => $blockers,
|
|
'raw' => $raw,
|
|
'checked_at' => date('c'),
|
|
];
|
|
|
|
if ($this->shouldRepairDatabaseReplicationAccess($host, $status)) {
|
|
$repair = $this->repairDatabaseReplicationAccess($host, $status);
|
|
if (($repair['ok'] ?? false) === true) {
|
|
$retested = $this->testDatabaseHost(array_merge($host, [
|
|
'skip_replication_access_repair' => true,
|
|
'skip_replication_thread_repair' => true,
|
|
]));
|
|
$retested['raw']['replication_access_repair'] = $repair;
|
|
return $retested;
|
|
}
|
|
|
|
if (!empty($repair['message'])) {
|
|
$status['blockers'][] = 'Database replication access repair failed: ' . $repair['message'];
|
|
$status['blockers'] = array_values(array_unique(array_filter($status['blockers'])));
|
|
$status['status'] = self::replicationHealthStatus(
|
|
$reachable,
|
|
(string)($host['role'] ?? ''),
|
|
(float)$status['replication_percent'],
|
|
$status['blockers']
|
|
);
|
|
}
|
|
$status['raw']['replication_access_repair'] = $repair;
|
|
}
|
|
|
|
if ($this->shouldRepairDatabaseReplicationThreads($host, $status)) {
|
|
$repair = $this->repairDatabaseReplicationThreads($host);
|
|
if (($repair['ok'] ?? false) === true) {
|
|
$retested = $this->testDatabaseHost(array_merge($host, [
|
|
'skip_replication_access_repair' => true,
|
|
'skip_replication_thread_repair' => true,
|
|
]));
|
|
$retested['raw']['replication_thread_repair'] = $repair;
|
|
return $retested;
|
|
}
|
|
|
|
if (!empty($repair['message'])) {
|
|
$status['blockers'][] = 'Database replication thread restart failed: ' . $repair['message'];
|
|
$status['blockers'] = array_values(array_unique(array_filter($status['blockers'])));
|
|
$status['status'] = self::replicationHealthStatus(
|
|
$reachable,
|
|
(string)($host['role'] ?? ''),
|
|
(float)$status['replication_percent'],
|
|
$status['blockers']
|
|
);
|
|
}
|
|
$status['raw']['replication_thread_repair'] = $repair;
|
|
}
|
|
|
|
return $status;
|
|
}
|
|
|
|
private function shouldRepairDatabaseReplicationAccess(array $host, array $status): bool
|
|
{
|
|
if (!empty($host['skip_replication_access_repair'])
|
|
|| !empty($host['test_connectivity_only'])
|
|
|| (string)($host['role'] ?? '') === 'primary') {
|
|
return false;
|
|
}
|
|
|
|
if ($this->databaseDeniedAccountHostsFromStatus($status) === []) {
|
|
return false;
|
|
}
|
|
|
|
$hostCredentials = $this->credentials($host);
|
|
$primary = $this->primaryHost(self::KIND_DATABASE);
|
|
$primaryCredentials = $primary !== null ? $this->credentials($primary) : [];
|
|
|
|
return (($hostCredentials['replication_username'] ?? '') !== '' && ($hostCredentials['replication_password'] ?? '') !== '')
|
|
|| (($primaryCredentials['replication_username'] ?? '') !== '' && ($primaryCredentials['replication_password'] ?? '') !== '');
|
|
}
|
|
|
|
private function repairDatabaseReplicationAccess(array $host, array $status): array
|
|
{
|
|
$deniedHosts = $this->databaseDeniedAccountHostsFromStatus($status);
|
|
if ($deniedHosts === []) {
|
|
return [
|
|
'ok' => false,
|
|
'skipped' => true,
|
|
'message' => 'No denied replication account host was detected.',
|
|
];
|
|
}
|
|
|
|
$primary = $this->primaryHost(self::KIND_DATABASE);
|
|
if ($primary === null) {
|
|
return [
|
|
'ok' => false,
|
|
'message' => 'No database primary is registered.',
|
|
];
|
|
}
|
|
|
|
$primaryCredentials = $this->credentials($primary);
|
|
$hostCredentials = $this->credentials($host);
|
|
$replicationUser = $hostCredentials['replication_username']
|
|
?: ($primaryCredentials['replication_username'] ?? '');
|
|
$replicationPassword = $hostCredentials['replication_password']
|
|
?: ($primaryCredentials['replication_password'] ?? '');
|
|
|
|
if ($replicationUser === '' || $replicationPassword === '') {
|
|
return [
|
|
'ok' => false,
|
|
'skipped' => true,
|
|
'denied_hosts' => $deniedHosts,
|
|
'message' => 'Replication credentials are not available for automatic grant repair.',
|
|
];
|
|
}
|
|
|
|
try {
|
|
$grantHosts = $this->databaseReplicationGrantHosts($host, $status);
|
|
$this->ensureDatabaseReplicationUser($primary, $replicationUser, $replicationPassword, $grantHosts);
|
|
|
|
$target = $this->databaseConnection($host, true);
|
|
try {
|
|
$this->refreshDatabaseReplicationConnection(
|
|
$target,
|
|
$this->databaseServerStatus($target),
|
|
$primary,
|
|
$replicationUser,
|
|
$replicationPassword
|
|
);
|
|
} finally {
|
|
$target->close();
|
|
}
|
|
|
|
return [
|
|
'ok' => true,
|
|
'denied_hosts' => $deniedHosts,
|
|
'grant_hosts' => $grantHosts,
|
|
];
|
|
} catch (Throwable $throwable) {
|
|
return [
|
|
'ok' => false,
|
|
'denied_hosts' => $deniedHosts,
|
|
'message' => $throwable->getMessage(),
|
|
];
|
|
}
|
|
}
|
|
|
|
private function shouldRepairDatabaseReplicationThreads(array $host, array $status): bool
|
|
{
|
|
if (!empty($host['skip_replication_thread_repair'])
|
|
|| !empty($host['test_connectivity_only'])
|
|
|| (string)($host['role'] ?? '') === 'primary') {
|
|
return false;
|
|
}
|
|
|
|
$replicaStatus = $status['raw']['replica_status'] ?? null;
|
|
return is_array($replicaStatus)
|
|
&& $replicaStatus !== []
|
|
&& self::databaseOnlyReplicationThreadBlockers($status['blockers'] ?? []);
|
|
}
|
|
|
|
private static function databaseOnlyReplicationThreadBlockers(array $blockers): bool
|
|
{
|
|
$blockers = array_values(array_filter(array_map(
|
|
static fn(mixed $blocker): string => trim((string)$blocker),
|
|
$blockers
|
|
)));
|
|
if ($blockers === []) {
|
|
return false;
|
|
}
|
|
|
|
$allowed = [
|
|
'Database replication IO and SQL threads must both be running.',
|
|
'Database replication IO thread is not running.',
|
|
'Database replication SQL thread is not running.',
|
|
];
|
|
|
|
return array_values(array_diff($blockers, $allowed)) === [];
|
|
}
|
|
|
|
private function repairDatabaseReplicationThreads(array $host): array
|
|
{
|
|
try {
|
|
$target = $this->databaseConnection($host, true);
|
|
try {
|
|
$this->restartDatabaseReplicationThreads($target, $this->databaseServerStatus($target));
|
|
} finally {
|
|
$target->close();
|
|
}
|
|
|
|
return ['ok' => true];
|
|
} catch (Throwable $throwable) {
|
|
return [
|
|
'ok' => false,
|
|
'message' => $throwable->getMessage(),
|
|
];
|
|
}
|
|
}
|
|
|
|
private function refreshDatabaseReplicationConnection(
|
|
mysqli $target,
|
|
array $serverStatus,
|
|
array $primary,
|
|
string $replicationUser,
|
|
string $replicationPassword
|
|
): void {
|
|
$isMariaDb = self::databaseEngine($serverStatus) === 'mariadb';
|
|
if ($isMariaDb) {
|
|
try {
|
|
$this->mysqliExec($target, 'STOP SLAVE');
|
|
} catch (Throwable) {
|
|
}
|
|
$sql = sprintf(
|
|
"CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, MASTER_USER = '%s', MASTER_PASSWORD = '%s', MASTER_USE_GTID = slave_pos",
|
|
$target->real_escape_string((string)$primary['host']),
|
|
(int)$primary['port'],
|
|
$target->real_escape_string($replicationUser),
|
|
$target->real_escape_string($replicationPassword)
|
|
);
|
|
$this->mysqliExec($target, $sql);
|
|
} else {
|
|
try {
|
|
$this->mysqliExec($target, 'STOP REPLICA');
|
|
} catch (Throwable) {
|
|
}
|
|
$sql = sprintf(
|
|
"CHANGE REPLICATION SOURCE TO SOURCE_HOST = '%s', SOURCE_PORT = %d, SOURCE_USER = '%s', SOURCE_PASSWORD = '%s', SOURCE_AUTO_POSITION = 1",
|
|
$target->real_escape_string((string)$primary['host']),
|
|
(int)$primary['port'],
|
|
$target->real_escape_string($replicationUser),
|
|
$target->real_escape_string($replicationPassword)
|
|
);
|
|
$this->mysqliExec($target, $sql);
|
|
}
|
|
|
|
$this->restartDatabaseReplicationThreads($target, $serverStatus);
|
|
}
|
|
|
|
private function restartDatabaseReplicationThreads(mysqli $target, array $serverStatus): void
|
|
{
|
|
$isMariaDb = self::databaseEngine($serverStatus) === 'mariadb';
|
|
$startStatements = $isMariaDb
|
|
? ['START SLAVE', 'START SLAVE IO_THREAD', 'START SLAVE SQL_THREAD']
|
|
: ['START REPLICA', 'START REPLICA IO_THREAD', 'START REPLICA SQL_THREAD'];
|
|
|
|
$lastError = null;
|
|
$startedAnyThread = false;
|
|
foreach ($startStatements as $index => $statement) {
|
|
try {
|
|
$this->mysqliExec($target, $statement);
|
|
if ($index === 0) {
|
|
return;
|
|
}
|
|
$startedAnyThread = true;
|
|
} catch (Throwable $throwable) {
|
|
$lastError = $throwable;
|
|
}
|
|
}
|
|
|
|
if ($startedAnyThread) {
|
|
return;
|
|
}
|
|
|
|
if ($lastError !== null) {
|
|
throw $lastError;
|
|
}
|
|
}
|
|
|
|
private function testRedisHost(array $host): array
|
|
{
|
|
$blockers = [];
|
|
$raw = [];
|
|
$percent = (float)(($host['role'] ?? '') === 'primary' ? 100 : 0);
|
|
$reachable = true;
|
|
|
|
try {
|
|
$client = $this->redisClient($host);
|
|
$ping = (string)$client->ping();
|
|
if (stripos($ping, 'PONG') === false && stripos($ping, 'OK') === false) {
|
|
$blockers[] = 'Redis PING did not return PONG.';
|
|
}
|
|
|
|
$role = $client->executeRaw(['ROLE']);
|
|
$info = $this->redisInfo($client);
|
|
$raw = [
|
|
'role' => $role,
|
|
'replication' => $info,
|
|
];
|
|
|
|
try {
|
|
$client->executeRaw(['CONFIG', 'GET', 'appendonly']);
|
|
} catch (Throwable $throwable) {
|
|
$blockers[] = 'Redis ACL must allow CONFIG GET/SET/REWRITE for durable replication changes.';
|
|
}
|
|
|
|
if (($host['role'] ?? '') !== 'primary' && empty($host['test_connectivity_only'])) {
|
|
$primary = $this->primaryHost(self::KIND_REDIS);
|
|
if ($primary === null) {
|
|
$blockers[] = 'No Redis primary is registered.';
|
|
} else {
|
|
$primaryClient = $this->redisClient($primary);
|
|
$primaryInfo = $this->redisInfo($primaryClient);
|
|
$percent = self::redisReplicationPercentFromInfo($primaryInfo, $info);
|
|
$raw['primary_replication'] = $primaryInfo;
|
|
|
|
if (!in_array(strtolower((string)($info['role'] ?? '')), ['slave', 'replica'], true)) {
|
|
$blockers[] = 'Redis host is not currently a replica.';
|
|
}
|
|
if (strtolower((string)($info['master_link_status'] ?? '')) !== 'up') {
|
|
$blockers[] = 'Redis replica link to primary is not up.';
|
|
}
|
|
if ($blockers !== [] && $percent >= 100.0) {
|
|
$percent = 99.99;
|
|
}
|
|
}
|
|
}
|
|
} catch (Throwable $throwable) {
|
|
$reachable = false;
|
|
$blockers[] = $throwable->getMessage();
|
|
}
|
|
|
|
$blockers = array_values(array_unique(array_filter($blockers)));
|
|
return [
|
|
'status' => self::replicationHealthStatus(
|
|
$reachable,
|
|
(string)($host['role'] ?? ''),
|
|
(float)$percent,
|
|
$blockers,
|
|
empty($host['test_connectivity_only'])
|
|
),
|
|
'replication_percent' => round($percent, 2),
|
|
'lag_seconds' => null,
|
|
'blockers' => $blockers,
|
|
'raw' => $raw,
|
|
'checked_at' => date('c'),
|
|
];
|
|
}
|
|
|
|
private function testMinioHost(array $host): array
|
|
{
|
|
$blockers = [];
|
|
$raw = [];
|
|
$percent = (float)(($host['role'] ?? '') === 'primary' ? 100 : 0);
|
|
$reachable = true;
|
|
$connectivityOnly = !empty($host['test_connectivity_only']);
|
|
$isPrimary = (string)($host['role'] ?? '') === 'primary';
|
|
$forceStorageScan = !empty($host['force_storage_scan']);
|
|
$measureStorage = !$connectivityOnly
|
|
&& empty($host['skip_storage_scan'])
|
|
&& $forceStorageScan;
|
|
$options = $this->decodeOptions($host);
|
|
$buckets = self::normalizeMinioBuckets($options['buckets'] ?? self::MINIO_DEFAULT_BUCKETS);
|
|
|
|
try {
|
|
$client = $this->minioS3Client($host);
|
|
$client->listBuckets();
|
|
|
|
if ($isPrimary) {
|
|
$sourceStats = $this->minioBucketStats($client, $buckets, true, 'MinIO source bucket', $measureStorage);
|
|
$blockers = array_merge($blockers, $sourceStats['blockers']);
|
|
$raw['buckets'] = $sourceStats['buckets'];
|
|
$raw['storage'] = [
|
|
'source_bytes' => $sourceStats['bytes'],
|
|
'source_objects' => $sourceStats['objects'],
|
|
'measured' => $measureStorage,
|
|
];
|
|
} else {
|
|
$retentionDaysByBucket = self::minioReplicaRetentionDaysByBucket($buckets);
|
|
$targetStats = $this->minioBucketStats(
|
|
$client,
|
|
$buckets,
|
|
!$connectivityOnly,
|
|
'MinIO target bucket',
|
|
$measureStorage,
|
|
$retentionDaysByBucket
|
|
);
|
|
$blockers = array_merge($blockers, $targetStats['blockers']);
|
|
$raw['target_buckets'] = $targetStats['buckets'];
|
|
$raw['storage'] = [
|
|
'target_bytes' => $targetStats['bytes'],
|
|
'target_objects' => $targetStats['objects'],
|
|
'target_expired_bytes' => $targetStats['expired_bytes'] ?? 0,
|
|
'target_expired_objects' => $targetStats['expired_objects'] ?? 0,
|
|
'measured' => $measureStorage,
|
|
];
|
|
|
|
if (!$connectivityOnly) {
|
|
$primary = $this->primaryHost(self::KIND_MINIO);
|
|
if ($primary === null) {
|
|
$blockers[] = 'No MinIO primary is registered.';
|
|
} else {
|
|
$sourceStats = $this->minioBucketStats(
|
|
$this->minioS3Client($primary),
|
|
$buckets,
|
|
true,
|
|
'MinIO source bucket',
|
|
$measureStorage,
|
|
$retentionDaysByBucket
|
|
);
|
|
$blockers = array_merge($blockers, $sourceStats['blockers']);
|
|
$blockers = array_merge($blockers, self::minioBackupRetentionBlockers($targetStats));
|
|
$headroom = (float)($options['space_headroom_percent'] ?? self::MINIO_SPACE_HEADROOM_PERCENT);
|
|
$availableBytes = $measureStorage ? $this->minioTargetFreeBytes($host) : null;
|
|
$requiredBytes = $measureStorage ? self::minioRequiredFreeBytes((int)$sourceStats['bytes'], $headroom) : null;
|
|
$spaceBlockers = $requiredBytes === null ? [] : self::minioSpaceBlockers($availableBytes, $requiredBytes);
|
|
$blockers = array_merge($blockers, $spaceBlockers);
|
|
$percent = $measureStorage
|
|
? self::minioByteReplicationPercent((int)$sourceStats['bytes'], (int)$targetStats['bytes'])
|
|
: self::minioIncompleteProgress(self::lastStatusReplicationPercent($host, 5.0));
|
|
$replicationConfigured = $this->minioReplicationConfigured($primary, $buckets);
|
|
$progressStatus = null;
|
|
if ($replicationConfigured) {
|
|
$progressStatus = $this->minioReplicationProgressStatus($primary, $host, $buckets);
|
|
if ($progressStatus !== null) {
|
|
$percent = round((float)$progressStatus['replication_percent'], 2);
|
|
$raw['progress_source'] = 'minio_replicate_status';
|
|
$raw['replication_status'] = $progressStatus;
|
|
} else {
|
|
$raw['progress_source'] = 'minio_replicate_status_unavailable';
|
|
}
|
|
}
|
|
if (!$replicationConfigured) {
|
|
$blockers[] = 'MinIO bucket replication is not configured.';
|
|
} elseif (in_array(self::MINIO_BACKUP_BUCKET, $buckets, true)
|
|
&& !$this->minioBackupReplicaRetentionConfigured($host, self::MINIO_BACKUP_BUCKET)) {
|
|
$blockers[] = 'MinIO backup replica retention is not configured for the backups bucket.';
|
|
} elseif (($progressStatus !== null || $measureStorage) && $percent < 100.0) {
|
|
$blockers[] = self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER;
|
|
} elseif ($progressStatus === null && !$measureStorage) {
|
|
$blockers[] = self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER;
|
|
}
|
|
$raw['source_buckets'] = $sourceStats['buckets'];
|
|
$raw['storage'] = array_merge($raw['storage'], [
|
|
'source_bytes' => $sourceStats['bytes'],
|
|
'source_objects' => $sourceStats['objects'],
|
|
'source_expired_bytes' => $sourceStats['expired_bytes'] ?? 0,
|
|
'source_expired_objects' => $sourceStats['expired_objects'] ?? 0,
|
|
'required_free_bytes' => $requiredBytes,
|
|
'available_free_bytes' => $availableBytes,
|
|
'space_headroom_percent' => $headroom,
|
|
'space_ok' => $measureStorage ? ($availableBytes === null ? null : $spaceBlockers === []) : null,
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
} catch (Throwable $throwable) {
|
|
$reachable = false;
|
|
$blockers[] = $throwable->getMessage();
|
|
}
|
|
|
|
$blockers = array_values(array_unique(array_filter($blockers)));
|
|
return [
|
|
'status' => self::replicationHealthStatus(
|
|
$reachable,
|
|
(string)($host['role'] ?? ''),
|
|
(float)$percent,
|
|
$blockers,
|
|
!$connectivityOnly
|
|
),
|
|
'replication_percent' => round($percent, 2),
|
|
'lag_seconds' => null,
|
|
'blockers' => $blockers,
|
|
'raw' => $raw,
|
|
'checked_at' => date('c'),
|
|
];
|
|
}
|
|
|
|
private function minioProgressScanHost(array $host): array
|
|
{
|
|
if (!$this->minioCanReuseRecentMeasuredStatus($host)) {
|
|
return $host;
|
|
}
|
|
|
|
return array_merge($host, ['skip_storage_scan' => true]);
|
|
}
|
|
|
|
private function minioCanReuseRecentMeasuredStatus(array $host): bool
|
|
{
|
|
$lastStatus = self::jsonDecode($host['last_status_json'] ?? null);
|
|
if (!is_array($lastStatus)) {
|
|
return false;
|
|
}
|
|
|
|
$storage = $lastStatus['raw']['storage'] ?? null;
|
|
if (!is_array($storage) || empty($storage['measured'])) {
|
|
return false;
|
|
}
|
|
|
|
$checkedAt = strtotime((string)($lastStatus['checked_at'] ?? $host['last_checked_at'] ?? ''));
|
|
if ($checkedAt === false) {
|
|
return false;
|
|
}
|
|
|
|
return (time() - $checkedAt) < self::MINIO_PROGRESS_SCAN_INTERVAL_SECONDS;
|
|
}
|
|
|
|
private static function minioOnlyProgressBlockers(array $blockers): bool
|
|
{
|
|
$blockers = array_values(array_filter(array_map(
|
|
static fn(mixed $blocker): string => trim((string)$blocker),
|
|
$blockers
|
|
)));
|
|
if ($blockers === []) {
|
|
return true;
|
|
}
|
|
|
|
return array_values(array_diff($blockers, [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER])) === [];
|
|
}
|
|
|
|
private static function minioProvisionProgressMessage(array $status): string
|
|
{
|
|
$storage = is_array($status['raw']['storage'] ?? null) ? $status['raw']['storage'] : [];
|
|
if (!empty($storage['measured'])) {
|
|
$targetBytes = (int)($storage['target_bytes'] ?? 0);
|
|
$sourceBytes = (int)($storage['source_bytes'] ?? 0);
|
|
if ($sourceBytes > 0) {
|
|
return 'MinIO replica is syncing. Copied ' . $targetBytes . ' of ' . $sourceBytes . ' bytes.';
|
|
}
|
|
}
|
|
|
|
if ((string)($status['raw']['progress_source'] ?? '') === 'minio_replicate_status') {
|
|
$percent = round((float)($status['replication_percent'] ?? 0), 2);
|
|
return 'MinIO replica is syncing. Replication status reports ' . $percent . '% complete.';
|
|
}
|
|
|
|
return 'MinIO replica is syncing in the background. Waiting for the next progress sample.';
|
|
}
|
|
|
|
private function minioProvisionStatus(array $primary, array $host): array
|
|
{
|
|
$status = $this->testMinioHost(array_merge($host, [
|
|
'test_connectivity_only' => true,
|
|
'skip_storage_scan' => true,
|
|
]));
|
|
if ($status['blockers'] !== []) {
|
|
return $status;
|
|
}
|
|
|
|
$primaryOptions = $this->decodeOptions($primary);
|
|
$targetOptions = $this->decodeOptions($host);
|
|
$buckets = self::normalizeMinioBuckets($targetOptions['buckets'] ?? $primaryOptions['buckets'] ?? self::MINIO_DEFAULT_BUCKETS);
|
|
$progress = $this->minioReplicationProgressStatus($primary, $host, $buckets);
|
|
|
|
if ($progress === null) {
|
|
$percent = self::minioIncompleteProgress(self::lastStatusReplicationPercent($host, 5.0));
|
|
$status['replication_percent'] = $percent;
|
|
$status['blockers'] = array_values(array_unique(array_merge(
|
|
$status['blockers'],
|
|
[self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER]
|
|
)));
|
|
$status['raw']['progress_source'] = 'minio_replicate_status_unavailable';
|
|
$status['raw']['replication_status'] = [
|
|
'available' => false,
|
|
'message' => 'MinIO replication status did not report progress yet.',
|
|
];
|
|
$status['status'] = self::replicationHealthStatus(
|
|
true,
|
|
(string)($host['role'] ?? ''),
|
|
$percent,
|
|
$status['blockers']
|
|
);
|
|
|
|
return $status;
|
|
}
|
|
|
|
$status['replication_percent'] = round((float)$progress['replication_percent'], 2);
|
|
$status['blockers'] = array_values(array_unique(array_merge(
|
|
$status['blockers'],
|
|
$progress['blockers'] ?? []
|
|
)));
|
|
$status['raw']['progress_source'] = 'minio_replicate_status';
|
|
$status['raw']['replication_status'] = $progress;
|
|
$status = self::normalizeMinioCaughtUpStatus($status, (string)($host['role'] ?? ''));
|
|
$status['status'] = self::replicationHealthStatus(
|
|
true,
|
|
(string)($host['role'] ?? ''),
|
|
(float)$status['replication_percent'],
|
|
$status['blockers']
|
|
);
|
|
|
|
return $status;
|
|
}
|
|
|
|
private static function lastStatusReplicationPercent(array $host, float $default): float
|
|
{
|
|
$lastStatus = self::jsonDecode($host['last_status_json'] ?? null);
|
|
if (is_array($lastStatus) && is_numeric($lastStatus['replication_percent'] ?? null)) {
|
|
return round((float)$lastStatus['replication_percent'], 2);
|
|
}
|
|
|
|
return $default;
|
|
}
|
|
|
|
private static function normalizeMinioCaughtUpStatus(array $status, string $role): array
|
|
{
|
|
if ($role === 'primary' || round((float)($status['replication_percent'] ?? 0), 2) < 100.0) {
|
|
return $status;
|
|
}
|
|
|
|
$blockers = array_values(array_filter(array_map(
|
|
static fn(mixed $blocker): string => trim((string)$blocker),
|
|
$status['blockers'] ?? []
|
|
)));
|
|
if ($blockers === []) {
|
|
return $status;
|
|
}
|
|
|
|
$status['blockers'] = array_values(array_diff($blockers, [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER]));
|
|
return $status;
|
|
}
|
|
|
|
public static function databasePrerequisiteBlockers(array $status): array
|
|
{
|
|
if (self::databaseEngine($status) === 'mariadb') {
|
|
return self::mariaDbPrerequisiteBlockers($status);
|
|
}
|
|
|
|
$blockers = [];
|
|
if (strtoupper((string)($status['gtid_mode'] ?? '')) !== 'ON') {
|
|
$blockers[] = isset($status['gtid_mode'])
|
|
? 'MySQL GTID mode must be ON.'
|
|
: 'MySQL GTID mode is unavailable. Managed replication requires Oracle MySQL 8.x with GTID enabled.';
|
|
}
|
|
if (!self::mysqlBooleanEnabled($status['log_bin'] ?? null)) {
|
|
$blockers[] = isset($status['log_bin'])
|
|
? 'MySQL binary logging must be enabled.'
|
|
: 'MySQL binary logging status is unavailable.';
|
|
}
|
|
if ((int)($status['server_id'] ?? 0) <= 0) {
|
|
$blockers[] = 'MySQL server_id must be configured.';
|
|
}
|
|
if (trim((string)($status['server_uuid'] ?? '')) === '') {
|
|
$blockers[] = 'MySQL server_uuid must be available. Managed replication requires Oracle MySQL 8.x.';
|
|
}
|
|
$serverVersion = (string)($status['server_version'] ?? '');
|
|
if (!str_starts_with($serverVersion, '8.') || stripos($serverVersion, 'mariadb') !== false) {
|
|
$blockers[] = $serverVersion !== ''
|
|
? 'Oracle MySQL 8.x is required for managed replication. Current server reports ' . $serverVersion . '.'
|
|
: 'Oracle MySQL 8.x is required for managed replication.';
|
|
}
|
|
|
|
return $blockers;
|
|
}
|
|
|
|
public static function missingDatabaseTables(array $sourceTables, array $replicaTables): array
|
|
{
|
|
$source = array_values(array_unique(array_filter(array_map(
|
|
static fn(mixed $table): string => trim((string)$table),
|
|
$sourceTables
|
|
))));
|
|
$replicaLookup = array_flip(array_values(array_unique(array_filter(array_map(
|
|
static fn(mixed $table): string => trim((string)$table),
|
|
$replicaTables
|
|
)))));
|
|
|
|
return array_values(array_filter(
|
|
$source,
|
|
static fn(string $table): bool => !isset($replicaLookup[$table])
|
|
));
|
|
}
|
|
|
|
private static function mariaDbPrerequisiteBlockers(array $status): array
|
|
{
|
|
$blockers = [];
|
|
if (!self::mysqlBooleanEnabled($status['log_bin'] ?? null)) {
|
|
$blockers[] = isset($status['log_bin'])
|
|
? 'MariaDB binary logging must be enabled.'
|
|
: 'MariaDB binary logging status is unavailable.';
|
|
}
|
|
if ((int)($status['server_id'] ?? 0) <= 0) {
|
|
$blockers[] = 'MariaDB server_id must be configured.';
|
|
}
|
|
if (!self::mariaDbGtidPositionAvailable($status)) {
|
|
$blockers[] = 'MariaDB GTID position must be available.';
|
|
}
|
|
|
|
$serverVersion = (string)($status['server_version'] ?? '');
|
|
if (!self::mariaDbVersionSupported($serverVersion)) {
|
|
$blockers[] = $serverVersion !== ''
|
|
? 'MariaDB 10.6 or newer is required for managed replication. Current server reports ' . $serverVersion . '.'
|
|
: 'MariaDB 10.6 or newer is required for managed replication.';
|
|
}
|
|
|
|
return $blockers;
|
|
}
|
|
|
|
private static function databaseEngine(array $status): string
|
|
{
|
|
return stripos((string)($status['server_version'] ?? ''), 'mariadb') !== false ? 'mariadb' : 'mysql';
|
|
}
|
|
|
|
private static function databaseEngineKnown(array $status): bool
|
|
{
|
|
return trim((string)($status['server_version'] ?? '')) !== '';
|
|
}
|
|
|
|
private static function databaseGtidPosition(array $status): string
|
|
{
|
|
if (self::databaseEngine($status) === 'mariadb') {
|
|
return trim((string)($status['gtid_binlog_pos'] ?? $status['gtid_current_pos'] ?? $status['gtid_slave_pos'] ?? ''));
|
|
}
|
|
|
|
return trim((string)($status['gtid_executed'] ?? ''));
|
|
}
|
|
|
|
private static function mariaDbGtidPositionAvailable(array $status): bool
|
|
{
|
|
foreach (['gtid_binlog_pos', 'gtid_current_pos', 'gtid_slave_pos'] as $key) {
|
|
if (array_key_exists($key, $status) && $status[$key] !== null) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private static function mariaDbVersionSupported(string $serverVersion): bool
|
|
{
|
|
if (!preg_match('/(\d+)\.(\d+)/', $serverVersion, $matches)) {
|
|
return false;
|
|
}
|
|
|
|
$major = (int)$matches[1];
|
|
$minor = (int)$matches[2];
|
|
return $major > 10 || ($major === 10 && $minor >= 6);
|
|
}
|
|
|
|
private static function mysqlBooleanEnabled(mixed $value): bool
|
|
{
|
|
$normalized = strtoupper(trim((string)$value));
|
|
return in_array($normalized, ['1', 'ON', 'YES', 'TRUE'], true);
|
|
}
|
|
|
|
private function databaseServerStatus(mysqli $connection): array
|
|
{
|
|
$row = $this->mysqliSelectOne($connection, "SELECT VERSION() AS server_version");
|
|
$variables = $connection->query(
|
|
"SHOW GLOBAL VARIABLES WHERE Variable_name IN (
|
|
'gtid_mode',
|
|
'log_bin',
|
|
'server_id',
|
|
'server_uuid',
|
|
'read_only',
|
|
'super_read_only',
|
|
'gtid_executed',
|
|
'gtid_binlog_pos',
|
|
'gtid_current_pos',
|
|
'gtid_slave_pos',
|
|
'gtid_strict_mode'
|
|
)"
|
|
);
|
|
if ($variables !== false) {
|
|
while ($variable = $variables->fetch_assoc()) {
|
|
$name = strtolower((string)($variable['Variable_name'] ?? ''));
|
|
if ($name !== '') {
|
|
$row[$name] = $variable['Value'] ?? null;
|
|
}
|
|
}
|
|
}
|
|
|
|
$plugin = $this->mysqliSelectOne(
|
|
$connection,
|
|
"SELECT PLUGIN_STATUS AS plugin_status FROM information_schema.PLUGINS WHERE PLUGIN_NAME = 'clone' LIMIT 1"
|
|
);
|
|
$row['clone_plugin_active'] = strtoupper((string)($plugin['plugin_status'] ?? '')) === 'ACTIVE';
|
|
|
|
return $row;
|
|
}
|
|
|
|
private function showReplicaStatus(mysqli $connection): array
|
|
{
|
|
try {
|
|
$status = $this->mysqliSelectOne($connection, 'SHOW REPLICA STATUS');
|
|
if ($status !== []) {
|
|
return $status;
|
|
}
|
|
} catch (Throwable) {
|
|
}
|
|
|
|
return $this->mysqliSelectOne($connection, 'SHOW SLAVE STATUS');
|
|
}
|
|
|
|
private function databaseConnection(array $host, bool $admin = false, bool $connectWithoutDatabase = false): mysqli
|
|
{
|
|
$credentials = $this->credentials($host);
|
|
$username = $admin && $credentials['admin_username'] !== ''
|
|
? $credentials['admin_username']
|
|
: $credentials['username'];
|
|
$password = $admin && $credentials['admin_password'] !== ''
|
|
? $credentials['admin_password']
|
|
: $credentials['password'];
|
|
|
|
mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
|
|
$connection = new mysqli(
|
|
(string)$host['host'],
|
|
$username,
|
|
$password,
|
|
$connectWithoutDatabase ? '' : (string)($host['database_name'] ?? ''),
|
|
(int)$host['port']
|
|
);
|
|
$connection->set_charset('utf8mb4');
|
|
return $connection;
|
|
}
|
|
|
|
private function databaseReplicaSeedBlockers(?array $primary, array $host, mysqli $target): array
|
|
{
|
|
if ($primary === null) {
|
|
return ['No database primary is registered.'];
|
|
}
|
|
|
|
$primaryDatabase = trim((string)($primary['database_name'] ?? ''));
|
|
$targetDatabase = trim((string)($host['database_name'] ?? ''));
|
|
if ($primaryDatabase === '' || $targetDatabase === '') {
|
|
return [];
|
|
}
|
|
|
|
$primaryConnection = $this->databaseConnection($primary, true);
|
|
try {
|
|
$primaryTables = $this->databaseTableNames($primaryConnection, $primaryDatabase);
|
|
$targetTables = $this->databaseTableNames($target, $targetDatabase);
|
|
} finally {
|
|
$primaryConnection->close();
|
|
}
|
|
|
|
if ($primaryTables === []) {
|
|
return [];
|
|
}
|
|
|
|
$missingTables = self::missingDatabaseTables($primaryTables, $targetTables);
|
|
if ($missingTables === []) {
|
|
$schemaOnlyTablesWithRows = $this->databaseSchemaOnlyTablesWithRows($target, $targetDatabase);
|
|
if ($schemaOnlyTablesWithRows === []) {
|
|
return [];
|
|
}
|
|
|
|
return [self::schemaOnlyTablesContainRowsBlocker($targetDatabase, $schemaOnlyTablesWithRows)];
|
|
}
|
|
|
|
return [self::missingDatabaseTablesBlocker($targetDatabase, $missingTables)];
|
|
}
|
|
|
|
private function advanceMariaDbReplicaSeed(int $operationId, array $primary, array $host, mysqli $target): array
|
|
{
|
|
$owner = 'replication-seed:' . $operationId;
|
|
$context = $this->operationContext($operationId);
|
|
$freezeState = application_write_freeze::state();
|
|
if (($context['phase'] ?? '') !== '' && ($freezeState['owner'] ?? null) !== $owner) {
|
|
$context = [];
|
|
}
|
|
if (self::mariaDbSeedContextRequiresFilterReset($context)) {
|
|
$context = [];
|
|
}
|
|
application_write_freeze::freeze('MariaDB replica seed is copying data.', $owner, self::MARIADB_SEED_FREEZE_TTL_SECONDS);
|
|
|
|
$source = $this->databaseConnection($primary, true);
|
|
$deadline = microtime(true) + self::MARIADB_SEED_STEP_SECONDS;
|
|
|
|
try {
|
|
$this->mysqliExec($target, 'SET SESSION FOREIGN_KEY_CHECKS = 0');
|
|
$this->mysqliExec($target, 'SET SESSION UNIQUE_CHECKS = 0');
|
|
|
|
if (($context['phase'] ?? '') === '') {
|
|
$context = $this->initializeMariaDbSeedContext($source, $target, $primary, $host);
|
|
}
|
|
|
|
while (microtime(true) < $deadline && ($context['phase'] ?? '') !== 'complete') {
|
|
if (($context['phase'] ?? '') === 'schema') {
|
|
$context = $this->advanceMariaDbSeedSchema($source, $target, $context);
|
|
continue;
|
|
}
|
|
|
|
if (($context['phase'] ?? '') === 'copy') {
|
|
$context = $this->advanceMariaDbSeedRows($source, $target, $context);
|
|
continue;
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
$progress = self::mariaDbSeedProgress($context);
|
|
$message = self::mariaDbSeedMessage($context);
|
|
$this->updateOperationProgress($operationId, $progress, $message, $context);
|
|
|
|
if (($context['phase'] ?? '') === 'complete') {
|
|
application_write_freeze::unfreeze($owner);
|
|
return [
|
|
'running' => false,
|
|
'message' => 'MariaDB replica seed completed.',
|
|
'status' => [
|
|
'status' => 'provisioning',
|
|
'replication_percent' => 90.0,
|
|
'lag_seconds' => null,
|
|
'blockers' => [],
|
|
'raw' => ['seed' => $context],
|
|
'checked_at' => date('c'),
|
|
],
|
|
];
|
|
}
|
|
|
|
return [
|
|
'running' => true,
|
|
'message' => $message,
|
|
'status' => [
|
|
'status' => 'provisioning',
|
|
'replication_percent' => $progress,
|
|
'lag_seconds' => null,
|
|
'blockers' => ['MariaDB replica seed is running.'],
|
|
'raw' => ['seed' => $context],
|
|
'checked_at' => date('c'),
|
|
],
|
|
];
|
|
} catch (Throwable $throwable) {
|
|
application_write_freeze::unfreeze($owner);
|
|
throw $throwable;
|
|
} finally {
|
|
$source->close();
|
|
}
|
|
}
|
|
|
|
private function initializeMariaDbSeedContext(mysqli $source, mysqli $target, array $primary, array $host): array
|
|
{
|
|
$primaryDatabase = trim((string)($primary['database_name'] ?? ''));
|
|
$targetDatabase = trim((string)($host['database_name'] ?? ''));
|
|
if ($primaryDatabase === '' || $targetDatabase === '') {
|
|
throw new RuntimeException('Database names are required before a MariaDB replica can be seeded.');
|
|
}
|
|
|
|
$sourceGtid = $this->mariaDbCurrentGtid($source);
|
|
$this->prepareMariaDbReplicaTarget($target, $source, $primaryDatabase, $targetDatabase);
|
|
|
|
$tables = [];
|
|
foreach ($this->databaseTableNames($source, $primaryDatabase) as $tableName) {
|
|
$skipData = self::databaseReplicaSeedSkipsTableData($tableName);
|
|
$tables[] = [
|
|
'name' => $tableName,
|
|
'rows' => $skipData ? 0 : $this->estimatedDatabaseTableRows($source, $primaryDatabase, $tableName),
|
|
'copied' => 0,
|
|
'schema_created' => false,
|
|
'skip_data' => $skipData,
|
|
'skip_reason' => $skipData ? 'excluded from managed replication' : '',
|
|
];
|
|
}
|
|
|
|
return [
|
|
'phase' => 'schema',
|
|
'source_database' => $primaryDatabase,
|
|
'target_database' => $targetDatabase,
|
|
'source_gtid' => $sourceGtid,
|
|
'schema_index' => 0,
|
|
'copy_index' => 0,
|
|
'tables' => $tables,
|
|
'started_at' => date('c'),
|
|
'updated_at' => date('c'),
|
|
];
|
|
}
|
|
|
|
private function advanceMariaDbSeedSchema(mysqli $source, mysqli $target, array $context): array
|
|
{
|
|
$tables = $context['tables'] ?? [];
|
|
$index = (int)($context['schema_index'] ?? 0);
|
|
if (!isset($tables[$index])) {
|
|
$context['phase'] = 'copy';
|
|
$context['copy_index'] = 0;
|
|
$context['updated_at'] = date('c');
|
|
return $context;
|
|
}
|
|
|
|
$table = $tables[$index];
|
|
$this->createMariaDbReplicaTable(
|
|
$source,
|
|
$target,
|
|
(string)$context['source_database'],
|
|
(string)$context['target_database'],
|
|
(string)$table['name']
|
|
);
|
|
|
|
$context['tables'][$index]['schema_created'] = true;
|
|
$context['schema_index'] = $index + 1;
|
|
$context['updated_at'] = date('c');
|
|
return $context;
|
|
}
|
|
|
|
private function advanceMariaDbSeedRows(mysqli $source, mysqli $target, array $context): array
|
|
{
|
|
$tables = $context['tables'] ?? [];
|
|
$index = (int)($context['copy_index'] ?? 0);
|
|
if (!isset($tables[$index])) {
|
|
if (trim((string)($context['source_gtid'] ?? '')) !== '') {
|
|
$this->mysqliExec($target, 'SET GLOBAL gtid_slave_pos = ' . self::sqlString($target, (string)$context['source_gtid']));
|
|
}
|
|
$this->mysqliExec($target, 'SET SESSION FOREIGN_KEY_CHECKS = 1');
|
|
$this->mysqliExec($target, 'SET SESSION UNIQUE_CHECKS = 1');
|
|
$context['phase'] = 'complete';
|
|
$context['updated_at'] = date('c');
|
|
$context['completed_at'] = date('c');
|
|
return $context;
|
|
}
|
|
|
|
$tableName = (string)($tables[$index]['name'] ?? '');
|
|
$copied = (int)($tables[$index]['copied'] ?? 0);
|
|
if (!empty($tables[$index]['skip_data'])) {
|
|
$context['tables'][$index]['copied'] = (int)($tables[$index]['rows'] ?? 0);
|
|
$context['copy_index'] = $index + 1;
|
|
$context['updated_at'] = date('c');
|
|
return $context;
|
|
}
|
|
|
|
$copiedNow = $this->copyMariaDbReplicaTableRowsChunk(
|
|
$source,
|
|
$target,
|
|
(string)$context['source_database'],
|
|
(string)$context['target_database'],
|
|
$tableName,
|
|
$copied,
|
|
self::MARIADB_SEED_BATCH_ROWS
|
|
);
|
|
|
|
$context['tables'][$index]['copied'] = $copied + $copiedNow;
|
|
if ($copiedNow < self::MARIADB_SEED_BATCH_ROWS) {
|
|
$context['copy_index'] = $index + 1;
|
|
}
|
|
$context['updated_at'] = date('c');
|
|
return $context;
|
|
}
|
|
|
|
private static function mariaDbSeedProgress(array $context): float
|
|
{
|
|
$tables = is_array($context['tables'] ?? null) ? $context['tables'] : [];
|
|
if (($context['phase'] ?? '') === 'complete') {
|
|
return 90.0;
|
|
}
|
|
if ($tables === []) {
|
|
return 5.0;
|
|
}
|
|
|
|
$schemaCount = count($tables);
|
|
$schemaDone = min($schemaCount, (int)($context['schema_index'] ?? 0));
|
|
$schemaProgress = $schemaCount > 0 ? ($schemaDone / $schemaCount) * 20.0 : 20.0;
|
|
|
|
$totalRows = 0;
|
|
$copiedRows = 0;
|
|
foreach ($tables as $table) {
|
|
if (!empty($table['skip_data'])) {
|
|
continue;
|
|
}
|
|
$rows = max(1, (int)($table['rows'] ?? 0));
|
|
$totalRows += $rows;
|
|
$copiedRows += min($rows, (int)($table['copied'] ?? 0));
|
|
}
|
|
$copyProgress = $totalRows > 0 ? ($copiedRows / $totalRows) * 65.0 : 0.0;
|
|
|
|
return round(min(89.0, 5.0 + $schemaProgress + $copyProgress), 2);
|
|
}
|
|
|
|
private static function mariaDbSeedMessage(array $context): string
|
|
{
|
|
$tables = is_array($context['tables'] ?? null) ? $context['tables'] : [];
|
|
if (($context['phase'] ?? '') === 'schema') {
|
|
return 'Creating replica schema ' . min(count($tables), (int)($context['schema_index'] ?? 0)) . ' of ' . count($tables) . '.';
|
|
}
|
|
if (($context['phase'] ?? '') === 'copy') {
|
|
$index = (int)($context['copy_index'] ?? 0);
|
|
$table = $tables[$index]['name'] ?? 'table data';
|
|
if (!empty($tables[$index]['skip_data'])) {
|
|
return 'Skipping replica data for ' . $table . '.';
|
|
}
|
|
return 'Copying replica data for ' . $table . '.';
|
|
}
|
|
if (($context['phase'] ?? '') === 'complete') {
|
|
return 'Replica seed completed.';
|
|
}
|
|
return 'Preparing replica seed.';
|
|
}
|
|
|
|
private function seedMariaDbReplicaFromPrimary(array $primary, array $host, mysqli $target): void
|
|
{
|
|
$primaryDatabase = trim((string)($primary['database_name'] ?? ''));
|
|
$targetDatabase = trim((string)($host['database_name'] ?? ''));
|
|
if ($primaryDatabase === '' || $targetDatabase === '') {
|
|
throw new RuntimeException('Database names are required before a MariaDB replica can be seeded.');
|
|
}
|
|
|
|
$source = $this->databaseConnection($primary, true);
|
|
$readLockAcquired = false;
|
|
$transactionStarted = false;
|
|
|
|
try {
|
|
$source->query('SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ');
|
|
$source->query('FLUSH TABLES WITH READ LOCK');
|
|
$readLockAcquired = true;
|
|
$source->query('START TRANSACTION WITH CONSISTENT SNAPSHOT');
|
|
$transactionStarted = true;
|
|
$sourceGtid = $this->mariaDbCurrentGtid($source);
|
|
$source->query('UNLOCK TABLES');
|
|
$readLockAcquired = false;
|
|
|
|
$this->prepareMariaDbReplicaTarget($target, $source, $primaryDatabase, $targetDatabase);
|
|
foreach ($this->databaseTableNames($source, $primaryDatabase) as $tableName) {
|
|
$this->createMariaDbReplicaTable($source, $target, $primaryDatabase, $targetDatabase, $tableName);
|
|
$this->copyMariaDbReplicaTableRows($source, $target, $primaryDatabase, $targetDatabase, $tableName);
|
|
}
|
|
|
|
if ($sourceGtid !== '') {
|
|
$this->mysqliExec($target, 'SET GLOBAL gtid_slave_pos = ' . self::sqlString($target, $sourceGtid));
|
|
}
|
|
$this->mysqliExec($target, 'SET SESSION FOREIGN_KEY_CHECKS = 1');
|
|
|
|
$source->query('COMMIT');
|
|
$transactionStarted = false;
|
|
} catch (Throwable $throwable) {
|
|
if ($readLockAcquired) {
|
|
try {
|
|
$source->query('UNLOCK TABLES');
|
|
} catch (Throwable) {
|
|
}
|
|
}
|
|
if ($transactionStarted) {
|
|
try {
|
|
$source->query('ROLLBACK');
|
|
} catch (Throwable) {
|
|
}
|
|
}
|
|
throw new RuntimeException('MariaDB replica seed failed: ' . $throwable->getMessage(), 0, $throwable);
|
|
} finally {
|
|
$source->close();
|
|
}
|
|
}
|
|
|
|
private function mariaDbCurrentGtid(mysqli $source): string
|
|
{
|
|
foreach (['gtid_binlog_pos', 'gtid_current_pos'] as $variable) {
|
|
$row = $this->mysqliSelectOne($source, "SELECT @@GLOBAL.$variable AS value");
|
|
$value = trim((string)($row['value'] ?? ''));
|
|
if ($value !== '') {
|
|
return $value;
|
|
}
|
|
}
|
|
|
|
return '';
|
|
}
|
|
|
|
private function prepareMariaDbReplicaTarget(mysqli $target, mysqli $source, string $primaryDatabase, string $targetDatabase): void
|
|
{
|
|
foreach (['STOP SLAVE', 'RESET SLAVE ALL'] as $statement) {
|
|
try {
|
|
$this->mysqliExec($target, $statement);
|
|
} catch (Throwable) {
|
|
}
|
|
}
|
|
|
|
$this->mysqliExec($target, 'SET SESSION FOREIGN_KEY_CHECKS = 0');
|
|
$this->mysqliExec($target, 'DROP DATABASE IF EXISTS ' . self::quoteIdentifier($targetDatabase));
|
|
$this->mysqliExec($target, $this->createDatabaseSql($source, $primaryDatabase, $targetDatabase));
|
|
$this->mysqliExec($target, 'USE ' . self::quoteIdentifier($targetDatabase));
|
|
|
|
try {
|
|
$this->mysqliExec($target, 'RESET MASTER');
|
|
} catch (Throwable) {
|
|
}
|
|
try {
|
|
$this->mysqliExec($target, "SET GLOBAL gtid_slave_pos = ''");
|
|
} catch (Throwable) {
|
|
}
|
|
}
|
|
|
|
private function createDatabaseSql(mysqli $source, string $primaryDatabase, string $targetDatabase): string
|
|
{
|
|
$stmt = $source->prepare(
|
|
'SELECT DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME
|
|
FROM information_schema.SCHEMATA
|
|
WHERE SCHEMA_NAME = ?
|
|
LIMIT 1'
|
|
);
|
|
if ($stmt === false) {
|
|
throw new RuntimeException('Could not prepare database schema lookup.');
|
|
}
|
|
|
|
$stmt->bind_param('s', $primaryDatabase);
|
|
$stmt->execute();
|
|
$result = $stmt->get_result();
|
|
$row = $result ? $result->fetch_assoc() : null;
|
|
$stmt->close();
|
|
|
|
$charset = preg_replace('/[^a-zA-Z0-9_]/', '', (string)($row['DEFAULT_CHARACTER_SET_NAME'] ?? 'utf8mb4')) ?: 'utf8mb4';
|
|
$collation = preg_replace('/[^a-zA-Z0-9_]/', '', (string)($row['DEFAULT_COLLATION_NAME'] ?? 'utf8mb4_unicode_ci')) ?: 'utf8mb4_unicode_ci';
|
|
|
|
return 'CREATE DATABASE ' . self::quoteIdentifier($targetDatabase)
|
|
. ' CHARACTER SET ' . $charset
|
|
. ' COLLATE ' . $collation;
|
|
}
|
|
|
|
private function createMariaDbReplicaTable(mysqli $source, mysqli $target, string $primaryDatabase, string $targetDatabase, string $tableName): void
|
|
{
|
|
$create = $this->mysqliSelectOne(
|
|
$source,
|
|
'SHOW CREATE TABLE ' . self::quoteIdentifier($primaryDatabase) . '.' . self::quoteIdentifier($tableName)
|
|
);
|
|
$createSql = (string)($create['Create Table'] ?? '');
|
|
if ($createSql === '') {
|
|
throw new RuntimeException('Could not read CREATE TABLE for ' . $primaryDatabase . '.' . $tableName . '.');
|
|
}
|
|
|
|
$this->mysqliExec($target, 'USE ' . self::quoteIdentifier($targetDatabase));
|
|
$this->mysqliExec($target, $createSql);
|
|
}
|
|
|
|
private static function mariaDbSeedContextRequiresFilterReset(array $context): bool
|
|
{
|
|
if (($context['phase'] ?? '') === '' || ($context['phase'] ?? '') === 'complete') {
|
|
return false;
|
|
}
|
|
|
|
foreach (($context['tables'] ?? []) as $table) {
|
|
if (self::databaseReplicaSeedSkipsTableData((string)($table['name'] ?? ''))
|
|
&& empty($table['skip_data'])) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private function copyMariaDbReplicaTableRows(mysqli $source, mysqli $target, string $primaryDatabase, string $targetDatabase, string $tableName): void
|
|
{
|
|
if (self::databaseReplicaSeedSkipsTableData($tableName)) {
|
|
return;
|
|
}
|
|
|
|
$offset = 0;
|
|
do {
|
|
$copied = $this->copyMariaDbReplicaTableRowsChunk(
|
|
$source,
|
|
$target,
|
|
$primaryDatabase,
|
|
$targetDatabase,
|
|
$tableName,
|
|
$offset,
|
|
self::MARIADB_SEED_BATCH_ROWS
|
|
);
|
|
$offset += $copied;
|
|
} while ($copied >= self::MARIADB_SEED_BATCH_ROWS);
|
|
}
|
|
|
|
private function copyMariaDbReplicaTableRowsChunk(
|
|
mysqli $source,
|
|
mysqli $target,
|
|
string $primaryDatabase,
|
|
string $targetDatabase,
|
|
string $tableName,
|
|
int $offset,
|
|
int $limit
|
|
): int {
|
|
$columnNames = $this->databaseWritableColumnNames($source, $primaryDatabase, $tableName);
|
|
if ($columnNames === []) {
|
|
return 0;
|
|
}
|
|
|
|
$quotedColumns = array_map(static fn(string $column): string => self::quoteIdentifier($column), $columnNames);
|
|
$primaryKeyColumns = $this->databasePrimaryKeyColumnNames($source, $primaryDatabase, $tableName);
|
|
$orderSql = $primaryKeyColumns !== []
|
|
? ' ORDER BY ' . implode(', ', array_map(static fn(string $column): string => self::quoteIdentifier($column), $primaryKeyColumns))
|
|
: '';
|
|
$result = $source->query(
|
|
'SELECT ' . implode(', ', $quotedColumns)
|
|
. ' FROM ' . self::quoteIdentifier($primaryDatabase) . '.' . self::quoteIdentifier($tableName)
|
|
. $orderSql
|
|
. ' LIMIT ' . max(0, $offset) . ', ' . max(1, $limit),
|
|
MYSQLI_USE_RESULT
|
|
);
|
|
if ($result === false) {
|
|
throw new RuntimeException('Could not read rows from ' . $primaryDatabase . '.' . $tableName . '.');
|
|
}
|
|
|
|
$fields = $result->fetch_fields();
|
|
$insertPrefix = 'INSERT INTO ' . self::quoteIdentifier($targetDatabase) . '.' . self::quoteIdentifier($tableName)
|
|
. ' (' . implode(', ', $quotedColumns) . ') VALUES ';
|
|
$rows = [];
|
|
$batchSize = 200;
|
|
$copied = 0;
|
|
|
|
try {
|
|
$target->begin_transaction();
|
|
while (true) {
|
|
$row = $result->fetch_assoc();
|
|
if (!is_array($row)) {
|
|
break;
|
|
}
|
|
|
|
$values = [];
|
|
foreach ($fields as $field) {
|
|
$value = $row[$field->name] ?? null;
|
|
$values[] = $value === null ? 'NULL' : self::sqlString($target, (string)$value);
|
|
}
|
|
$rows[] = '(' . implode(', ', $values) . ')';
|
|
$copied++;
|
|
|
|
if (count($rows) >= $batchSize) {
|
|
$this->mysqliExec($target, $insertPrefix . implode(', ', $rows));
|
|
$rows = [];
|
|
}
|
|
}
|
|
|
|
if ($rows !== []) {
|
|
$this->mysqliExec($target, $insertPrefix . implode(', ', $rows));
|
|
}
|
|
$target->commit();
|
|
} catch (Throwable $throwable) {
|
|
try {
|
|
$target->rollback();
|
|
} catch (Throwable) {
|
|
}
|
|
throw $throwable;
|
|
} finally {
|
|
$result->free();
|
|
}
|
|
|
|
return $copied;
|
|
}
|
|
|
|
private function estimatedDatabaseTableRows(mysqli $connection, string $database, string $tableName): int
|
|
{
|
|
$stmt = $connection->prepare(
|
|
"SELECT TABLE_ROWS
|
|
FROM information_schema.TABLES
|
|
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND TABLE_TYPE = 'BASE TABLE'
|
|
LIMIT 1"
|
|
);
|
|
if ($stmt === false) {
|
|
return 1;
|
|
}
|
|
|
|
$stmt->bind_param('ss', $database, $tableName);
|
|
$stmt->execute();
|
|
$result = $stmt->get_result();
|
|
$row = $result ? $result->fetch_assoc() : null;
|
|
$stmt->close();
|
|
|
|
return max(1, (int)($row['TABLE_ROWS'] ?? 1));
|
|
}
|
|
|
|
private static function databaseReplicaSeedSkipsTableData(string $tableName): bool
|
|
{
|
|
return in_array(strtolower($tableName), self::MARIADB_SCHEMA_ONLY_TABLES, true);
|
|
}
|
|
|
|
private function databaseWritableColumnNames(mysqli $connection, string $database, string $tableName): array
|
|
{
|
|
$stmt = $connection->prepare(
|
|
"SELECT COLUMN_NAME
|
|
FROM information_schema.COLUMNS
|
|
WHERE TABLE_SCHEMA = ?
|
|
AND TABLE_NAME = ?
|
|
AND EXTRA NOT LIKE '%GENERATED%'
|
|
ORDER BY ORDINAL_POSITION"
|
|
);
|
|
if ($stmt === false) {
|
|
throw new RuntimeException('Could not prepare table column lookup.');
|
|
}
|
|
|
|
$stmt->bind_param('ss', $database, $tableName);
|
|
$stmt->execute();
|
|
$result = $stmt->get_result();
|
|
$rows = $result ? $result->fetch_all(MYSQLI_ASSOC) : [];
|
|
$stmt->close();
|
|
|
|
return array_values(array_filter(array_map(
|
|
static fn(array $row): string => (string)($row['COLUMN_NAME'] ?? ''),
|
|
$rows
|
|
)));
|
|
}
|
|
|
|
private function databasePrimaryKeyColumnNames(mysqli $connection, string $database, string $tableName): array
|
|
{
|
|
$stmt = $connection->prepare(
|
|
"SELECT COLUMN_NAME
|
|
FROM information_schema.KEY_COLUMN_USAGE
|
|
WHERE TABLE_SCHEMA = ?
|
|
AND TABLE_NAME = ?
|
|
AND CONSTRAINT_NAME = 'PRIMARY'
|
|
ORDER BY ORDINAL_POSITION"
|
|
);
|
|
if ($stmt === false) {
|
|
return [];
|
|
}
|
|
|
|
$stmt->bind_param('ss', $database, $tableName);
|
|
$stmt->execute();
|
|
$result = $stmt->get_result();
|
|
$rows = $result ? $result->fetch_all(MYSQLI_ASSOC) : [];
|
|
$stmt->close();
|
|
|
|
return array_values(array_filter(array_map(
|
|
static fn(array $row): string => (string)($row['COLUMN_NAME'] ?? ''),
|
|
$rows
|
|
)));
|
|
}
|
|
|
|
private function databaseTableNames(mysqli $connection, string $database): array
|
|
{
|
|
$stmt = $connection->prepare(
|
|
"SELECT TABLE_NAME FROM information_schema.TABLES
|
|
WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE'
|
|
ORDER BY TABLE_NAME"
|
|
);
|
|
if ($stmt === false) {
|
|
throw new RuntimeException('Could not prepare database table comparison query.');
|
|
}
|
|
|
|
$stmt->bind_param('s', $database);
|
|
$stmt->execute();
|
|
$result = $stmt->get_result();
|
|
$rows = $result ? $result->fetch_all(MYSQLI_ASSOC) : [];
|
|
$stmt->close();
|
|
|
|
return array_values(array_map(
|
|
static fn(array $row): string => (string)($row['TABLE_NAME'] ?? ''),
|
|
$rows
|
|
));
|
|
}
|
|
|
|
private function databaseSchemaOnlyTablesWithRows(mysqli $connection, string $database): array
|
|
{
|
|
$existingTables = [];
|
|
foreach ($this->databaseTableNames($connection, $database) as $tableName) {
|
|
$existingTables[strtolower($tableName)] = $tableName;
|
|
}
|
|
|
|
$tablesWithRows = [];
|
|
foreach (self::MARIADB_SCHEMA_ONLY_TABLES as $schemaOnlyTable) {
|
|
$actualTable = $existingTables[strtolower($schemaOnlyTable)] ?? null;
|
|
if ($actualTable === null) {
|
|
continue;
|
|
}
|
|
|
|
$row = $this->mysqliSelectOne(
|
|
$connection,
|
|
'SELECT 1 AS has_rows FROM ' . self::quoteIdentifier($database) . '.' . self::quoteIdentifier($actualTable) . ' LIMIT 1'
|
|
);
|
|
if (($row['has_rows'] ?? null) !== null) {
|
|
$tablesWithRows[] = $actualTable;
|
|
}
|
|
}
|
|
|
|
return $tablesWithRows;
|
|
}
|
|
|
|
private static function missingDatabaseTablesBlocker(string $database, array $missingTables): string
|
|
{
|
|
$shownTables = array_slice($missingTables, 0, 3);
|
|
$qualifiedTables = array_map(
|
|
static fn(string $table): string => $database . '.' . $table,
|
|
$shownTables
|
|
);
|
|
$tableWord = count($missingTables) === 1 ? 'table' : 'tables';
|
|
$sample = $qualifiedTables !== [] ? ': ' . implode(', ', $qualifiedTables) : '';
|
|
|
|
return 'Replica seed is incomplete. Missing ' . count($missingTables) . ' database ' . $tableWord . ' on replica' . $sample . '.';
|
|
}
|
|
|
|
private static function schemaOnlyTablesContainRowsBlocker(string $database, array $tables): string
|
|
{
|
|
$shownTables = array_slice($tables, 0, 3);
|
|
$qualifiedTables = array_map(
|
|
static fn(string $table): string => $database . '.' . $table,
|
|
$shownTables
|
|
);
|
|
$sample = $qualifiedTables !== [] ? ': ' . implode(', ', $qualifiedTables) : '';
|
|
|
|
return 'Replica seed includes data for schema-only tables' . $sample . '. Re-run provisioning to rebuild the replica without schema-only table data.';
|
|
}
|
|
|
|
private function minioS3Client(array $host): S3Client
|
|
{
|
|
$credentials = $this->credentials($host);
|
|
return new S3Client([
|
|
'version' => 'latest',
|
|
'region' => 'us-east-1',
|
|
'endpoint' => self::minioEndpoint($host),
|
|
'use_path_style_endpoint' => true,
|
|
'retries' => 0,
|
|
'http' => [
|
|
'connect_timeout' => self::MINIO_S3_CONNECT_TIMEOUT_SECONDS,
|
|
'timeout' => self::MINIO_S3_REQUEST_TIMEOUT_SECONDS,
|
|
],
|
|
'credentials' => [
|
|
'key' => $credentials['username'],
|
|
'secret' => $credentials['password'],
|
|
],
|
|
]);
|
|
}
|
|
|
|
private static function minioObjectLastModifiedTimestamp(mixed $value): ?int
|
|
{
|
|
if ($value instanceof \DateTimeInterface) {
|
|
return $value->getTimestamp();
|
|
}
|
|
if (is_numeric($value)) {
|
|
return (int)$value;
|
|
}
|
|
$timestamp = strtotime((string)$value);
|
|
return $timestamp === false ? null : $timestamp;
|
|
}
|
|
|
|
private function minioBucketStats(
|
|
S3Client $client,
|
|
array $buckets,
|
|
bool $requireExists,
|
|
string $missingPrefix,
|
|
bool $measureObjects = true,
|
|
array $retentionDaysByBucket = []
|
|
): array
|
|
{
|
|
$stats = [
|
|
'bytes' => 0,
|
|
'objects' => 0,
|
|
'expired_bytes' => 0,
|
|
'expired_objects' => 0,
|
|
'buckets' => [],
|
|
'blockers' => [],
|
|
];
|
|
|
|
foreach ($buckets as $bucket) {
|
|
$retentionDays = isset($retentionDaysByBucket[$bucket]) ? (int)$retentionDaysByBucket[$bucket] : null;
|
|
$retentionCutoff = $retentionDays !== null ? time() - ($retentionDays * 86400) : null;
|
|
try {
|
|
$exists = (bool)$client->doesBucketExist($bucket);
|
|
if (!$exists) {
|
|
$stats['buckets'][] = [
|
|
'name' => $bucket,
|
|
'status' => 'missing',
|
|
'bytes' => 0,
|
|
'objects' => 0,
|
|
];
|
|
if ($requireExists) {
|
|
$stats['blockers'][] = $missingPrefix . ' ' . $bucket . ' is missing.';
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (!$measureObjects) {
|
|
$stats['buckets'][] = [
|
|
'name' => $bucket,
|
|
'status' => 'ok',
|
|
'bytes' => null,
|
|
'objects' => null,
|
|
'expired_bytes' => null,
|
|
'expired_objects' => null,
|
|
'measured' => false,
|
|
'retention_days' => $retentionDays,
|
|
];
|
|
continue;
|
|
}
|
|
|
|
$bucketBytes = 0;
|
|
$bucketObjects = 0;
|
|
$bucketExpiredBytes = 0;
|
|
$bucketExpiredObjects = 0;
|
|
$token = null;
|
|
do {
|
|
$args = ['Bucket' => $bucket];
|
|
if ($token !== null) {
|
|
$args['ContinuationToken'] = $token;
|
|
}
|
|
$result = $client->listObjectsV2($args);
|
|
foreach (($result['Contents'] ?? []) as $object) {
|
|
$size = (int)($object['Size'] ?? 0);
|
|
$lastModified = self::minioObjectLastModifiedTimestamp($object['LastModified'] ?? null);
|
|
if ($retentionCutoff !== null && $lastModified !== null && $lastModified < $retentionCutoff) {
|
|
$bucketExpiredBytes += $size;
|
|
$bucketExpiredObjects++;
|
|
continue;
|
|
}
|
|
|
|
$bucketBytes += $size;
|
|
$bucketObjects++;
|
|
}
|
|
$token = isset($result['NextContinuationToken']) ? (string)$result['NextContinuationToken'] : null;
|
|
} while ($token !== null);
|
|
|
|
$stats['bytes'] += $bucketBytes;
|
|
$stats['objects'] += $bucketObjects;
|
|
$stats['expired_bytes'] += $bucketExpiredBytes;
|
|
$stats['expired_objects'] += $bucketExpiredObjects;
|
|
$stats['buckets'][] = [
|
|
'name' => $bucket,
|
|
'status' => 'ok',
|
|
'bytes' => $bucketBytes,
|
|
'objects' => $bucketObjects,
|
|
'expired_bytes' => $bucketExpiredBytes,
|
|
'expired_objects' => $bucketExpiredObjects,
|
|
'retention_days' => $retentionDays,
|
|
'retention_cutoff' => $retentionCutoff !== null ? date('c', $retentionCutoff) : null,
|
|
];
|
|
} catch (Throwable $throwable) {
|
|
$stats['buckets'][] = [
|
|
'name' => $bucket,
|
|
'status' => 'down',
|
|
'bytes' => 0,
|
|
'objects' => 0,
|
|
'expired_bytes' => 0,
|
|
'expired_objects' => 0,
|
|
'retention_days' => $retentionDays,
|
|
'error' => $throwable->getMessage(),
|
|
];
|
|
$stats['blockers'][] = $missingPrefix . ' ' . $bucket . ' could not be inspected: ' . $throwable->getMessage();
|
|
}
|
|
}
|
|
|
|
$stats['blockers'] = array_values(array_unique($stats['blockers']));
|
|
return $stats;
|
|
}
|
|
|
|
private function configureMinioReplication(array $primary, array $target): void
|
|
{
|
|
$primaryOptions = $this->decodeOptions($primary);
|
|
$targetOptions = $this->decodeOptions($target);
|
|
$buckets = self::normalizeMinioBuckets($targetOptions['buckets'] ?? $primaryOptions['buckets'] ?? self::MINIO_DEFAULT_BUCKETS);
|
|
$transferLimit = self::minioReplicationTransferLimitFromOptions($targetOptions);
|
|
$configDir = $this->createMinioConfigDir();
|
|
|
|
try {
|
|
$this->prepareMinioAlias($configDir, 'source', $primary);
|
|
$this->prepareMinioAlias($configDir, 'target', $target);
|
|
|
|
foreach ($buckets as $index => $bucket) {
|
|
$this->runMinioClient($configDir, ['mb', '--with-lock', '--ignore-existing', 'target/' . $bucket]);
|
|
$this->runMinioClient($configDir, ['version', 'enable', 'source/' . $bucket]);
|
|
$this->runMinioClient($configDir, ['version', 'enable', 'target/' . $bucket]);
|
|
|
|
if (self::minioBucketUsesBoundedReplicaRetention($bucket)) {
|
|
$this->configureMinioReplicaBackupRetention($target, $bucket);
|
|
// Backups are bounded on replicas; avoid bulk seeding large historical objects.
|
|
$this->pruneMinioReplicaBackupRetention($target, $bucket);
|
|
}
|
|
|
|
$this->addMinioReplicationRule($configDir, $target, $bucket, $index + 1, $transferLimit);
|
|
}
|
|
} finally {
|
|
$this->removeDirectory($configDir);
|
|
}
|
|
}
|
|
|
|
private function configureMinioReplicaBackupRetention(array $target, string $bucket): void
|
|
{
|
|
$client = $this->minioS3Client($target);
|
|
$rules = [];
|
|
|
|
try {
|
|
$current = $client->getBucketLifecycleConfiguration(['Bucket' => $bucket]);
|
|
foreach (($current['Rules'] ?? []) as $rule) {
|
|
if ((string)($rule['ID'] ?? '') !== self::MINIO_BACKUP_REPLICA_RETENTION_RULE_ID) {
|
|
$rules[] = $rule;
|
|
}
|
|
}
|
|
} catch (Throwable $throwable) {
|
|
if (!self::minioMissingLifecycleConfiguration($throwable)) {
|
|
throw $throwable;
|
|
}
|
|
}
|
|
|
|
$rules[] = self::minioBackupReplicaRetentionLifecycleRule();
|
|
|
|
$client->putBucketLifecycleConfiguration([
|
|
'Bucket' => $bucket,
|
|
'LifecycleConfiguration' => [
|
|
'Rules' => $rules,
|
|
],
|
|
]);
|
|
}
|
|
|
|
private static function minioBackupReplicaRetentionLifecycleRule(): array
|
|
{
|
|
return [
|
|
'ID' => self::MINIO_BACKUP_REPLICA_RETENTION_RULE_ID,
|
|
'Status' => 'Enabled',
|
|
'Filter' => ['Prefix' => ''],
|
|
'Expiration' => ['Days' => self::MINIO_BACKUP_REPLICA_RETENTION_DAYS],
|
|
'NoncurrentVersionExpiration' => ['NoncurrentDays' => self::MINIO_BACKUP_REPLICA_RETENTION_DAYS],
|
|
'AbortIncompleteMultipartUpload' => ['DaysAfterInitiation' => 7],
|
|
];
|
|
}
|
|
|
|
private static function minioMissingLifecycleConfiguration(Throwable $throwable): bool
|
|
{
|
|
$message = strtolower($throwable->getMessage());
|
|
return str_contains($message, 'nosuchlifecycleconfiguration')
|
|
|| str_contains($message, 'lifecycle configuration does not exist')
|
|
|| str_contains($message, 'the lifecycle configuration does not exist');
|
|
}
|
|
|
|
private function minioBackupReplicaRetentionConfigured(array $target, string $bucket): bool
|
|
{
|
|
try {
|
|
$current = $this->minioS3Client($target)->getBucketLifecycleConfiguration(['Bucket' => $bucket]);
|
|
} catch (Throwable) {
|
|
return false;
|
|
}
|
|
|
|
foreach (($current['Rules'] ?? []) as $rule) {
|
|
if ((string)($rule['ID'] ?? '') !== self::MINIO_BACKUP_REPLICA_RETENTION_RULE_ID) {
|
|
continue;
|
|
}
|
|
if (strtolower((string)($rule['Status'] ?? '')) !== 'enabled') {
|
|
return false;
|
|
}
|
|
|
|
$expirationDays = (int)($rule['Expiration']['Days'] ?? 0);
|
|
$noncurrentDays = (int)($rule['NoncurrentVersionExpiration']['NoncurrentDays'] ?? 0);
|
|
return $expirationDays > 0
|
|
&& $expirationDays <= self::MINIO_BACKUP_REPLICA_RETENTION_DAYS
|
|
&& $noncurrentDays > 0
|
|
&& $noncurrentDays <= self::MINIO_BACKUP_REPLICA_RETENTION_DAYS;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private function pruneMinioReplicaBackupRetention(array $target, string $bucket): array
|
|
{
|
|
$client = $this->minioS3Client($target);
|
|
$cutoff = time() - (self::MINIO_BACKUP_REPLICA_RETENTION_DAYS * 86400);
|
|
$deleted = [
|
|
'versions' => 0,
|
|
'delete_markers' => 0,
|
|
'cutoff' => date('c', $cutoff),
|
|
];
|
|
|
|
try {
|
|
$this->pruneMinioReplicaBackupVersions($client, $bucket, $cutoff, $deleted);
|
|
} catch (Throwable $throwable) {
|
|
if (!self::minioVersionListingUnsupported($throwable)) {
|
|
throw $throwable;
|
|
}
|
|
$this->pruneMinioReplicaBackupCurrentObjects($client, $bucket, $cutoff, $deleted);
|
|
}
|
|
|
|
return $deleted;
|
|
}
|
|
|
|
private static function minioVersionListingUnsupported(Throwable $throwable): bool
|
|
{
|
|
$message = strtolower($throwable->getMessage());
|
|
return str_contains($message, 'not implemented')
|
|
|| str_contains($message, 'not supported')
|
|
|| str_contains($message, 'unsupported')
|
|
|| str_contains($message, 'listobjectversions');
|
|
}
|
|
|
|
private function pruneMinioReplicaBackupVersions(S3Client $client, string $bucket, int $cutoff, array &$deleted): void
|
|
{
|
|
$keyMarker = null;
|
|
$versionIdMarker = null;
|
|
do {
|
|
$args = ['Bucket' => $bucket];
|
|
if ($keyMarker !== null) {
|
|
$args['KeyMarker'] = $keyMarker;
|
|
}
|
|
if ($versionIdMarker !== null) {
|
|
$args['VersionIdMarker'] = $versionIdMarker;
|
|
}
|
|
|
|
$result = $client->listObjectVersions($args);
|
|
$objects = [];
|
|
|
|
foreach (($result['Versions'] ?? []) as $version) {
|
|
if (self::minioObjectVersionIsOlderThan($version, $cutoff)) {
|
|
$objects[] = [
|
|
'Key' => (string)($version['Key'] ?? ''),
|
|
'VersionId' => (string)($version['VersionId'] ?? ''),
|
|
];
|
|
$deleted['versions']++;
|
|
}
|
|
}
|
|
|
|
foreach (($result['DeleteMarkers'] ?? []) as $marker) {
|
|
if (self::minioObjectVersionIsOlderThan($marker, $cutoff)) {
|
|
$objects[] = [
|
|
'Key' => (string)($marker['Key'] ?? ''),
|
|
'VersionId' => (string)($marker['VersionId'] ?? ''),
|
|
];
|
|
$deleted['delete_markers']++;
|
|
}
|
|
}
|
|
|
|
$this->deleteMinioObjectsInBatches($client, $bucket, $objects);
|
|
|
|
$keyMarker = isset($result['NextKeyMarker']) ? (string)$result['NextKeyMarker'] : null;
|
|
$versionIdMarker = isset($result['NextVersionIdMarker']) ? (string)$result['NextVersionIdMarker'] : null;
|
|
} while (!empty($result['IsTruncated']));
|
|
}
|
|
|
|
private function pruneMinioReplicaBackupCurrentObjects(S3Client $client, string $bucket, int $cutoff, array &$deleted): void
|
|
{
|
|
$token = null;
|
|
do {
|
|
$args = ['Bucket' => $bucket];
|
|
if ($token !== null) {
|
|
$args['ContinuationToken'] = $token;
|
|
}
|
|
|
|
$result = $client->listObjectsV2($args);
|
|
$objects = [];
|
|
foreach (($result['Contents'] ?? []) as $object) {
|
|
if (!self::minioObjectVersionIsOlderThan($object, $cutoff)) {
|
|
continue;
|
|
}
|
|
$objects[] = ['Key' => (string)($object['Key'] ?? '')];
|
|
$deleted['versions']++;
|
|
}
|
|
|
|
$this->deleteMinioObjectsInBatches($client, $bucket, $objects);
|
|
$token = isset($result['NextContinuationToken']) ? (string)$result['NextContinuationToken'] : null;
|
|
} while ($token !== null);
|
|
}
|
|
|
|
private static function minioObjectVersionIsOlderThan(array $object, int $cutoff): bool
|
|
{
|
|
$key = trim((string)($object['Key'] ?? ''));
|
|
if ($key === '') {
|
|
return false;
|
|
}
|
|
|
|
$lastModified = self::minioObjectLastModifiedTimestamp($object['LastModified'] ?? null);
|
|
return $lastModified !== null && $lastModified < $cutoff;
|
|
}
|
|
|
|
private function deleteMinioObjectsInBatches(S3Client $client, string $bucket, array $objects): void
|
|
{
|
|
foreach (array_chunk($objects, 1000) as $chunk) {
|
|
$chunk = array_values(array_filter(
|
|
$chunk,
|
|
static fn(array $object): bool => trim((string)($object['Key'] ?? '')) !== ''
|
|
));
|
|
if ($chunk === []) {
|
|
continue;
|
|
}
|
|
|
|
$client->deleteObjects([
|
|
'Bucket' => $bucket,
|
|
'Delete' => [
|
|
'Objects' => $chunk,
|
|
'Quiet' => true,
|
|
],
|
|
]);
|
|
}
|
|
}
|
|
|
|
private function addMinioReplicationRule(string $configDir, array $target, string $bucket, int $priority, string $transferLimit): void
|
|
{
|
|
try {
|
|
$this->runMinioClient($configDir, self::minioReplicationRuleCommand($bucket, $priority, $transferLimit));
|
|
return;
|
|
} catch (Throwable $throwable) {
|
|
$message = strtolower($throwable->getMessage());
|
|
if (self::minioReplicationRuleAlreadyExists($message)) {
|
|
$this->updateMinioReplicationRulesForBucket($configDir, $bucket, $transferLimit);
|
|
return;
|
|
}
|
|
if (!$this->repairMinioTargetBucketObjectLockIfEmpty($configDir, $target, $bucket, $message)) {
|
|
throw $throwable;
|
|
}
|
|
}
|
|
|
|
try {
|
|
$this->runMinioClient($configDir, self::minioReplicationRuleCommand($bucket, $priority, $transferLimit));
|
|
} catch (Throwable $throwable) {
|
|
$message = strtolower($throwable->getMessage());
|
|
if (!self::minioReplicationRuleAlreadyExists($message)) {
|
|
throw $throwable;
|
|
}
|
|
$this->updateMinioReplicationRulesForBucket($configDir, $bucket, $transferLimit);
|
|
}
|
|
}
|
|
|
|
private function updateMinioReplicationRulesForBucket(string $configDir, string $bucket, string $transferLimit): void
|
|
{
|
|
$result = $this->runMinioClient($configDir, ['replicate', 'ls', '--json', 'source/' . $bucket]);
|
|
$ruleIds = self::minioReplicationRuleIdsFromList(self::decodeMinioJsonOutput((string)$result['stdout']));
|
|
foreach ($ruleIds as $ruleId) {
|
|
$this->runMinioClient($configDir, array_merge([
|
|
'replicate',
|
|
'update',
|
|
'--id',
|
|
$ruleId,
|
|
'--replicate',
|
|
self::minioReplicationFeatures($bucket),
|
|
], self::minioReplicationTransferLimitArgs($transferLimit), [
|
|
'source/' . $bucket,
|
|
]));
|
|
}
|
|
}
|
|
|
|
private static function minioReplicationRuleCommand(string $bucket, int $priority, string $transferLimit): array
|
|
{
|
|
return array_merge([
|
|
'replicate',
|
|
'add',
|
|
'--remote-bucket',
|
|
'target/' . $bucket,
|
|
'--replicate',
|
|
self::minioReplicationFeatures($bucket),
|
|
'--priority',
|
|
(string)$priority,
|
|
], self::minioReplicationTransferLimitArgs($transferLimit), [
|
|
'source/' . $bucket,
|
|
]);
|
|
}
|
|
|
|
private static function minioReplicationFeatures(string $bucket): string
|
|
{
|
|
return self::minioBucketUsesBoundedReplicaRetention($bucket)
|
|
? 'delete,delete-marker'
|
|
: 'delete,delete-marker,existing-objects';
|
|
}
|
|
|
|
private static function minioReplicationRuleIdsFromList(mixed $value): array
|
|
{
|
|
$ids = [];
|
|
self::collectMinioReplicationRuleIds($value, $ids);
|
|
return array_values(array_unique(array_filter($ids)));
|
|
}
|
|
|
|
private static function collectMinioReplicationRuleIds(mixed $value, array &$ids): void
|
|
{
|
|
if (!is_array($value)) {
|
|
return;
|
|
}
|
|
|
|
foreach ($value as $key => $entry) {
|
|
$normalizedKey = strtolower(str_replace(['_', '-'], '', (string)$key));
|
|
if (in_array($normalizedKey, ['id', 'ruleid'], true) && is_scalar($entry)) {
|
|
$id = trim((string)$entry);
|
|
if ($id !== '') {
|
|
$ids[] = $id;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
self::collectMinioReplicationRuleIds($entry, $ids);
|
|
}
|
|
}
|
|
|
|
private static function minioReplicationRuleAlreadyExists(string $message): bool
|
|
{
|
|
return str_contains($message, 'already')
|
|
|| str_contains($message, 'replication rule exists')
|
|
|| str_contains($message, 'replication configuration exists');
|
|
}
|
|
|
|
private function repairMinioTargetBucketObjectLockIfEmpty(string $configDir, array $target, string $bucket, string $message): bool
|
|
{
|
|
if (!self::minioObjectLockRequiredError($message)) {
|
|
return false;
|
|
}
|
|
|
|
if ($this->minioBucketHasObjects($target, $bucket)) {
|
|
throw new RuntimeException(
|
|
'MinIO target bucket ' . $bucket . ' was created without Object Lock and is not empty. '
|
|
. 'Create a new empty replica bucket with Object Lock enabled, or empty and recreate this bucket before provisioning.'
|
|
);
|
|
}
|
|
|
|
$this->runMinioClient($configDir, ['rb', 'target/' . $bucket]);
|
|
$this->runMinioClient($configDir, ['mb', '--with-lock', 'target/' . $bucket]);
|
|
$this->runMinioClient($configDir, ['version', 'enable', 'target/' . $bucket]);
|
|
|
|
return true;
|
|
}
|
|
|
|
private static function minioObjectLockRequiredError(string $message): bool
|
|
{
|
|
return (str_contains($message, 'object lock') || str_contains($message, 'object locking'))
|
|
&& str_contains($message, 'destination bucket');
|
|
}
|
|
|
|
private function minioBucketHasObjects(array $host, string $bucket): bool
|
|
{
|
|
$client = $this->minioS3Client($host);
|
|
$objects = $client->listObjectsV2([
|
|
'Bucket' => $bucket,
|
|
'MaxKeys' => 1,
|
|
]);
|
|
if (!empty($objects['Contents'])) {
|
|
return true;
|
|
}
|
|
|
|
try {
|
|
$versions = $client->listObjectVersions([
|
|
'Bucket' => $bucket,
|
|
'MaxKeys' => 1,
|
|
]);
|
|
return !empty($versions['Versions']) || !empty($versions['DeleteMarkers']);
|
|
} catch (Throwable) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private function minioReplicationConfiguredForHosts(array $primary, array $target): bool
|
|
{
|
|
$primaryOptions = $this->decodeOptions($primary);
|
|
$targetOptions = $this->decodeOptions($target);
|
|
$buckets = self::normalizeMinioBuckets($targetOptions['buckets'] ?? $primaryOptions['buckets'] ?? self::MINIO_DEFAULT_BUCKETS);
|
|
|
|
if (!$this->minioReplicationConfigured($primary, $buckets)) {
|
|
return false;
|
|
}
|
|
|
|
if (in_array(self::MINIO_BACKUP_BUCKET, $buckets, true)
|
|
&& !$this->minioBackupReplicaRetentionConfigured($target, self::MINIO_BACKUP_BUCKET)) {
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
private function minioReplicationConfigured(array $primary, array $buckets): bool
|
|
{
|
|
$configDir = $this->createMinioConfigDir();
|
|
try {
|
|
$this->prepareMinioAlias($configDir, 'source', $primary);
|
|
foreach ($buckets as $bucket) {
|
|
try {
|
|
$result = $this->runMinioClient($configDir, ['replicate', 'list', '--json', 'source/' . $bucket]);
|
|
} catch (Throwable) {
|
|
return false;
|
|
}
|
|
if (trim((string)$result['stdout']) === '') {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
} finally {
|
|
$this->removeDirectory($configDir);
|
|
}
|
|
}
|
|
|
|
private function minioReplicationProgressStatus(array $primary, array $target, array $buckets): ?array
|
|
{
|
|
$targetOptions = $this->decodeOptions($target);
|
|
$transferLimit = self::minioReplicationTransferLimitFromOptions($targetOptions);
|
|
$configDir = $this->createMinioConfigDir();
|
|
$bucketProgress = [];
|
|
$bucketOutput = [];
|
|
$requiredUnavailableBuckets = [];
|
|
|
|
try {
|
|
$this->prepareMinioAlias($configDir, 'source', $primary);
|
|
foreach ($buckets as $bucket) {
|
|
$countsTowardCatchUp = self::minioBucketCountsTowardCatchUp((string)$bucket);
|
|
try {
|
|
// MinIO keeps removed/re-added ARNs in JSON status output. Prefer standard
|
|
// output so stale targets do not keep a healthy current target below 100%.
|
|
$result = $this->runMinioClient($configDir, array_merge([
|
|
'replicate',
|
|
'status',
|
|
'source/' . $bucket,
|
|
], self::minioReplicationTransferLimitArgs($transferLimit)));
|
|
$stdout = (string)$result['stdout'];
|
|
$raw = $stdout;
|
|
$progress = self::minioReplicationProgressFromStatusOutput($stdout);
|
|
|
|
if ($progress === null) {
|
|
$result = $this->runMinioClient($configDir, array_merge([
|
|
'replicate',
|
|
'status',
|
|
'--json',
|
|
'source/' . $bucket,
|
|
], self::minioReplicationTransferLimitArgs($transferLimit)));
|
|
$stdout = (string)$result['stdout'];
|
|
$decoded = self::decodeMinioJsonOutput($stdout);
|
|
$raw = $decoded ?? $stdout;
|
|
$progress = self::minioReplicationProgressFromStatusOutput($raw)
|
|
?? self::minioReplicationProgressFromStatusOutput($stdout);
|
|
}
|
|
|
|
$bucketOutput[$bucket] = [
|
|
'ok' => true,
|
|
'counts_toward_catch_up' => $countsTowardCatchUp,
|
|
'raw' => $raw,
|
|
'progress' => $progress,
|
|
];
|
|
if ($progress !== null) {
|
|
$bucketProgress[$bucket] = $progress;
|
|
} elseif ($countsTowardCatchUp) {
|
|
$requiredUnavailableBuckets[] = (string)$bucket;
|
|
}
|
|
} catch (Throwable $throwable) {
|
|
$bucketOutput[$bucket] = [
|
|
'ok' => false,
|
|
'counts_toward_catch_up' => $countsTowardCatchUp,
|
|
'error' => $throwable->getMessage(),
|
|
];
|
|
if ($countsTowardCatchUp) {
|
|
$requiredUnavailableBuckets[] = (string)$bucket;
|
|
}
|
|
}
|
|
}
|
|
} finally {
|
|
$this->removeDirectory($configDir);
|
|
}
|
|
|
|
$progress = self::minioCatchUpProgressFromBucketStatuses($bucketProgress);
|
|
if ($progress === null) {
|
|
return null;
|
|
}
|
|
|
|
$requiredUnavailableBuckets = array_values(array_unique($requiredUnavailableBuckets));
|
|
if ($requiredUnavailableBuckets !== []) {
|
|
$progress['replication_percent'] = self::minioIncompleteProgress((float)$progress['replication_percent']);
|
|
$progress['blockers'] = array_values(array_unique(array_merge($progress['blockers'] ?? [], [
|
|
'MinIO replication status is unavailable for bucket(s): ' . implode(', ', $requiredUnavailableBuckets) . '.',
|
|
])));
|
|
$progress['unavailable_required_buckets'] = $requiredUnavailableBuckets;
|
|
}
|
|
|
|
$progress['buckets'] = $bucketOutput;
|
|
$progress['target_endpoint'] = self::minioEndpoint($target);
|
|
|
|
return $progress;
|
|
}
|
|
|
|
public static function minioCatchUpProgressFromBucketStatuses(array $bucketProgress): ?array
|
|
{
|
|
$requiredProgress = [];
|
|
$ignoredBuckets = [];
|
|
|
|
foreach ($bucketProgress as $bucket => $progress) {
|
|
$bucket = (string)$bucket;
|
|
if (self::minioBucketCountsTowardCatchUp($bucket)) {
|
|
$requiredProgress[$bucket] = $progress;
|
|
continue;
|
|
}
|
|
|
|
$ignoredBuckets[] = $bucket;
|
|
}
|
|
|
|
$progress = self::aggregateMinioReplicationProgress($requiredProgress);
|
|
if ($progress === null && $ignoredBuckets !== []) {
|
|
$progress = [
|
|
'replication_percent' => 100.0,
|
|
'blockers' => [],
|
|
'basis' => 'bounded_retention_only',
|
|
'stats' => [
|
|
'completed_bytes' => 0.0,
|
|
'pending_bytes' => 0.0,
|
|
'failed_bytes' => 0.0,
|
|
'total_bytes' => 0.0,
|
|
'completed_count' => 0.0,
|
|
'pending_count' => 0.0,
|
|
'failed_count' => 0.0,
|
|
'total_count' => 0.0,
|
|
],
|
|
'bucket_count' => 0,
|
|
];
|
|
}
|
|
|
|
if ($progress === null) {
|
|
return null;
|
|
}
|
|
|
|
$progress['ignored_buckets'] = $ignoredBuckets;
|
|
$progress['catch_up_bucket_count'] = count($requiredProgress);
|
|
|
|
return $progress;
|
|
}
|
|
|
|
private static function aggregateMinioReplicationProgress(array $bucketProgress): ?array
|
|
{
|
|
if ($bucketProgress === []) {
|
|
return null;
|
|
}
|
|
|
|
$stats = [
|
|
'completed_bytes' => 0.0,
|
|
'pending_bytes' => 0.0,
|
|
'failed_bytes' => 0.0,
|
|
'total_bytes' => 0.0,
|
|
'completed_count' => 0.0,
|
|
'pending_count' => 0.0,
|
|
'failed_count' => 0.0,
|
|
'total_count' => 0.0,
|
|
];
|
|
$percentages = [];
|
|
foreach ($bucketProgress as $progress) {
|
|
$percentages[] = (float)($progress['replication_percent'] ?? 0);
|
|
$progressStats = is_array($progress['stats'] ?? null) ? $progress['stats'] : [];
|
|
foreach (array_keys($stats) as $key) {
|
|
$stats[$key] += (float)($progressStats[$key] ?? 0);
|
|
}
|
|
}
|
|
|
|
$completedBytes = $stats['completed_bytes'];
|
|
$remainingBytes = $stats['pending_bytes'] + $stats['failed_bytes'];
|
|
$totalBytes = $stats['total_bytes'];
|
|
$completedCount = $stats['completed_count'];
|
|
$remainingCount = $stats['pending_count'] + $stats['failed_count'];
|
|
$totalCount = $stats['total_count'];
|
|
$basis = 'bucket_average';
|
|
|
|
if ($totalBytes > 0.0) {
|
|
$percent = ($completedBytes / $totalBytes) * 100;
|
|
$basis = 'total_bytes';
|
|
} elseif (($completedBytes + $remainingBytes) > 0.0) {
|
|
$percent = ($completedBytes / ($completedBytes + $remainingBytes)) * 100;
|
|
$basis = 'byte_balance';
|
|
} elseif ($totalCount > 0.0) {
|
|
$percent = ($completedCount / $totalCount) * 100;
|
|
$basis = 'total_count';
|
|
} elseif (($completedCount + $remainingCount) > 0.0) {
|
|
$percent = ($completedCount / ($completedCount + $remainingCount)) * 100;
|
|
$basis = 'count_balance';
|
|
} else {
|
|
$percent = array_sum($percentages) / max(1, count($percentages));
|
|
}
|
|
|
|
$percent = round(min(100.0, max(0.0, $percent)), 2);
|
|
$withinTolerance = $stats['failed_bytes'] <= 0.0
|
|
&& $stats['failed_count'] <= 0.0
|
|
&& $stats['pending_bytes'] <= self::MINIO_CATCH_UP_PENDING_BYTES_TOLERANCE
|
|
&& $stats['pending_count'] <= self::MINIO_CATCH_UP_PENDING_OBJECTS_TOLERANCE;
|
|
if ($percent < 100.0 && $withinTolerance) {
|
|
$percent = 100.0;
|
|
$basis .= '_within_live_tolerance';
|
|
}
|
|
|
|
return [
|
|
'replication_percent' => $percent,
|
|
'blockers' => $percent < 100.0 ? [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER] : [],
|
|
'basis' => $basis,
|
|
'stats' => $stats,
|
|
'bucket_count' => count($bucketProgress),
|
|
'live_tolerance' => [
|
|
'pending_bytes' => self::MINIO_CATCH_UP_PENDING_BYTES_TOLERANCE,
|
|
'pending_objects' => self::MINIO_CATCH_UP_PENDING_OBJECTS_TOLERANCE,
|
|
'within_tolerance' => $withinTolerance,
|
|
],
|
|
];
|
|
}
|
|
|
|
private function minioTargetFreeBytes(array $host): ?int
|
|
{
|
|
$configDir = $this->createMinioConfigDir();
|
|
try {
|
|
$this->prepareMinioAlias($configDir, 'target', $host);
|
|
$result = $this->runMinioClient($configDir, ['admin', 'info', '--json', 'target']);
|
|
$decoded = self::decodeMinioJsonOutput((string)$result['stdout']);
|
|
return self::minioAvailableBytesFromAdminInfo($decoded);
|
|
} catch (Throwable) {
|
|
return null;
|
|
} finally {
|
|
$this->removeDirectory($configDir);
|
|
}
|
|
}
|
|
|
|
private function prepareMinioAlias(string $configDir, string $alias, array $host): void
|
|
{
|
|
$credentials = $this->credentials($host);
|
|
$this->runMinioClient($configDir, [
|
|
'alias',
|
|
'set',
|
|
$alias,
|
|
self::minioEndpoint($host),
|
|
$credentials['username'],
|
|
$credentials['password'],
|
|
]);
|
|
}
|
|
|
|
private function runMinioClient(string $configDir, array $arguments): array
|
|
{
|
|
$binary = self::minioClientBinary();
|
|
if ($binary === null) {
|
|
throw new RuntimeException('MinIO Client (mc) is not available in the PHP runtime. Install mc, set MINIO_MC_BINARY, or enable MINIO_MC_AUTO_INSTALL.');
|
|
}
|
|
|
|
$command = array_merge([$binary, '--config-dir', $configDir], array_map('strval', $arguments));
|
|
$timeoutSeconds = self::minioClientCommandTimeoutSeconds();
|
|
$pipes = [];
|
|
$process = @proc_open($command, [
|
|
1 => ['pipe', 'w'],
|
|
2 => ['pipe', 'w'],
|
|
], $pipes);
|
|
if (!is_resource($process)) {
|
|
throw new RuntimeException('MinIO Client (mc) is not available.');
|
|
}
|
|
|
|
stream_set_blocking($pipes[1], false);
|
|
stream_set_blocking($pipes[2], false);
|
|
|
|
$stdout = '';
|
|
$stderr = '';
|
|
$exitCode = null;
|
|
$timedOut = false;
|
|
$deadline = microtime(true) + $timeoutSeconds;
|
|
|
|
while (true) {
|
|
$stdout .= (string)stream_get_contents($pipes[1]);
|
|
$stderr .= (string)stream_get_contents($pipes[2]);
|
|
|
|
$status = proc_get_status($process);
|
|
if (empty($status['running'])) {
|
|
$exitCode = (int)($status['exitcode'] ?? -1);
|
|
break;
|
|
}
|
|
|
|
if (microtime(true) >= $deadline) {
|
|
$timedOut = true;
|
|
proc_terminate($process);
|
|
usleep(100000);
|
|
$status = proc_get_status($process);
|
|
if (!empty($status['running'])) {
|
|
proc_terminate($process, 9);
|
|
}
|
|
break;
|
|
}
|
|
|
|
usleep(50000);
|
|
}
|
|
|
|
$stdout .= (string)stream_get_contents($pipes[1]);
|
|
fclose($pipes[1]);
|
|
$stderr .= (string)stream_get_contents($pipes[2]);
|
|
fclose($pipes[2]);
|
|
|
|
if ($timedOut) {
|
|
throw new RuntimeException(
|
|
'MinIO Client command timed out after ' . $timeoutSeconds . ' seconds: '
|
|
. self::minioClientCommandLabel($arguments)
|
|
);
|
|
}
|
|
|
|
$closeCode = proc_close($process);
|
|
if ($exitCode === null || $exitCode < 0) {
|
|
$exitCode = $closeCode;
|
|
}
|
|
|
|
if ($exitCode !== 0) {
|
|
$message = trim((string)$stderr) ?: trim((string)$stdout) ?: 'MinIO Client command failed.';
|
|
throw new RuntimeException($message);
|
|
}
|
|
|
|
return [
|
|
'stdout' => (string)$stdout,
|
|
'stderr' => (string)$stderr,
|
|
'exit_code' => $exitCode,
|
|
];
|
|
}
|
|
|
|
private static function minioClientCommandTimeoutSeconds(): int
|
|
{
|
|
$configured = getenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS');
|
|
if (is_numeric($configured) && (int)$configured > 0) {
|
|
return (int)$configured;
|
|
}
|
|
|
|
return self::MINIO_MC_COMMAND_TIMEOUT_SECONDS;
|
|
}
|
|
|
|
private static function minioClientCommandLabel(array $arguments): string
|
|
{
|
|
$parts = array_values(array_map('strval', $arguments));
|
|
if (($parts[0] ?? '') === 'alias' && ($parts[1] ?? '') === 'set') {
|
|
if (isset($parts[4])) {
|
|
$parts[4] = '[redacted]';
|
|
}
|
|
if (isset($parts[5])) {
|
|
$parts[5] = '[redacted]';
|
|
}
|
|
}
|
|
|
|
return 'mc ' . implode(' ', array_slice($parts, 0, 8));
|
|
}
|
|
|
|
private static function minioClientBinary(): ?string
|
|
{
|
|
$configured = trim((string)(getenv('MINIO_MC_BINARY') ?: ''));
|
|
if ($configured !== '') {
|
|
return $configured;
|
|
}
|
|
|
|
foreach (['/usr/local/bin/mc', '/usr/bin/mc'] as $candidate) {
|
|
if (is_file($candidate) && is_executable($candidate)) {
|
|
return $candidate;
|
|
}
|
|
}
|
|
|
|
return self::executableFromPath('mc') ?? self::cachedMinioClientBinary();
|
|
}
|
|
|
|
private static function cachedMinioClientBinary(): ?string
|
|
{
|
|
if (!self::minioClientAutoInstallEnabled()) {
|
|
return null;
|
|
}
|
|
|
|
$cacheDir = trim((string)(getenv('MINIO_MC_CACHE_DIR') ?: ''));
|
|
if ($cacheDir === '') {
|
|
$cacheDir = rtrim(sys_get_temp_dir(), DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . 'truckwash-minio-client';
|
|
}
|
|
$binary = rtrim($cacheDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . 'mc';
|
|
if (is_file($binary) && is_executable($binary)) {
|
|
return $binary;
|
|
}
|
|
|
|
if (!is_dir($cacheDir) && !mkdir($cacheDir, 0700, true) && !is_dir($cacheDir)) {
|
|
throw new RuntimeException('Could not create MinIO Client cache directory.');
|
|
}
|
|
|
|
$lock = @fopen($cacheDir . DIRECTORY_SEPARATOR . 'mc.lock', 'c');
|
|
if (is_resource($lock)) {
|
|
@flock($lock, LOCK_EX);
|
|
}
|
|
|
|
try {
|
|
if (is_file($binary) && is_executable($binary)) {
|
|
return $binary;
|
|
}
|
|
|
|
self::downloadMinioClientBinary($binary);
|
|
self::assertMinioClientUsable($binary);
|
|
} finally {
|
|
if (is_resource($lock)) {
|
|
@flock($lock, LOCK_UN);
|
|
fclose($lock);
|
|
}
|
|
}
|
|
|
|
return $binary;
|
|
}
|
|
|
|
private static function minioClientAutoInstallEnabled(): bool
|
|
{
|
|
$configured = getenv('MINIO_MC_AUTO_INSTALL');
|
|
$value = strtolower(trim((string)($configured === false ? '1' : $configured)));
|
|
return !in_array($value, ['0', 'false', 'no', 'off'], true);
|
|
}
|
|
|
|
private static function downloadMinioClientBinary(string $binary): void
|
|
{
|
|
$url = self::minioClientDownloadUrl();
|
|
$temp = $binary . '.download-' . getmypid();
|
|
$output = @fopen($temp, 'wb');
|
|
if (!is_resource($output)) {
|
|
throw new RuntimeException('Could not write MinIO Client download cache.');
|
|
}
|
|
|
|
$ok = false;
|
|
$error = '';
|
|
try {
|
|
if (function_exists('curl_init')) {
|
|
$curl = curl_init($url);
|
|
if ($curl === false) {
|
|
throw new RuntimeException('Could not initialize MinIO Client download.');
|
|
}
|
|
curl_setopt_array($curl, [
|
|
CURLOPT_FILE => $output,
|
|
CURLOPT_FOLLOWLOCATION => true,
|
|
CURLOPT_CONNECTTIMEOUT => min(2, self::MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS),
|
|
CURLOPT_TIMEOUT => self::minioClientDownloadTimeoutSeconds(),
|
|
CURLOPT_FAILONERROR => true,
|
|
CURLOPT_USERAGENT => 'truckwash-replication-manager/1.0',
|
|
]);
|
|
$ok = curl_exec($curl) === true;
|
|
$error = curl_error($curl);
|
|
$status = (int)curl_getinfo($curl, CURLINFO_RESPONSE_CODE);
|
|
curl_close($curl);
|
|
if (!$ok && $status > 0) {
|
|
$error = 'HTTP ' . $status;
|
|
}
|
|
} else {
|
|
$context = stream_context_create([
|
|
'http' => ['timeout' => self::minioClientDownloadTimeoutSeconds()],
|
|
'https' => ['timeout' => self::minioClientDownloadTimeoutSeconds()],
|
|
]);
|
|
$input = @fopen($url, 'rb', false, $context);
|
|
if (is_resource($input)) {
|
|
$ok = stream_copy_to_stream($input, $output) !== false;
|
|
fclose($input);
|
|
} else {
|
|
$error = 'download stream could not be opened';
|
|
}
|
|
}
|
|
} finally {
|
|
fclose($output);
|
|
}
|
|
|
|
if (!$ok || !is_file($temp) || (int)filesize($temp) <= 0) {
|
|
@unlink($temp);
|
|
throw new RuntimeException('Could not download MinIO Client (mc): ' . ($error !== '' ? $error : 'empty response'));
|
|
}
|
|
|
|
@chmod($temp, 0755);
|
|
if (!@rename($temp, $binary)) {
|
|
@unlink($temp);
|
|
throw new RuntimeException('Could not install downloaded MinIO Client (mc).');
|
|
}
|
|
@chmod($binary, 0755);
|
|
}
|
|
|
|
private static function minioClientDownloadTimeoutSeconds(): int
|
|
{
|
|
$configured = getenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS');
|
|
if (is_numeric($configured) && (int)$configured > 0) {
|
|
return (int)$configured;
|
|
}
|
|
|
|
return self::MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS;
|
|
}
|
|
|
|
private static function minioClientDownloadUrl(): string
|
|
{
|
|
$configured = trim((string)(getenv('MINIO_MC_DOWNLOAD_URL') ?: ''));
|
|
if ($configured !== '') {
|
|
return $configured;
|
|
}
|
|
|
|
$platform = self::minioClientDownloadPlatform();
|
|
if ($platform === null) {
|
|
throw new RuntimeException('Automatic MinIO Client download is not supported on this PHP runtime platform.');
|
|
}
|
|
|
|
return self::MINIO_MC_DOWNLOAD_BASE_URL . '/' . $platform . '/mc';
|
|
}
|
|
|
|
private static function minioClientDownloadPlatform(): ?string
|
|
{
|
|
if (PHP_OS_FAMILY !== 'Linux') {
|
|
return null;
|
|
}
|
|
|
|
$machine = strtolower((string)php_uname('m'));
|
|
return match ($machine) {
|
|
'x86_64', 'amd64' => 'linux-amd64',
|
|
'aarch64', 'arm64' => 'linux-arm64',
|
|
default => null,
|
|
};
|
|
}
|
|
|
|
private static function assertMinioClientUsable(string $binary): void
|
|
{
|
|
$result = self::runProcessWithTimeout([$binary, '--version'], self::MINIO_MC_COMMAND_TIMEOUT_SECONDS);
|
|
if (($result['exit_code'] ?? 1) !== 0) {
|
|
@unlink($binary);
|
|
$message = trim((string)($result['stderr'] ?? '')) ?: trim((string)($result['stdout'] ?? '')) ?: 'mc --version failed';
|
|
throw new RuntimeException('Downloaded MinIO Client (mc) failed verification: ' . $message);
|
|
}
|
|
}
|
|
|
|
private static function runProcessWithTimeout(array $command, int $timeoutSeconds): array
|
|
{
|
|
$pipes = [];
|
|
$process = @proc_open(array_map('strval', $command), [
|
|
1 => ['pipe', 'w'],
|
|
2 => ['pipe', 'w'],
|
|
], $pipes);
|
|
if (!is_resource($process)) {
|
|
return [
|
|
'stdout' => '',
|
|
'stderr' => 'Process could not be started.',
|
|
'exit_code' => 127,
|
|
'timed_out' => false,
|
|
];
|
|
}
|
|
|
|
stream_set_blocking($pipes[1], false);
|
|
stream_set_blocking($pipes[2], false);
|
|
|
|
$stdout = '';
|
|
$stderr = '';
|
|
$exitCode = null;
|
|
$timedOut = false;
|
|
$deadline = microtime(true) + max(1, $timeoutSeconds);
|
|
|
|
while (true) {
|
|
$stdout .= (string)stream_get_contents($pipes[1]);
|
|
$stderr .= (string)stream_get_contents($pipes[2]);
|
|
|
|
$status = proc_get_status($process);
|
|
if (empty($status['running'])) {
|
|
$exitCode = (int)($status['exitcode'] ?? -1);
|
|
break;
|
|
}
|
|
|
|
if (microtime(true) >= $deadline) {
|
|
$timedOut = true;
|
|
proc_terminate($process);
|
|
usleep(100000);
|
|
$status = proc_get_status($process);
|
|
if (!empty($status['running'])) {
|
|
proc_terminate($process, 9);
|
|
}
|
|
break;
|
|
}
|
|
|
|
usleep(50000);
|
|
}
|
|
|
|
$stdout .= (string)stream_get_contents($pipes[1]);
|
|
fclose($pipes[1]);
|
|
$stderr .= (string)stream_get_contents($pipes[2]);
|
|
fclose($pipes[2]);
|
|
|
|
if (!$timedOut) {
|
|
$closeCode = proc_close($process);
|
|
if ($exitCode === null || $exitCode < 0) {
|
|
$exitCode = $closeCode;
|
|
}
|
|
}
|
|
|
|
return [
|
|
'stdout' => $stdout,
|
|
'stderr' => $stderr,
|
|
'exit_code' => $timedOut ? 124 : (int)$exitCode,
|
|
'timed_out' => $timedOut,
|
|
];
|
|
}
|
|
|
|
private static function executableFromPath(string $name): ?string
|
|
{
|
|
$path = (string)(getenv('PATH') ?: '');
|
|
if ($path === '') {
|
|
return null;
|
|
}
|
|
|
|
foreach (explode(PATH_SEPARATOR, $path) as $dir) {
|
|
$dir = rtrim((string)$dir, DIRECTORY_SEPARATOR);
|
|
if ($dir === '') {
|
|
continue;
|
|
}
|
|
$candidate = $dir . DIRECTORY_SEPARATOR . $name;
|
|
if (is_file($candidate) && is_executable($candidate)) {
|
|
return $candidate;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private function createMinioConfigDir(): string
|
|
{
|
|
$dir = rtrim(sys_get_temp_dir(), DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . 'truckwash-mc-' . bin2hex(random_bytes(6));
|
|
if (!mkdir($dir, 0700, true) && !is_dir($dir)) {
|
|
throw new RuntimeException('Could not create MinIO Client config directory.');
|
|
}
|
|
return $dir;
|
|
}
|
|
|
|
private function removeDirectory(string $dir): void
|
|
{
|
|
if (!is_dir($dir)) {
|
|
return;
|
|
}
|
|
$entries = scandir($dir);
|
|
if (!is_array($entries)) {
|
|
@rmdir($dir);
|
|
return;
|
|
}
|
|
foreach ($entries as $entry) {
|
|
if ($entry === '.' || $entry === '..') {
|
|
continue;
|
|
}
|
|
$path = $dir . DIRECTORY_SEPARATOR . $entry;
|
|
if (is_dir($path)) {
|
|
$this->removeDirectory($path);
|
|
} else {
|
|
@unlink($path);
|
|
}
|
|
}
|
|
@rmdir($dir);
|
|
}
|
|
|
|
private static function decodeMinioJsonOutput(string $output): mixed
|
|
{
|
|
$trimmed = trim($output);
|
|
if ($trimmed === '') {
|
|
return null;
|
|
}
|
|
|
|
$decoded = json_decode($trimmed, true);
|
|
if (json_last_error() === JSON_ERROR_NONE) {
|
|
return $decoded;
|
|
}
|
|
|
|
$items = [];
|
|
foreach (preg_split('/\R/', $trimmed) ?: [] as $line) {
|
|
$line = trim($line);
|
|
if ($line === '') {
|
|
continue;
|
|
}
|
|
$decodedLine = json_decode($line, true);
|
|
if (json_last_error() === JSON_ERROR_NONE) {
|
|
$items[] = $decodedLine;
|
|
}
|
|
}
|
|
|
|
return $items !== [] ? $items : null;
|
|
}
|
|
|
|
public static function minioAvailableBytesFromAdminInfo(mixed $value): ?int
|
|
{
|
|
$values = [];
|
|
self::collectMinioAvailableByteValues($value, $values);
|
|
if ($values === []) {
|
|
return null;
|
|
}
|
|
|
|
return max($values);
|
|
}
|
|
|
|
private static function collectMinioAvailableByteValues(mixed $value, array &$values): void
|
|
{
|
|
if (!is_array($value)) {
|
|
return;
|
|
}
|
|
|
|
foreach ($value as $key => $entry) {
|
|
$normalizedKey = strtolower((string)$key);
|
|
if (in_array($normalizedKey, ['available', 'availablebytes', 'available_bytes', 'avail', 'availspace', 'avail_space', 'availablespace', 'available_space', 'free', 'freebytes', 'free_bytes', 'freespace', 'free_space'], true)) {
|
|
$bytes = self::parseMinioByteValue($entry);
|
|
if ($bytes !== null && $bytes >= 0) {
|
|
$values[] = $bytes;
|
|
}
|
|
}
|
|
if (is_array($entry)) {
|
|
self::collectMinioAvailableByteValues($entry, $values);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static function parseMinioByteValue(mixed $value): ?int
|
|
{
|
|
if (is_int($value)) {
|
|
return $value;
|
|
}
|
|
if (is_float($value)) {
|
|
return (int)$value;
|
|
}
|
|
|
|
$text = trim((string)$value);
|
|
if ($text === '') {
|
|
return null;
|
|
}
|
|
if (ctype_digit($text)) {
|
|
return (int)$text;
|
|
}
|
|
if (preg_match('/^([0-9]+(?:\.[0-9]+)?)\s*([kmgtp]?i?b?|bytes?)$/i', $text, $matches) !== 1) {
|
|
return null;
|
|
}
|
|
|
|
$number = (float)$matches[1];
|
|
$unit = strtolower($matches[2]);
|
|
$multipliers = [
|
|
'b' => 1,
|
|
'byte' => 1,
|
|
'bytes' => 1,
|
|
'kb' => 1000,
|
|
'kib' => 1024,
|
|
'mb' => 1000 ** 2,
|
|
'mib' => 1024 ** 2,
|
|
'gb' => 1000 ** 3,
|
|
'gib' => 1024 ** 3,
|
|
'tb' => 1000 ** 4,
|
|
'tib' => 1024 ** 4,
|
|
'pb' => 1000 ** 5,
|
|
'pib' => 1024 ** 5,
|
|
];
|
|
|
|
return (int)floor($number * ($multipliers[$unit] ?? 1));
|
|
}
|
|
|
|
private function redisClient(array $host): PredisClient
|
|
{
|
|
$credentials = $this->credentials($host);
|
|
$params = [
|
|
'scheme' => 'tcp',
|
|
'host' => (string)$host['host'],
|
|
'port' => (int)$host['port'],
|
|
'database' => (int)($host['database_index'] ?? 0),
|
|
'password' => $credentials['password'],
|
|
];
|
|
|
|
if (($credentials['username'] ?? '') !== '' && $credentials['username'] !== 'default') {
|
|
$params['username'] = $credentials['username'];
|
|
}
|
|
|
|
return new PredisClient($params);
|
|
}
|
|
|
|
private function redisInfo(PredisClient $client): array
|
|
{
|
|
$info = $client->info('replication');
|
|
if (is_array($info)) {
|
|
return isset($info['Replication']) && is_array($info['Replication'])
|
|
? $info['Replication']
|
|
: $info;
|
|
}
|
|
|
|
$parsed = [];
|
|
foreach (explode("\n", (string)$info) as $line) {
|
|
$line = trim($line);
|
|
if ($line === '' || str_starts_with($line, '#') || !str_contains($line, ':')) {
|
|
continue;
|
|
}
|
|
[$key, $value] = explode(':', $line, 2);
|
|
$parsed[$key] = trim($value);
|
|
}
|
|
return $parsed;
|
|
}
|
|
|
|
private function mysqliExec(mysqli $connection, string $sql): void
|
|
{
|
|
$connection->query($sql);
|
|
}
|
|
|
|
private function mysqliSelectOne(mysqli $connection, string $sql): array
|
|
{
|
|
$result = $connection->query($sql);
|
|
if ($result === false) {
|
|
return [];
|
|
}
|
|
$row = $result->fetch_assoc();
|
|
return is_array($row) ? $row : [];
|
|
}
|
|
|
|
private function refreshStatuses(): void
|
|
{
|
|
foreach ($this->listHosts() as $host) {
|
|
$activeOperation = $this->activeOperation((string)$host['kind'], (int)$host['id']);
|
|
if ($activeOperation !== null) {
|
|
if ($this->shouldAdvanceActiveProvisionDuringRefresh($host, $activeOperation)) {
|
|
try {
|
|
$this->provisionHost((string)$host['kind'], (int)$host['id']);
|
|
} catch (Throwable $throwable) {
|
|
$this->storeStatus($host, [
|
|
'status' => 'degraded',
|
|
'replication_percent' => self::lastStatusReplicationPercent($host, 0.0),
|
|
'lag_seconds' => null,
|
|
'blockers' => [$throwable->getMessage()],
|
|
'raw' => ['active_operation_refresh_failed' => true],
|
|
'checked_at' => date('c'),
|
|
]);
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
$status = match ((string)$host['kind']) {
|
|
self::KIND_DATABASE => $this->testDatabaseHost($host),
|
|
self::KIND_REDIS => $this->testRedisHost($host),
|
|
self::KIND_MINIO => $this->testMinioHost($host),
|
|
default => throw new RuntimeException('Unsupported replication kind.'),
|
|
};
|
|
$this->storeStatus($host, $status);
|
|
} catch (Throwable $throwable) {
|
|
$this->storeStatus($host, [
|
|
'status' => 'degraded',
|
|
'replication_percent' => 0,
|
|
'lag_seconds' => null,
|
|
'blockers' => [$throwable->getMessage()],
|
|
'raw' => [],
|
|
'checked_at' => date('c'),
|
|
]);
|
|
}
|
|
}
|
|
|
|
$this->writeBootstrapSnapshot();
|
|
}
|
|
|
|
private function shouldAdvanceActiveProvisionDuringRefresh(array $host, array $activeOperation): bool
|
|
{
|
|
return (string)($host['kind'] ?? '') === self::KIND_MINIO
|
|
&& (string)($host['role'] ?? '') !== 'primary'
|
|
&& (string)($activeOperation['operation'] ?? '') === 'provision';
|
|
}
|
|
|
|
private function storeStatus(array $host, array $status): void
|
|
{
|
|
$publicStatus = [
|
|
'status' => (string)($status['status'] ?? 'unknown'),
|
|
'replication_percent' => round((float)($status['replication_percent'] ?? 0), 2),
|
|
'lag_seconds' => $status['lag_seconds'] ?? null,
|
|
'blockers' => array_values(array_filter($status['blockers'] ?? [])),
|
|
'raw' => $status['raw'] ?? [],
|
|
'checked_at' => (string)($status['checked_at'] ?? date('c')),
|
|
];
|
|
|
|
$this->execute(
|
|
"UPDATE replication_hosts SET status = ?, last_status_json = ?, last_checked_at = NOW() WHERE id = ?",
|
|
'ssi',
|
|
[$publicStatus['status'], self::jsonEncode($publicStatus), (int)$host['id']]
|
|
);
|
|
$this->execute(
|
|
"INSERT INTO replication_status_snapshots
|
|
(host_id, kind, status, replication_percent, lag_seconds, blockers_json, raw_status_json)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
'issdiss',
|
|
[
|
|
(int)$host['id'],
|
|
(string)$host['kind'],
|
|
$publicStatus['status'],
|
|
(float)$publicStatus['replication_percent'],
|
|
$publicStatus['lag_seconds'],
|
|
self::jsonEncode($publicStatus['blockers']),
|
|
self::jsonEncode($publicStatus['raw']),
|
|
]
|
|
);
|
|
|
|
$this->completeReadyMinioProvisionOperation($host, $publicStatus);
|
|
|
|
if (class_exists(coolify_manager::class)) {
|
|
coolify_manager::syncDeploymentStateForReplicationHost((int)$host['id']);
|
|
}
|
|
}
|
|
|
|
private function completeReadyMinioProvisionOperation(array $host, array $status): void
|
|
{
|
|
if ((string)($host['kind'] ?? '') !== self::KIND_MINIO || (string)($host['role'] ?? '') === 'primary') {
|
|
return;
|
|
}
|
|
|
|
$blockers = array_values(array_filter($status['blockers'] ?? []));
|
|
if ((string)($status['status'] ?? '') !== 'ok'
|
|
|| round((float)($status['replication_percent'] ?? 0), 2) < 100.0
|
|
|| $blockers !== []) {
|
|
return;
|
|
}
|
|
|
|
$operationId = $this->activeOperationId(self::KIND_MINIO, (int)$host['id'], 'provision');
|
|
if ($operationId !== null) {
|
|
$this->finishOperation($operationId, 'completed', 100.0, 'MinIO replication target is caught up.', []);
|
|
}
|
|
}
|
|
|
|
private function buildReplicationSummary(string $kind, array $hosts): array
|
|
{
|
|
$replicas = [];
|
|
$blockers = [];
|
|
$percents = [];
|
|
$statuses = [];
|
|
|
|
foreach ($hosts as $host) {
|
|
if (($host['role'] ?? '') === 'primary') {
|
|
continue;
|
|
}
|
|
|
|
$public = $this->publicHost($host);
|
|
$replicas[] = $public;
|
|
$status = is_array($public['last_status'] ?? null) ? $public['last_status'] : [];
|
|
$activeProgress = $public['active_operation']['progress_percent'] ?? null;
|
|
$percent = is_numeric($activeProgress) && (float)$activeProgress > 0
|
|
? (float)$activeProgress
|
|
: (float)($status['replication_percent'] ?? $public['replication_percent'] ?? 0);
|
|
$percents[] = $percent;
|
|
$statuses[] = (string)($status['status'] ?? $public['status'] ?? 'unknown');
|
|
foreach ($status['blockers'] ?? [] as $blocker) {
|
|
$blockers[] = (string)$blocker;
|
|
}
|
|
}
|
|
|
|
if ($replicas === []) {
|
|
return [
|
|
'status' => 'not_configured',
|
|
'min_percent' => 0.0,
|
|
'average_percent' => 0.0,
|
|
'replicas' => [],
|
|
'blockers' => ['No ' . $kind . ' replicas configured.'],
|
|
];
|
|
}
|
|
|
|
$min = min($percents);
|
|
$average = array_sum($percents) / max(1, count($percents));
|
|
$status = ($min >= 100.0 && $blockers === []) ? 'ok' : 'degraded';
|
|
if (in_array('down', $statuses, true)) {
|
|
$status = 'down';
|
|
}
|
|
|
|
return [
|
|
'status' => $status,
|
|
'min_percent' => round($min, 2),
|
|
'average_percent' => round($average, 2),
|
|
'replicas' => $replicas,
|
|
'blockers' => array_values(array_unique($blockers)),
|
|
];
|
|
}
|
|
|
|
private function publicHost(?array $host): ?array
|
|
{
|
|
if ($host === null) {
|
|
return null;
|
|
}
|
|
|
|
$lastStatus = self::jsonDecode($host['last_status_json'] ?? null);
|
|
$credentials = $this->credentials($host);
|
|
$options = $this->decodeOptions($host);
|
|
$hasCoolifyDeployment = isset($options['coolify_target_id'])
|
|
|| isset($options['coolify_instance_id'])
|
|
|| (string)($options['deployment_provider'] ?? '') === 'coolify';
|
|
$coolifyDeployment = $hasCoolifyDeployment && isset($host['id'])
|
|
? coolify_manager::deploymentMetadataForReplicationHost((int)$host['id'])
|
|
: null;
|
|
$activeOperation = isset($host['id'])
|
|
? $this->activeOperation((string)$host['kind'], (int)$host['id'])
|
|
: null;
|
|
$replicationPercent = round((float)($lastStatus['replication_percent'] ?? (($host['role'] ?? '') === 'primary' ? 100 : 0)), 2);
|
|
$status = self::publicReplicationStatus($host, $lastStatus, $activeOperation, $replicationPercent);
|
|
$database = match ((string)$host['kind']) {
|
|
self::KIND_DATABASE => (string)($host['database_name'] ?? ''),
|
|
self::KIND_REDIS => (int)($host['database_index'] ?? 0),
|
|
self::KIND_MINIO => null,
|
|
default => null,
|
|
};
|
|
|
|
return [
|
|
'id' => (int)$host['id'],
|
|
'kind' => (string)$host['kind'],
|
|
'label' => (string)$host['label'],
|
|
'host' => (string)$host['host'],
|
|
'port' => (int)$host['port'],
|
|
'database' => $database,
|
|
'endpoint' => (string)($options['endpoint'] ?? (((string)$host['kind'] === self::KIND_MINIO) ? self::minioEndpoint($host) : '')),
|
|
'scheme' => $options['scheme'] ?? null,
|
|
'buckets' => ((string)$host['kind'] === self::KIND_MINIO) ? self::normalizeMinioBuckets($options['buckets'] ?? []) : [],
|
|
'console_port' => ((string)$host['kind'] === self::KIND_MINIO) ? (int)($options['console_port'] ?? 9001) : null,
|
|
'replication_transfer_limit' => ((string)$host['kind'] === self::KIND_MINIO) ? self::minioReplicationTransferLimitFromOptions($options) : null,
|
|
'space_headroom_percent' => ((string)$host['kind'] === self::KIND_MINIO) ? (float)($options['space_headroom_percent'] ?? self::MINIO_SPACE_HEADROOM_PERCENT) : null,
|
|
'role' => (string)$host['role'],
|
|
'status' => $status,
|
|
'replication_source_id' => isset($host['replication_source_id']) ? (int)$host['replication_source_id'] : null,
|
|
'ssl_mode' => $host['ssl_mode'] ?? null,
|
|
'replication_percent' => $replicationPercent,
|
|
'last_status' => array_replace($lastStatus, ['status' => $status]),
|
|
'active_operation' => $activeOperation,
|
|
'deployment_provider' => (string)($options['deployment_provider'] ?? ($coolifyDeployment !== null ? 'coolify' : 'manual')),
|
|
'coolify' => $coolifyDeployment,
|
|
'availability_state' => $coolifyDeployment['availability_state'] ?? null,
|
|
'last_checked_at' => $host['last_checked_at'] ?? null,
|
|
'credential_summary' => [
|
|
'username' => $credentials['username'] !== '' ? replication_secret_box::mask($credentials['username']) : '',
|
|
'password_set' => $credentials['password'] !== '',
|
|
'admin_username' => $credentials['admin_username'] !== '' ? replication_secret_box::mask($credentials['admin_username']) : '',
|
|
'admin_password_set' => $credentials['admin_password'] !== '',
|
|
'replication_username' => $credentials['replication_username'] !== '' ? replication_secret_box::mask($credentials['replication_username']) : '',
|
|
'replication_password_set' => $credentials['replication_password'] !== '',
|
|
],
|
|
'created_at' => $host['created_at'] ?? null,
|
|
'updated_at' => $host['updated_at'] ?? null,
|
|
'deleted_at' => $host['deleted_at'] ?? null,
|
|
];
|
|
}
|
|
|
|
private static function publicReplicationStatus(array $host, array $lastStatus, ?array $activeOperation, float $replicationPercent): string
|
|
{
|
|
$hostStatus = (string)($host['status'] ?? 'unknown');
|
|
$status = (string)($lastStatus['status'] ?? $hostStatus);
|
|
|
|
if (in_array($hostStatus, ['removed', 'inactive', 'not_configured', 'down'], true)) {
|
|
return $hostStatus;
|
|
}
|
|
if (in_array($status, ['removed', 'inactive', 'not_configured', 'down'], true)) {
|
|
return $status;
|
|
}
|
|
if ($activeOperation !== null || $hostStatus === 'provisioning' || $status === 'provisioning') {
|
|
return 'provisioning';
|
|
}
|
|
if ($status === 'ok') {
|
|
return self::replicationHealthStatus(
|
|
true,
|
|
(string)($host['role'] ?? ''),
|
|
$replicationPercent,
|
|
array_values(array_filter($lastStatus['blockers'] ?? []))
|
|
);
|
|
}
|
|
|
|
return $status !== '' ? $status : 'unknown';
|
|
}
|
|
|
|
private static function sanitizePublicLastStatus(array $host, array $lastStatus): array
|
|
{
|
|
if ((string)($host['kind'] ?? '') !== self::KIND_MINIO || (string)($host['role'] ?? '') !== 'primary') {
|
|
return $lastStatus;
|
|
}
|
|
|
|
$blockers = array_values(array_filter(array_map(
|
|
static fn(mixed $blocker): string => trim((string)$blocker),
|
|
$lastStatus['blockers'] ?? []
|
|
)));
|
|
if ($blockers === []) {
|
|
return $lastStatus;
|
|
}
|
|
|
|
$onlyObjectScanTimeouts = true;
|
|
foreach ($blockers as $blocker) {
|
|
$normalized = strtolower($blocker);
|
|
if (!str_contains($normalized, 'could not be inspected')
|
|
|| !str_contains($normalized, 'listobjectsv2')
|
|
|| !str_contains($normalized, 'timed out')) {
|
|
$onlyObjectScanTimeouts = false;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!$onlyObjectScanTimeouts || round((float)($lastStatus['replication_percent'] ?? 0), 2) < 100.0) {
|
|
return $lastStatus;
|
|
}
|
|
|
|
$lastStatus['status'] = 'ok';
|
|
$lastStatus['blockers'] = [];
|
|
$lastStatus['raw']['suppressed_blockers'] = $blockers;
|
|
$lastStatus['raw']['suppressed_reason'] = 'MinIO primary object-scan timeouts do not indicate primary availability failure.';
|
|
return $lastStatus;
|
|
}
|
|
|
|
private static function normalizeMinioAddress(string $host, mixed $port, mixed $scheme): array
|
|
{
|
|
$raw = trim($host);
|
|
$hasScheme = preg_match('/^https?:\/\//i', $raw) === 1;
|
|
$parsed = parse_url($hasScheme ? $raw : 'http://' . $raw);
|
|
if (!is_array($parsed) || empty($parsed['host'])) {
|
|
throw new RuntimeException('MinIO endpoint host is invalid.');
|
|
}
|
|
|
|
$normalizedScheme = strtolower(trim((string)($scheme ?: ($parsed['scheme'] ?? 'http'))));
|
|
if (!in_array($normalizedScheme, ['http', 'https'], true)) {
|
|
throw new RuntimeException('MinIO scheme must be http or https.');
|
|
}
|
|
|
|
$normalizedHost = trim((string)$parsed['host']);
|
|
$portValue = ($port !== null && trim((string)$port) !== '')
|
|
? $port
|
|
: ($parsed['port'] ?? ($normalizedScheme === 'https' ? 443 : 9000));
|
|
$normalizedPort = (int)$portValue;
|
|
|
|
return [$normalizedHost, $normalizedPort, $normalizedScheme];
|
|
}
|
|
|
|
private static function minioEndpointFromParts(string $scheme, string $host, int $port): string
|
|
{
|
|
return strtolower($scheme) . '://' . $host . ':' . $port;
|
|
}
|
|
|
|
private static function minioEndpoint(array $host): string
|
|
{
|
|
$options = isset($host['options']) && is_array($host['options'])
|
|
? $host['options']
|
|
: self::jsonDecode($host['options_json'] ?? null);
|
|
$endpoint = trim((string)($options['endpoint'] ?? ''));
|
|
if ($endpoint !== '') {
|
|
return $endpoint;
|
|
}
|
|
|
|
return self::minioEndpointFromParts((string)($options['scheme'] ?? 'http'), (string)$host['host'], (int)$host['port']);
|
|
}
|
|
|
|
private function normalizeHostInput(string $kind, array $input): array
|
|
{
|
|
$host = trim((string)($input['host'] ?? $input['endpoint'] ?? ''));
|
|
if ($host === '') {
|
|
throw new RuntimeException('Host is required.');
|
|
}
|
|
|
|
$scheme = null;
|
|
$defaultPort = match ($kind) {
|
|
self::KIND_DATABASE => 3306,
|
|
self::KIND_REDIS => 6379,
|
|
self::KIND_MINIO => 9000,
|
|
};
|
|
if ($kind === self::KIND_MINIO) {
|
|
[$host, $port, $scheme] = self::normalizeMinioAddress($host, $input['port'] ?? null, $input['scheme'] ?? null);
|
|
} else {
|
|
$port = (int)($input['port'] ?? $defaultPort);
|
|
}
|
|
if ($port <= 0 || $port > 65535) {
|
|
throw new RuntimeException('Port must be between 1 and 65535.');
|
|
}
|
|
|
|
$label = trim((string)($input['label'] ?? ''));
|
|
if ($label === '') {
|
|
$label = $host . ':' . $port;
|
|
}
|
|
|
|
$username = trim((string)($input['username'] ?? $input['access_key'] ?? $input['user'] ?? ''));
|
|
$password = (string)($input['password'] ?? $input['secret_key'] ?? '');
|
|
$databaseName = null;
|
|
$databaseIndex = null;
|
|
if ($kind === self::KIND_DATABASE) {
|
|
$databaseName = trim((string)($input['database'] ?? $input['database_name'] ?? ''));
|
|
if ($databaseName === '' || $username === '') {
|
|
throw new RuntimeException('Database name and username are required for database replication hosts.');
|
|
}
|
|
} elseif ($kind === self::KIND_REDIS) {
|
|
$databaseIndex = (int)($input['database'] ?? $input['database_index'] ?? 0);
|
|
if ($databaseIndex < 0) {
|
|
throw new RuntimeException('Redis database index must be zero or greater.');
|
|
}
|
|
} else {
|
|
if ($username === '' || $password === '') {
|
|
throw new RuntimeException('Access key and secret key are required for MinIO replication hosts.');
|
|
}
|
|
}
|
|
|
|
$options = is_array($input['options'] ?? null) ? $input['options'] : [];
|
|
if (isset($input['deployment_provider'])) {
|
|
$provider = strtolower(trim((string)$input['deployment_provider']));
|
|
if (!in_array($provider, ['manual', 'coolify'], true)) {
|
|
throw new RuntimeException('Deployment provider must be manual or coolify.');
|
|
}
|
|
$options['deployment_provider'] = $provider;
|
|
}
|
|
if (isset($input['coolify_target_id'])) {
|
|
$options['coolify_target_id'] = (int)$input['coolify_target_id'];
|
|
}
|
|
if (isset($input['coolify_instance_id'])) {
|
|
$options['coolify_instance_id'] = (int)$input['coolify_instance_id'];
|
|
}
|
|
if ($kind === self::KIND_MINIO) {
|
|
$headroom = (float)($input['space_headroom_percent'] ?? $options['space_headroom_percent'] ?? self::MINIO_SPACE_HEADROOM_PERCENT);
|
|
$transferLimit = array_key_exists('replication_transfer_limit', $input)
|
|
? self::normalizeMinioTransferLimit($input['replication_transfer_limit'], false)
|
|
: self::minioReplicationTransferLimitFromOptions($options);
|
|
$options = array_replace($options, [
|
|
'scheme' => $scheme ?: 'http',
|
|
'endpoint' => self::minioEndpointFromParts($scheme ?: 'http', $host, $port),
|
|
'buckets' => self::normalizeMinioBuckets($input['buckets'] ?? $options['buckets'] ?? self::MINIO_DEFAULT_BUCKETS),
|
|
'console_port' => (int)($input['console_port'] ?? $options['console_port'] ?? 9001),
|
|
'replication_transfer_limit' => $transferLimit,
|
|
'space_headroom_percent' => max(0.0, $headroom),
|
|
]);
|
|
}
|
|
|
|
return [
|
|
'label' => $label,
|
|
'host' => $host,
|
|
'port' => $port,
|
|
'database_name' => $databaseName,
|
|
'database_index' => $databaseIndex,
|
|
'username' => $username,
|
|
'password_secret' => replication_secret_box::encrypt($password),
|
|
'admin_username' => trim((string)($input['admin_username'] ?? '')),
|
|
'admin_password_secret' => replication_secret_box::encrypt((string)($input['admin_password'] ?? '')),
|
|
'replication_username' => trim((string)($input['replication_username'] ?? '')),
|
|
'replication_password_secret' => replication_secret_box::encrypt((string)($input['replication_password'] ?? '')),
|
|
'ssl_mode' => strtoupper(trim((string)($input['ssl_mode'] ?? 'DISABLED'))) ?: 'DISABLED',
|
|
'options' => $options,
|
|
];
|
|
}
|
|
|
|
private function transientHost(string $kind, array $input): array
|
|
{
|
|
$normalized = $this->normalizeHostInput($kind, $input);
|
|
$role = strtolower(trim((string)($input['role'] ?? 'replica')));
|
|
if (!in_array($role, ['primary', 'replica'], true)) {
|
|
$role = 'replica';
|
|
}
|
|
|
|
return array_merge($normalized, [
|
|
'id' => 0,
|
|
'kind' => $kind,
|
|
'role' => $role,
|
|
'status' => 'unknown',
|
|
'replication_source_id' => null,
|
|
'last_status_json' => null,
|
|
'last_checked_at' => null,
|
|
'created_at' => null,
|
|
'updated_at' => null,
|
|
'deleted_at' => null,
|
|
'test_connectivity_only' => true,
|
|
]);
|
|
}
|
|
|
|
private function ensureEnvironmentPrimaryRows(): void
|
|
{
|
|
if ($this->primaryHost(self::KIND_DATABASE) === null && isset($GLOBALS['CONFIG_DB']) && is_array($GLOBALS['CONFIG_DB'])) {
|
|
$config = $GLOBALS['CONFIG_DB'];
|
|
if (!empty($config['host']) && !empty($config['database']) && !empty($config['user'])) {
|
|
$this->insertEnvironmentPrimary(self::KIND_DATABASE, [
|
|
'label' => 'Current database primary',
|
|
'host' => (string)$config['host'],
|
|
'port' => (int)($config['port'] ?? 3306),
|
|
'database_name' => (string)$config['database'],
|
|
'database_index' => null,
|
|
'username' => (string)$config['user'],
|
|
'password_secret' => replication_secret_box::encrypt((string)($config['password'] ?? '')),
|
|
'ssl_mode' => (string)($config['ssl_mode'] ?? 'DISABLED'),
|
|
]);
|
|
}
|
|
}
|
|
|
|
if ($this->primaryHost(self::KIND_REDIS) === null && isset($GLOBALS['REDIS_CONFIG']) && is_array($GLOBALS['REDIS_CONFIG'])) {
|
|
$config = $GLOBALS['REDIS_CONFIG'];
|
|
if (!empty($config['host'])) {
|
|
$this->insertEnvironmentPrimary(self::KIND_REDIS, [
|
|
'label' => 'Current Redis primary',
|
|
'host' => (string)$config['host'],
|
|
'port' => (int)($config['port'] ?? 6379),
|
|
'database_name' => null,
|
|
'database_index' => (int)($config['database'] ?? 0),
|
|
'username' => (string)($config['user'] ?? ''),
|
|
'password_secret' => replication_secret_box::encrypt((string)($config['password'] ?? '')),
|
|
'ssl_mode' => null,
|
|
]);
|
|
}
|
|
}
|
|
|
|
if ($this->primaryHost(self::KIND_MINIO) === null && isset($GLOBALS['MINIO']) && is_array($GLOBALS['MINIO'])) {
|
|
$config = $GLOBALS['MINIO'];
|
|
$endpoint = trim((string)($config['endpoint'] ?? ''));
|
|
$accessKey = trim((string)($config['access_key'] ?? ''));
|
|
if ($endpoint !== '' && $accessKey !== '') {
|
|
[$host, $port, $scheme] = self::normalizeMinioAddress($endpoint, null, null);
|
|
$buckets = self::normalizeMinioBuckets($config['buckets'] ?? self::MINIO_DEFAULT_BUCKETS);
|
|
$this->insertEnvironmentPrimary(self::KIND_MINIO, [
|
|
'label' => 'Current MinIO primary',
|
|
'host' => $host,
|
|
'port' => $port,
|
|
'database_name' => null,
|
|
'database_index' => null,
|
|
'username' => $accessKey,
|
|
'password_secret' => replication_secret_box::encrypt((string)($config['secret_key'] ?? '')),
|
|
'ssl_mode' => null,
|
|
'options' => [
|
|
'source' => 'environment',
|
|
'scheme' => $scheme,
|
|
'endpoint' => self::minioEndpointFromParts($scheme, $host, $port),
|
|
'buckets' => $buckets,
|
|
'console_port' => (int)($config['console_port'] ?? 9001),
|
|
'space_headroom_percent' => self::MINIO_SPACE_HEADROOM_PERCENT,
|
|
],
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
|
|
private function insertEnvironmentPrimary(string $kind, array $host): void
|
|
{
|
|
$this->execute(
|
|
"INSERT INTO replication_hosts (
|
|
kind, label, host, port, database_name, database_index, username, password_secret,
|
|
role, status, ssl_mode, options_json
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'primary', 'unknown', ?, ?)",
|
|
'sssissssss',
|
|
[
|
|
$kind,
|
|
$host['label'],
|
|
$host['host'],
|
|
$host['port'],
|
|
$host['database_name'],
|
|
$host['database_index'],
|
|
$host['username'],
|
|
$host['password_secret'],
|
|
$host['ssl_mode'],
|
|
self::jsonEncode(array_replace(['source' => 'environment'], is_array($host['options'] ?? null) ? $host['options'] : [])),
|
|
]
|
|
);
|
|
}
|
|
|
|
private function writeBootstrapSnapshot(): void
|
|
{
|
|
$databasePrimary = $this->primaryHost(self::KIND_DATABASE);
|
|
$redisPrimary = $this->primaryHost(self::KIND_REDIS);
|
|
$minioPrimary = $this->primaryHost(self::KIND_MINIO);
|
|
$active = [];
|
|
|
|
if ($databasePrimary !== null) {
|
|
$credentials = $this->credentials($databasePrimary);
|
|
$active['database'] = [
|
|
'id' => (int)$databasePrimary['id'],
|
|
'host' => (string)$databasePrimary['host'],
|
|
'port' => (int)$databasePrimary['port'],
|
|
'database' => (string)$databasePrimary['database_name'],
|
|
'user' => $credentials['username'],
|
|
'password_secret' => $databasePrimary['password_secret'] ?? '',
|
|
'ssl_mode' => (string)($databasePrimary['ssl_mode'] ?? 'DISABLED'),
|
|
];
|
|
}
|
|
|
|
if ($redisPrimary !== null) {
|
|
$credentials = $this->credentials($redisPrimary);
|
|
$active['redis'] = [
|
|
'id' => (int)$redisPrimary['id'],
|
|
'host' => (string)$redisPrimary['host'],
|
|
'port' => (int)$redisPrimary['port'],
|
|
'database' => (int)($redisPrimary['database_index'] ?? 0),
|
|
'user' => $credentials['username'],
|
|
'password_secret' => $redisPrimary['password_secret'] ?? '',
|
|
];
|
|
}
|
|
|
|
if ($minioPrimary !== null) {
|
|
$credentials = $this->credentials($minioPrimary);
|
|
$options = $this->decodeOptions($minioPrimary);
|
|
$active['minio'] = [
|
|
'id' => (int)$minioPrimary['id'],
|
|
'endpoint' => self::minioEndpoint($minioPrimary),
|
|
'access_key' => $credentials['username'],
|
|
'secret_key_secret' => $minioPrimary['password_secret'] ?? '',
|
|
'buckets' => self::normalizeMinioBuckets($options['buckets'] ?? self::MINIO_DEFAULT_BUCKETS),
|
|
];
|
|
}
|
|
|
|
replication_bootstrap_config::writeSnapshot([
|
|
'version' => 1,
|
|
'generated_at' => date('c'),
|
|
'active' => $active,
|
|
'failover' => [
|
|
'config' => $this->failoverConfigForSnapshot(),
|
|
'hosts' => $this->failoverHostsForSnapshot(),
|
|
],
|
|
]);
|
|
}
|
|
|
|
private function failoverConfigForSnapshot(): array
|
|
{
|
|
$config = replica_failover_manager::configDefaults();
|
|
|
|
try {
|
|
foreach ($this->selectRows("SELECT variable, value FROM module_config WHERE module = 'Failover'") as $row) {
|
|
$variable = (string)($row['variable'] ?? '');
|
|
if (!array_key_exists($variable, $config)) {
|
|
continue;
|
|
}
|
|
$config[$variable] = $row['value'] ?? '';
|
|
}
|
|
} catch (Throwable) {
|
|
}
|
|
|
|
return replica_failover_manager::normalizeConfig($config);
|
|
}
|
|
|
|
private function failoverHostsForSnapshot(): array
|
|
{
|
|
return [
|
|
self::KIND_DATABASE => array_map(
|
|
fn(array $host): array => $this->bootstrapSnapshotHost($host),
|
|
$this->listHosts(self::KIND_DATABASE)
|
|
),
|
|
self::KIND_REDIS => array_map(
|
|
fn(array $host): array => $this->bootstrapSnapshotHost($host),
|
|
$this->listHosts(self::KIND_REDIS)
|
|
),
|
|
self::KIND_MINIO => array_map(
|
|
fn(array $host): array => $this->bootstrapSnapshotHost($host),
|
|
$this->listHosts(self::KIND_MINIO)
|
|
),
|
|
];
|
|
}
|
|
|
|
private function bootstrapSnapshotHost(array $host): array
|
|
{
|
|
return [
|
|
'id' => (int)$host['id'],
|
|
'kind' => (string)$host['kind'],
|
|
'label' => (string)$host['label'],
|
|
'host' => (string)$host['host'],
|
|
'port' => (int)$host['port'],
|
|
'database_name' => $host['database_name'] ?? null,
|
|
'database_index' => isset($host['database_index']) ? (int)$host['database_index'] : null,
|
|
'username' => (string)($host['username'] ?? ''),
|
|
'password_secret' => (string)($host['password_secret'] ?? ''),
|
|
'admin_username' => (string)($host['admin_username'] ?? ''),
|
|
'admin_password_secret' => (string)($host['admin_password_secret'] ?? ''),
|
|
'replication_username' => (string)($host['replication_username'] ?? ''),
|
|
'replication_password_secret' => (string)($host['replication_password_secret'] ?? ''),
|
|
'role' => (string)$host['role'],
|
|
'status' => (string)$host['status'],
|
|
'replication_source_id' => isset($host['replication_source_id']) ? (int)$host['replication_source_id'] : null,
|
|
'ssl_mode' => $host['ssl_mode'] ?? null,
|
|
'options_json' => $host['options_json'] ?? null,
|
|
'last_status_json' => $host['last_status_json'] ?? null,
|
|
'last_checked_at' => $host['last_checked_at'] ?? null,
|
|
'updated_at' => $host['updated_at'] ?? null,
|
|
'deleted_at' => $host['deleted_at'] ?? null,
|
|
];
|
|
}
|
|
|
|
private function switchPrimary(string $kind, int $newPrimaryId, int $oldPrimaryId): void
|
|
{
|
|
$this->execute(
|
|
"UPDATE replication_hosts SET role = 'inactive', status = 'inactive' WHERE kind = ? AND role = 'primary' AND id <> ?",
|
|
'si',
|
|
[$kind, $newPrimaryId]
|
|
);
|
|
$this->execute(
|
|
"UPDATE replication_hosts SET role = 'primary', status = 'ok', replication_source_id = NULL WHERE kind = ? AND id = ?",
|
|
'si',
|
|
[$kind, $newPrimaryId]
|
|
);
|
|
$this->execute(
|
|
"UPDATE replication_hosts SET replication_source_id = ? WHERE kind = ? AND role = 'replica'",
|
|
'is',
|
|
[$newPrimaryId, $kind]
|
|
);
|
|
}
|
|
|
|
private function credentials(array $host): array
|
|
{
|
|
return [
|
|
'username' => (string)($host['username'] ?? ''),
|
|
'password' => replication_secret_box::decrypt($host['password_secret'] ?? ''),
|
|
'admin_username' => (string)($host['admin_username'] ?? ''),
|
|
'admin_password' => replication_secret_box::decrypt($host['admin_password_secret'] ?? ''),
|
|
'replication_username' => (string)($host['replication_username'] ?? ''),
|
|
'replication_password' => replication_secret_box::decrypt($host['replication_password_secret'] ?? ''),
|
|
];
|
|
}
|
|
|
|
private function decodeOptions(array $host): array
|
|
{
|
|
if (isset($host['options']) && is_array($host['options'])) {
|
|
return $host['options'];
|
|
}
|
|
|
|
return self::jsonDecode($host['options_json'] ?? null);
|
|
}
|
|
|
|
private function listHosts(?string $kind = null, bool $includeDeleted = false): array
|
|
{
|
|
$where = [];
|
|
$types = '';
|
|
$params = [];
|
|
if ($kind !== null) {
|
|
$where[] = 'kind = ?';
|
|
$types .= 's';
|
|
$params[] = $kind;
|
|
}
|
|
if (!$includeDeleted) {
|
|
$where[] = 'deleted_at IS NULL';
|
|
}
|
|
|
|
$sql = 'SELECT * FROM replication_hosts';
|
|
if ($where !== []) {
|
|
$sql .= ' WHERE ' . implode(' AND ', $where);
|
|
}
|
|
$sql .= " ORDER BY FIELD(role, 'primary', 'replica', 'inactive'), id";
|
|
|
|
return $this->selectRows($sql, $types, $params);
|
|
}
|
|
|
|
private function primaryHost(string $kind): ?array
|
|
{
|
|
return $this->selectOne(
|
|
"SELECT * FROM replication_hosts WHERE kind = ? AND role = 'primary' AND deleted_at IS NULL ORDER BY id DESC LIMIT 1",
|
|
's',
|
|
[$kind]
|
|
);
|
|
}
|
|
|
|
private function getHost(string $kind, int $id, bool $includeDeleted = false): array
|
|
{
|
|
$sql = 'SELECT * FROM replication_hosts WHERE kind = ? AND id = ?';
|
|
if (!$includeDeleted) {
|
|
$sql .= ' AND deleted_at IS NULL';
|
|
}
|
|
$host = $this->selectOne($sql . ' LIMIT 1', 'si', [$kind, $id]);
|
|
if ($host === null) {
|
|
throw new RuntimeException('Replication host was not found.');
|
|
}
|
|
return $host;
|
|
}
|
|
|
|
private function startOperation(string $kind, int $hostId, string $operation, ?int $actorUserId): int
|
|
{
|
|
$this->execute(
|
|
"INSERT INTO replication_operations (kind, host_id, operation, status, actor_user_id)
|
|
VALUES (?, ?, ?, 'running', ?)",
|
|
'sisi',
|
|
[$kind, $hostId, $operation, $actorUserId]
|
|
);
|
|
return $this->insertId();
|
|
}
|
|
|
|
private function activeOperationId(string $kind, int $hostId, string $operation): ?int
|
|
{
|
|
$operationRow = $this->selectOne(
|
|
"SELECT id FROM replication_operations
|
|
WHERE kind = ? AND host_id = ? AND operation = ? AND status = 'running'
|
|
ORDER BY id DESC
|
|
LIMIT 1",
|
|
'sis',
|
|
[$kind, $hostId, $operation]
|
|
);
|
|
|
|
return $operationRow !== null ? (int)$operationRow['id'] : null;
|
|
}
|
|
|
|
private function activeOperation(string $kind, int $hostId): ?array
|
|
{
|
|
$operation = $this->selectOne(
|
|
"SELECT id, operation, status, progress_percent, message, error_message, started_at, updated_at
|
|
FROM replication_operations
|
|
WHERE kind = ? AND host_id = ? AND status = 'running'
|
|
ORDER BY id DESC
|
|
LIMIT 1",
|
|
'si',
|
|
[$kind, $hostId]
|
|
);
|
|
|
|
if ($operation === null) {
|
|
return null;
|
|
}
|
|
|
|
return [
|
|
'id' => (int)$operation['id'],
|
|
'operation' => (string)$operation['operation'],
|
|
'status' => (string)$operation['status'],
|
|
'progress_percent' => round((float)$operation['progress_percent'], 2),
|
|
'message' => $operation['message'] ?? null,
|
|
'error_message' => $operation['error_message'] ?? null,
|
|
'started_at' => $operation['started_at'] ?? null,
|
|
'updated_at' => $operation['updated_at'] ?? null,
|
|
];
|
|
}
|
|
|
|
private function operationContext(int $operationId): array
|
|
{
|
|
$operation = $this->selectOne(
|
|
'SELECT context_json FROM replication_operations WHERE id = ? LIMIT 1',
|
|
'i',
|
|
[$operationId]
|
|
);
|
|
|
|
return self::jsonDecode($operation['context_json'] ?? null);
|
|
}
|
|
|
|
private function updateOperationProgress(int $operationId, float $progress, string $message, array $context): void
|
|
{
|
|
$this->execute(
|
|
"UPDATE replication_operations
|
|
SET progress_percent = ?, message = ?, context_json = ?
|
|
WHERE id = ?",
|
|
'dssi',
|
|
[max(0, min(100, $progress)), $message, self::jsonEncode($context), $operationId]
|
|
);
|
|
}
|
|
|
|
private function finishOperation(int $operationId, string $status, float $progress, ?string $message, array $errors): void
|
|
{
|
|
$this->execute(
|
|
"UPDATE replication_operations
|
|
SET status = ?, progress_percent = ?, message = ?, error_message = ?, completed_at = NOW()
|
|
WHERE id = ?",
|
|
'sdssi',
|
|
[$status, $progress, $message, implode("\n", $errors), $operationId]
|
|
);
|
|
}
|
|
|
|
private function audit(string $kind, ?int $hostId, string $action, ?int $actorUserId, string $severity, array $context): void
|
|
{
|
|
$this->execute(
|
|
"INSERT INTO replication_audit_logs (kind, host_id, action, actor_user_id, severity, context_json)
|
|
VALUES (?, ?, ?, ?, ?, ?)",
|
|
'sisiss',
|
|
[$kind, $hostId, $action, $actorUserId, $severity, self::jsonEncode($context)]
|
|
);
|
|
}
|
|
|
|
private function acquirePromotionLock()
|
|
{
|
|
$path = (defined('WD') ? WD : dirname(__DIR__)) . DIRECTORY_SEPARATOR . 'storage' . DIRECTORY_SEPARATOR . 'replication-promotion.lock';
|
|
$dir = dirname($path);
|
|
if (!is_dir($dir) && !mkdir($dir, 0770, true) && !is_dir($dir)) {
|
|
throw new RuntimeException('Could not create promotion lock directory.');
|
|
}
|
|
$handle = fopen($path, 'c');
|
|
if ($handle === false || !flock($handle, LOCK_EX | LOCK_NB)) {
|
|
throw new RuntimeException('Another replication promotion is already running.');
|
|
}
|
|
return $handle;
|
|
}
|
|
|
|
private function releasePromotionLock($handle): void
|
|
{
|
|
if (is_resource($handle)) {
|
|
flock($handle, LOCK_UN);
|
|
fclose($handle);
|
|
}
|
|
}
|
|
|
|
private function selectOne(string $sql, string $types = '', array $params = []): ?array
|
|
{
|
|
$rows = $this->selectRows($sql, $types, $params);
|
|
return $rows[0] ?? null;
|
|
}
|
|
|
|
private function selectRows(string $sql, string $types = '', array $params = []): array
|
|
{
|
|
global $db;
|
|
if ($types === '') {
|
|
$result = $db->query($sql);
|
|
return $result ? $result->fetch_all(MYSQLI_ASSOC) : [];
|
|
}
|
|
|
|
$stmt = $db->prepare($sql);
|
|
if ($stmt === false) {
|
|
throw new RuntimeException('Could not prepare replication query.');
|
|
}
|
|
$stmt->bind_param($types, ...$params);
|
|
$stmt->execute();
|
|
$result = $stmt->get_result();
|
|
return $result ? $result->fetch_all(MYSQLI_ASSOC) : [];
|
|
}
|
|
|
|
private function execute(string $sql, string $types = '', array $params = []): void
|
|
{
|
|
global $db;
|
|
if ($types === '') {
|
|
$db->query($sql);
|
|
return;
|
|
}
|
|
|
|
$stmt = $db->prepare($sql);
|
|
if ($stmt === false) {
|
|
throw new RuntimeException('Could not prepare replication statement.');
|
|
}
|
|
$stmt->bind_param($types, ...$params);
|
|
$stmt->execute();
|
|
}
|
|
|
|
private function insertId(): int
|
|
{
|
|
global $db;
|
|
return (int)$db->insert_id();
|
|
}
|
|
|
|
private static function jsonEncode(mixed $value): string
|
|
{
|
|
$json = json_encode($value, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
|
|
if ($json === false) {
|
|
throw new RuntimeException('Could not encode replication JSON payload.');
|
|
}
|
|
return $json;
|
|
}
|
|
|
|
private static function jsonDecode(mixed $value): array
|
|
{
|
|
if (!is_string($value) || trim($value) === '') {
|
|
return [];
|
|
}
|
|
$decoded = json_decode($value, true);
|
|
return is_array($decoded) ? $decoded : [];
|
|
}
|
|
}
|