toBe(11); expect(replication_manager::mysqlGtidCoveragePercent($source, $executed))->toBe(72.73); }); it('reports empty source GTID sets as caught up', function (): void { expect(replication_manager::mysqlGtidCoveragePercent('', ''))->toBe(100.0); }); it('computes Redis offset percentages safely', function (): void { expect(replication_manager::redisOffsetPercent(1000, 750))->toBe(75.0); expect(replication_manager::redisOffsetPercent(0, 0))->toBe(100.0); expect(replication_manager::redisOffsetPercent(1000, 1250))->toBe(100.0); expect(replication_manager::redisReplicationPercentFromInfo( ['master_repl_offset' => 1000], ['slave_repl_offset' => 750] ))->toBe(75.0); expect(replication_manager::redisReplicationPercentFromInfo( ['master_repl_offset' => 1000], ['master_sync_in_progress' => '1', 'master_sync_total_bytes' => '1000', 'master_sync_left_bytes' => '250'] ))->toBe(75.0); expect(replication_manager::redisReplicationPercentFromInfo( ['master_repl_offset' => 1000], ['master_sync_in_progress' => '1'] ))->toBe(5.0); expect(replication_manager::redisProvisionProgress(0.0))->toBe(5.0); expect(replication_manager::redisProvisionProgress(100.0, ['Redis replica link to primary is not up.']))->toBe(99.99); expect(replication_manager::replicationHealthStatus(true, 'replica', 5.0, []))->toBe('degraded'); expect(replication_manager::replicationHealthStatus(true, 'replica', 5.0, [], false))->toBe('ok'); expect(replication_manager::replicationHealthStatus(true, 'replica', 100.0, []))->toBe('ok'); }); it('computes MariaDB GTID coverage by domain sequence', function (): void { $source = '0-1-10,1-1-20'; $replica = '0-2-8,1-3-20'; expect(replication_manager::mariadbGtidCoveragePercent($source, $replica))->toBe(93.33); expect(replication_manager::mariadbGtidCoveragePercent('', ''))->toBe(100.0); }); it('normalizes public replication kind aliases', function (): void { expect(replication_manager::normalizeKind('databases'))->toBe('database'); expect(replication_manager::normalizeKind('mysql'))->toBe('database'); expect(replication_manager::normalizeKind('redis'))->toBe('redis'); expect(replication_manager::normalizeKind('minio'))->toBe('minio'); expect(replication_manager::normalizeKind('s3'))->toBe('minio'); expect(replication_manager::normalizeKind('object-storage'))->toBe('minio'); }); it('generates replication-ready MariaDB compose templates without embedding secrets', function (): void { $template = replication_manager::composeTemplate([ 'kind' => 'database', 'role' => 'replica', 'service_name' => 'MariaDB Replica 2', 'database' => 'nnks_db', 'username' => 'nnks_db_user', 'host_port' => 5433, 'server_id' => 2, ]); expect($template['kind'])->toBe('database'); expect($template['role'])->toBe('replica'); expect($template['service_name'])->toBe('mariadb-replica-2'); expect($template['compose'])->toContain('image: "mariadb:11"'); expect($template['compose'])->toContain('"--server-id=2"'); expect($template['compose'])->toContain('"--log-bin=/var/lib/mysql/mariadb-bin"'); expect($template['compose'])->toContain('"--binlog-format=ROW"'); expect($template['compose'])->toContain('"--gtid-strict-mode=ON"'); expect($template['compose'])->toContain('"--read-only=ON"'); expect($template['compose'])->toContain('"--replicate-ignore-table=nnks_db.logs"'); expect($template['compose'])->toContain('"--replicate-ignore-table=nnks_db.edge_gateway_log_entries"'); expect($template['compose'])->toContain('"--replicate-ignore-table=nnks_db.replication_status_snapshots"'); expect($template['compose'])->toContain('"--replicate-ignore-table=nnks_db.system_search_documents"'); expect($template['compose'])->toContain('"5433:3306"'); expect($template['compose'])->toContain('mariadb-replica-2-seed'); expect($template['compose'])->toContain('MARIADB_PRIMARY_ADMIN_PASSWORD'); expect($template['compose'])->toContain('mariadb-dump --host="$${MARIADB_PRIMARY_HOST}"'); expect($template['compose'])->toContain('--ignore-table="$${MARIADB_SEED_DATABASE}.logs"'); expect($template['compose'])->toContain('--ignore-table="$${MARIADB_SEED_DATABASE}.edge_gateway_log_entries"'); expect($template['compose'])->toContain('--ignore-table="$${MARIADB_SEED_DATABASE}.replication_status_snapshots"'); expect($template['compose'])->toContain('--ignore-table="$${MARIADB_SEED_DATABASE}.system_search_documents"'); expect($template['compose'])->toContain('--no-data "$${MARIADB_SEED_DATABASE}" "$${table}"'); expect($template['compose'])->toContain('touch "$${marker}"'); expect($template['compose'])->toContain('${MARIADB_ROOT_PASSWORD:?set MARIADB_ROOT_PASSWORD}'); expect($template['compose'])->not->toContain(''); expect($template['env'])->toMatch('/MARIADB_ROOT_PASSWORD=[A-Za-z0-9_-]{32}/'); expect($template['env'])->toMatch('/MARIADB_PASSWORD=[A-Za-z0-9_-]{32}/'); expect($template['env'])->toContain('MARIADB_PRIMARY_ADMIN_PASSWORD='); expect($template['credentials']['password'])->toMatch('/^[A-Za-z0-9_-]{32}$/'); expect($template['credentials']['admin_password'])->toMatch('/^[A-Za-z0-9_-]{32}$/'); expect($template['credentials']['port'])->toBe(5433); expect($template['credentials']['allow_preseeded_replica'])->toBeTrue(); expect($template['seed_command'])->toContain('mariadb-dump'); expect($template['seed_command'])->toContain('--gtid'); expect($template['seed_command'])->toContain('--master-data=2'); expect($template['seed_command'])->toContain('--ignore-table=\'nnks_db.logs\''); expect($template['seed_command'])->toContain('--ignore-table=\'nnks_db.edge_gateway_log_entries\''); expect($template['seed_command'])->toContain('--ignore-table=\'nnks_db.replication_status_snapshots\''); expect($template['seed_command'])->toContain('--ignore-table=\'nnks_db.system_search_documents\''); }); it('can embed primary admin credentials in generated MariaDB replica env files', function (): void { $template = replication_manager::composeTemplate([ 'kind' => 'database', 'role' => 'replica', 'primary_admin_username' => 'primary-root', 'primary_admin_password' => 'primary-secret', ]); expect($template['env'])->toContain('MARIADB_PRIMARY_ADMIN_USER=primary-root'); expect($template['env'])->toContain('MARIADB_PRIMARY_ADMIN_PASSWORD=primary-secret'); }); it('detects missing database tables before provisioning a preseeded replica', function (): void { expect(replication_manager::missingDatabaseTables( ['customers', 'edge_gateways', 'orders'], ['customers', 'orders'] ))->toBe(['edge_gateways']); }); it('seeds MariaDB replicas in place instead of requiring container recreation', function (): void { $content = file_get_contents(app_path('classes/replication_manager.php')); expect($content)->toContain('advanceMariaDbReplicaSeed($operationId, $primary, $host, $target)'); expect($content)->toContain('MARIADB_SEED_BATCH_ROWS'); expect($content)->toContain('MARIADB_SEED_STEP_SECONDS'); expect($content)->toContain('activeOperationId($kind, $id, \'provision\')'); expect($content)->toContain('updateOperationProgress($operationId, $progress, $message, $context)'); expect($content)->toContain("application_write_freeze::freeze('MariaDB replica seed is copying data.'"); expect($content)->toContain('DROP DATABASE IF EXISTS'); expect($content)->toContain('CREATE DATABASE '); expect($content)->toContain('SHOW CREATE TABLE'); expect($content)->toContain('SET GLOBAL gtid_slave_pos'); expect($content)->toContain('databasePrimaryKeyColumnNames($source, $primaryDatabase, $tableName)'); expect($content)->toContain('$target->begin_transaction()'); expect($content)->toContain("'connect_without_database' => \$usePreseededReplica"); }); it('keeps operational and derived tables schema-only during MariaDB seeding and replication', function (): void { $content = file_get_contents(app_path('classes/replication_manager.php')); expect($content)->toContain('MARIADB_SCHEMA_ONLY_TABLES'); expect($content)->toContain("'logs'"); expect($content)->toContain("'edge_gateway_log_entries'"); expect($content)->toContain("'replication_status_snapshots'"); expect($content)->toContain("'replication_operations'"); expect($content)->toContain("'replication_audit_logs'"); expect($content)->toContain("'system_search_documents'"); expect($content)->toContain("'skip_data' => \$skipData"); expect($content)->toContain('createMariaDbReplicaTable('); expect($content)->toContain('SET GLOBAL replicate_ignore_table'); expect($content)->toContain('--replicate-ignore-table='); expect($content)->toContain('mariaDbSchemaOnlyDumpIgnoreArgs'); expect($content)->toContain('mariaDbSchemaOnlySeedCommandIgnoreArgs'); expect($content)->toContain('databaseSchemaOnlyTablesWithRows'); expect($content)->toContain('schemaOnlyTablesContainRowsBlocker'); expect($content)->toContain('RESET SLAVE ALL'); expect($content)->toContain('mariaDbSeedContextRequiresFilterReset($context)'); }); it('allows failed replicas to be removed without allowing primary or healthy replica removal', function (): void { expect(replication_manager::replicationHostCanBeRemoved(['role' => 'primary', 'status' => 'ok']))->toBeFalse(); expect(replication_manager::replicationHostCanBeRemoved(['role' => 'inactive', 'status' => 'inactive']))->toBeTrue(); expect(replication_manager::replicationHostCanBeRemoved(['role' => 'replica', 'status' => 'degraded']))->toBeTrue(); expect(replication_manager::replicationHostCanBeRemoved(['role' => 'replica', 'status' => 'ok']))->toBeFalse(); $content = file_get_contents(app_path('classes/replication_manager.php')); expect($content)->toContain('coolify_manager::replicationHostCanBeRemoved'); expect($content)->toContain('coolify_manager::markTargetsRemovedForReplicationHost'); }); it('supports metadata-only replication host renames', function (): void { $content = file_get_contents(app_path('classes/replication_manager.php')); expect($content)->toContain('function renameHost('); expect($content)->toContain('host_renamed'); expect($content)->toContain('coolify_manager::syncLabelForReplicationHost'); expect($content)->toContain('writeBootstrapSnapshot()'); expect($content)->toContain('Replication host label must be 128 characters or fewer.'); }); it('generates Redis replica compose templates with primary connection placeholders', function (): void { $template = replication_manager::composeTemplate([ 'kind' => 'redis', 'role' => 'replica', 'service_name' => 'Redis Replica', 'host_port' => 6380, 'primary_host' => 'redis-primary.internal', 'primary_port' => 6379, ]); expect($template['kind'])->toBe('redis'); expect($template['compose'])->toContain('image: "redis:7"'); expect($template['compose'])->toContain('REDIS_PASSWORD: "${REDIS_PASSWORD:?set REDIS_PASSWORD}"'); expect($template['compose'])->toContain('REDIS_PRIMARY_HOST: "${REDIS_PRIMARY_HOST:?set REDIS_PRIMARY_HOST}"'); expect($template['compose'])->toContain('REDIS_PRIMARY_PORT: "${REDIS_PRIMARY_PORT:-6379}"'); expect($template['compose'])->toContain('${REDIS_PRIMARY_PASSWORD:?set REDIS_PRIMARY_PASSWORD}'); expect($template['compose'])->toContain('REDIS_PRIMARY_USERNAME: "${REDIS_PRIMARY_USERNAME:-}"'); expect($template['compose'])->toContain('if [ ! -f /data/redis.conf ]; then'); expect($template['compose'])->toContain('> /data/redis.conf'); expect($template['compose'])->toContain('exec redis-server /data/redis.conf'); expect($template['compose'])->toContain('echo "replicaof $$REDIS_PRIMARY_HOST $${REDIS_PRIMARY_PORT:-6379}"'); expect($template['compose'])->toContain('echo "masterauth $$REDIS_PRIMARY_PASSWORD"'); expect($template['compose'])->toContain('echo "masteruser $$REDIS_PRIMARY_USERNAME"'); expect($template['compose'])->toContain('redis-cli --no-auth-warning -a \"$${REDIS_PASSWORD}\" ping | grep PONG'); expect($template['compose'])->not->toContain($template['credentials']['password']); expect($template['env'])->toMatch('/REDIS_PASSWORD=[A-Za-z0-9_-]{32}/'); expect($template['env'])->toContain('REDIS_PRIMARY_HOST=redis-primary.internal'); expect($template['env'])->toContain('REDIS_PRIMARY_PORT=6379'); expect($template['env'])->toContain("REDIS_PRIMARY_PASSWORD=\n"); expect($template['env'])->toContain("REDIS_PRIMARY_USERNAME=\n"); expect($template['credentials']['password'])->toMatch('/^[A-Za-z0-9_-]{32}$/'); }); it('generates MinIO replica compose templates without embedding secrets', function (): void { $template = replication_manager::composeTemplate([ 'kind' => 'minio', 'role' => 'replica', 'service_name' => 'MinIO Replica 1', 'host' => 'node2.truckwash.dk', 'scheme' => 'http', 'host_port' => 9010, 'console_port' => 9011, 'buckets' => ['attachments', 'uploads'], ]); expect($template['kind'])->toBe('minio'); expect($template['engine'])->toBe('minio'); expect($template['service_name'])->toBe('minio-replica-1'); expect($template['host_port'])->toBe(9010); expect($template['console_port'])->toBe(9011); expect($template['compose'])->toContain('image: "minio/minio:latest"'); expect($template['compose'])->toContain('MINIO_ROOT_USER: "${MINIO_ROOT_USER:?set MINIO_ROOT_USER}"'); expect($template['compose'])->toContain('MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:?set MINIO_ROOT_PASSWORD}"'); expect($template['compose'])->toContain('MINIO_SERVER_URL: "${MINIO_SERVER_URL:-}"'); expect($template['compose'])->toContain('MINIO_BROWSER_REDIRECT_URL: "${MINIO_BROWSER_REDIRECT_URL:-}"'); expect($template['compose'])->toContain('"9010:9000"'); expect($template['compose'])->toContain('"9011:9001"'); expect($template['compose'])->toContain('mc mb --with-lock --ignore-existing'); expect($template['compose'])->toContain('mc version enable'); expect($template['compose'])->toContain('MINIO_PRIMARY_ENDPOINT'); expect($template['compose'])->not->toContain($template['credentials']['password']); expect($template['env'])->toMatch('/MINIO_ROOT_USER=twminio[a-f0-9]{24}/'); expect($template['env'])->toMatch('/MINIO_ROOT_PASSWORD=[A-Za-z0-9_-]{32}/'); expect($template['env'])->toContain('MINIO_SERVER_URL=http://node2.truckwash.dk:9010'); expect($template['env'])->toContain('MINIO_BROWSER_REDIRECT_URL=http://node2.truckwash.dk:9011'); expect($template['env'])->toContain('MINIO_BUCKETS=attachments,uploads'); expect($template['env'])->toContain('MINIO_REPLICATION_TRANSFER_LIMIT=25Mi'); expect($template['env'])->toContain('MINIO_PRIMARY_ENDPOINT='); expect($template['credentials']['username'])->toMatch('/^twminio[a-f0-9]{24}$/'); expect($template['credentials']['scheme'])->toBe('http'); expect($template['credentials']['buckets'])->toBe(['attachments', 'uploads']); expect($template['credentials']['replication_transfer_limit'])->toBe('25Mi'); expect($template['credentials']['space_headroom_percent'])->toBe(20.0); }); it('keeps MinIO backup replicas bounded to the recent backup window', function (): void { $template = replication_manager::composeTemplate([ 'kind' => 'minio', 'role' => 'replica', 'service_name' => 'minio-replica-1', 'buckets' => ['backups', 'uploads'], ]); $content = file_get_contents(app_path('classes/replication_manager.php')); expect(replication_manager::minioBackupReplicaRetentionDays())->toBe(30); expect($template['compose'])->toContain('mc ilm rule add --expire-days "30" --noncurrent-expire-days "30"'); expect($template['env'])->toContain('MINIO_BACKUP_REPLICA_RETENTION_DAYS=30'); expect($template['steps'])->toContain('The backups bucket is retained on replicas for 30 days; other buckets are fully replicated.'); expect($content)->not->toContain('seedMinioReplicaBackupWindow'); expect($content)->not->toContain("'--newer-than'"); expect($content)->toContain("'--limit-upload'"); expect($content)->toContain("'--limit-download'"); expect($content)->toContain("? 'delete,delete-marker'"); expect($content)->toContain('putBucketLifecycleConfiguration'); expect($content)->toContain('listObjectVersions'); expect($content)->toContain('minioBackupReplicaRetentionConfigured'); expect(replication_manager::minioBackupRetentionBlockers([ 'buckets' => [ ['name' => 'backups', 'expired_objects' => 2], ], ]))->toBe([ 'MinIO backup replica contains 2 backup objects older than 30 days. Run provisioning to prune retained backups.', ]); }); it('prefills MinIO replica compose primary values from current config when available', function (): void { $previousMinio = $GLOBALS['MINIO'] ?? null; $GLOBALS['MINIO'] = [ 'endpoint' => 'https://minio-primary.internal:9000', 'access_key' => 'primary-access', 'secret_key' => 'primary-secret', ]; try { $template = replication_manager::composeTemplate([ 'kind' => 'minio', 'role' => 'replica', 'service_name' => 'minio-replica-1', ]); expect($template['env'])->toContain('MINIO_PRIMARY_ENDPOINT=https://minio-primary.internal:9000'); expect($template['env'])->toContain('MINIO_PRIMARY_ACCESS_KEY=primary-access'); expect($template['env'])->toContain('MINIO_PRIMARY_SECRET_KEY=primary-secret'); expect($template['compose'])->not->toContain('primary-secret'); } finally { if ($previousMinio === null) { unset($GLOBALS['MINIO']); } else { $GLOBALS['MINIO'] = $previousMinio; } } }); it('computes MinIO free-space and catch-up math safely', function (): void { expect(replication_manager::minioRequiredFreeBytes(1000))->toBe(1200); expect(replication_manager::minioByteReplicationPercent(1000, 750))->toBe(75.0); expect(replication_manager::minioByteReplicationPercent(0, 0))->toBe(100.0); expect(replication_manager::minioProvisionProgress([ 'replication_percent' => 3.2, 'raw' => ['storage' => ['measured' => true]], ]))->toBe(3.2); expect(replication_manager::minioProvisionProgress([ 'replication_percent' => 0, 'raw' => ['storage' => ['measured' => false]], ]))->toBe(5.0); expect(replication_manager::minioProvisionProgress([ 'replication_percent' => 2.5, 'raw' => ['progress_source' => 'minio_replicate_status'], ]))->toBe(2.5); expect(replication_manager::minioProvisionProgress([ 'replication_percent' => 100, 'raw' => ['storage' => ['measured' => true]], ]))->toBe(100.0); expect(replication_manager::minioProvisionProgress([ 'replication_percent' => 100, 'blockers' => ['MinIO replica has not caught up.'], 'raw' => ['storage' => ['measured' => true]], ]))->toBe(99.9); expect(replication_manager::minioReplicationProgressFromStatusOutput([ 'target' => [ 'replicated' => ['size' => 750], 'pending' => ['size' => 250], ], ])['replication_percent'])->toBe(75.0); expect(replication_manager::minioReplicationProgressFromStatusOutput([ 'objects' => [ 'completed' => 9, 'pending' => 1, ], ])['replication_percent'])->toBe(90.0); expect(replication_manager::minioReplicationProgressFromStatusOutput([ 'status' => 'complete', ])['replication_percent'])->toBe(100.0); expect(replication_manager::minioReplicationProgressFromStatusOutput( '{"replicatedSize": 750, "pendingSize": 250}' )['replication_percent'])->toBe(75.0); expect(replication_manager::minioReplicationProgressFromStatusOutput( '{"replicaSize": 750, "pendingSize": 250}' )['replication_percent'])->toBe(75.0); expect(replication_manager::minioReplicationProgressFromStatusOutput( 'target-a: 100%, target-b: 5%' )['replication_percent'])->toBe(5.0); expect(replication_manager::minioReplicationProgressFromStatusOutput( 'target-a: 100%, target-b: 100%' )['replication_percent'])->toBe(100.0); expect(replication_manager::minioReplicationProgressFromStatusOutput([ 'completedReplicationSize' => 1000, 'queued' => [ 'curr' => ['count' => 0, 'bytes' => 0], 'avg' => ['count' => 42, 'bytes' => 25000000], 'peak' => ['count' => 100, 'bytes' => 50000000], ], ])['replication_percent'])->toBe(100.0); expect(replication_manager::minioBucketCountsTowardCatchUp('uploads'))->toBeTrue(); expect(replication_manager::minioBucketCountsTowardCatchUp('backups'))->toBeFalse(); $boundedBackupProgress = replication_manager::minioCatchUpProgressFromBucketStatuses([ 'uploads' => ['replication_percent' => 100.0, 'blockers' => [], 'stats' => []], 'backups' => ['replication_percent' => 99.74, 'blockers' => ['MinIO replica has not caught up.'], 'stats' => []], ]); expect($boundedBackupProgress['replication_percent'])->toBe(100.0); expect($boundedBackupProgress['blockers'])->toBe([]); expect($boundedBackupProgress['ignored_buckets'])->toBe(['backups']); $onlyBoundedBackupProgress = replication_manager::minioCatchUpProgressFromBucketStatuses([ 'backups' => ['replication_percent' => 5.0, 'blockers' => ['MinIO replica has not caught up.'], 'stats' => []], ]); expect($onlyBoundedBackupProgress['replication_percent'])->toBe(100.0); expect($onlyBoundedBackupProgress['basis'])->toBe('bounded_retention_only'); $liveQueueProgress = replication_manager::minioCatchUpProgressFromBucketStatuses([ 'uploads' => [ 'replication_percent' => 99.98, 'blockers' => ['MinIO replica has not caught up.'], 'stats' => [ 'completed_bytes' => 53149249190, 'pending_bytes' => 3439936, 'failed_bytes' => 0, 'total_bytes' => 0, 'completed_count' => 199368, 'pending_count' => 7, 'failed_count' => 0, 'total_count' => 0, ], ], ]); expect($liveQueueProgress['replication_percent'])->toBe(100.0); expect($liveQueueProgress['blockers'])->toBe([]); expect($liveQueueProgress['live_tolerance']['within_tolerance'])->toBeTrue(); expect(replication_manager::minioSpaceBlockers(1199, 1200))->toContain('MinIO target does not have enough free space. Required 1200 bytes, available 1199 bytes.'); expect(replication_manager::minioSpaceBlockers(null, 1200))->toBe([]); expect(replication_manager::minioSpaceBlockers(1200, 1200))->toBe([]); expect(replication_manager::minioAvailableBytesFromAdminInfo([ 'servers' => [ ['drives' => [['availableSpace' => 4096]]], ], ]))->toBe(4096); expect(replication_manager::normalizeMinioBuckets('Attachments, uploads backups'))->toBe(['attachments', 'uploads', 'backups']); expect(replication_manager::minioDefaultReplicationTransferLimit())->toBe('25Mi'); expect(replication_manager::normalizeMinioTransferLimit('25MiB/s'))->toBe('25Mi'); expect(replication_manager::normalizeMinioTransferLimit('100 MB'))->toBe('100M'); expect(replication_manager::normalizeMinioTransferLimit('0'))->toBe(''); }); it('allows the MinIO client binary to be configured explicitly', function (): void { $previous = getenv('MINIO_MC_BINARY'); putenv('MINIO_MC_BINARY=/opt/minio/mc'); try { $method = new ReflectionMethod(replication_manager::class, 'minioClientBinary'); $method->setAccessible(true); expect($method->invoke(null))->toBe('/opt/minio/mc'); } finally { if ($previous === false) { putenv('MINIO_MC_BINARY'); } else { putenv('MINIO_MC_BINARY=' . $previous); } } }); it('supports MinIO client runtime fallback configuration', function (): void { $previousDownloadUrl = getenv('MINIO_MC_DOWNLOAD_URL'); $previousAutoInstall = getenv('MINIO_MC_AUTO_INSTALL'); $previousCommandTimeout = getenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS'); $previousDownloadTimeout = getenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS'); putenv('MINIO_MC_DOWNLOAD_URL=https://example.test/mc'); putenv('MINIO_MC_AUTO_INSTALL=0'); putenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS=3'); putenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS=4'); try { $downloadUrl = new ReflectionMethod(replication_manager::class, 'minioClientDownloadUrl'); $downloadUrl->setAccessible(true); $autoInstall = new ReflectionMethod(replication_manager::class, 'minioClientAutoInstallEnabled'); $autoInstall->setAccessible(true); $commandTimeout = new ReflectionMethod(replication_manager::class, 'minioClientCommandTimeoutSeconds'); $commandTimeout->setAccessible(true); $downloadTimeout = new ReflectionMethod(replication_manager::class, 'minioClientDownloadTimeoutSeconds'); $downloadTimeout->setAccessible(true); $commandLabel = new ReflectionMethod(replication_manager::class, 'minioClientCommandLabel'); $commandLabel->setAccessible(true); expect($downloadUrl->invoke(null))->toBe('https://example.test/mc'); expect($autoInstall->invoke(null))->toBeFalse(); expect($commandTimeout->invoke(null))->toBe(3); expect($downloadTimeout->invoke(null))->toBe(4); expect($commandLabel->invoke(null, [ 'alias', 'set', 'target', 'http://minio.example.test:9010', 'access-key', 'secret-key', ]))->toContain('[redacted]'); expect($commandLabel->invoke(null, [ 'alias', 'set', 'target', 'http://minio.example.test:9010', 'access-key', 'secret-key', ]))->not->toContain('secret-key'); } finally { if ($previousDownloadUrl === false) { putenv('MINIO_MC_DOWNLOAD_URL'); } else { putenv('MINIO_MC_DOWNLOAD_URL=' . $previousDownloadUrl); } if ($previousAutoInstall === false) { putenv('MINIO_MC_AUTO_INSTALL'); } else { putenv('MINIO_MC_AUTO_INSTALL=' . $previousAutoInstall); } if ($previousCommandTimeout === false) { putenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS'); } else { putenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS=' . $previousCommandTimeout); } if ($previousDownloadTimeout === false) { putenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS'); } else { putenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS=' . $previousDownloadTimeout); } } }); it('wires MinIO replication through routes and bootstrap snapshots', function (): void { $manager = file_get_contents(app_path('classes/replication_manager.php')); $routes = file_get_contents(app_path('routes/superuserReplicationRoute.php')); $openapi = file_get_contents(app_path('openapi.yaml')); expect($manager)->toContain('private const KIND_MINIO'); expect($manager)->toContain('provisionMinioHost($host, $operationId)'); expect($manager)->toContain('promoteMinioHost($host)'); expect($manager)->toContain('testMinioHost($host)'); expect($manager)->toContain('minioTargetFreeBytes($host)'); expect($manager)->toContain('minioRequiredFreeBytes'); expect($manager)->toContain('minioReplicationConfiguredForHosts($primary, $host)'); expect($manager)->toContain("'--priority',"); expect($manager)->toContain('minioReplicationTransferLimitArgs'); expect($manager)->toContain('MINIO_PROGRESS_SCAN_INTERVAL_SECONDS'); expect($manager)->toContain('MINIO_INCOMPLETE_PROGRESS_MAX_PERCENT'); expect($manager)->toContain('MINIO_MC_COMMAND_TIMEOUT_SECONDS'); expect($manager)->toContain('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS'); expect($manager)->toContain('proc_terminate($process'); expect($manager)->toContain("'connect_timeout' => self::MINIO_S3_CONNECT_TIMEOUT_SECONDS"); expect($manager)->toContain("'retries' => 0"); expect($manager)->toContain('&& $forceStorageScan;'); expect($manager)->not->toContain('$isPrimary || $forceStorageScan'); expect($manager)->toContain('sanitizePublicLastStatus'); expect($manager)->toContain('MinIO primary object-scan timeouts do not indicate primary availability failure.'); expect($manager)->toContain('minioProvisionProgress($status)'); expect($manager)->toContain('MinIO replica is syncing. Copied'); expect($manager)->toContain("['mb', '--with-lock', '--ignore-existing', 'target/' . \$bucket]"); expect($manager)->toContain('repairMinioTargetBucketObjectLockIfEmpty($configDir, $target, $bucket, $message)'); expect($manager)->toContain('minioBucketHasObjects($target, $bucket)'); expect($manager)->toContain("'skip_storage_scan' => true"); expect($manager)->toContain('lastStatusReplicationPercent($host, 5.0)'); expect($manager)->toContain('completeReadyMinioProvisionOperation'); expect($manager)->toContain('MinIO replication target is caught up.'); expect($manager)->toContain('shouldAdvanceActiveProvisionDuringRefresh'); expect($manager)->toContain('provisionHost((string)$host[\'kind\'], (int)$host[\'id\'])'); expect($manager)->toContain('stale targets do not keep a healthy current target below 100%'); $normalizedManager = str_replace("\r\n", "\n", $manager); expect($normalizedManager)->toContain("'replicate',\n 'status',\n 'source/' . \$bucket"); expect($normalizedManager)->toContain("'replicate',\n 'status',\n '--json',\n 'source/' . \$bucket"); expect($manager)->toContain('private function minioBucketStats('); expect($manager)->toContain("'minio' => ["); expect($routes)->toContain("/superuser/replication/minio"); expect($openapi)->toContain('enum: [database, redis, minio]'); expect($openapi)->toContain('endpoint:'); expect($openapi)->toContain('space_headroom_percent:'); }); it('provisions Redis replicas after a connectivity-only preflight and reports sync progress', function (): void { $content = file_get_contents(app_path('classes/replication_manager.php')); expect($content)->toContain('provisionRedisHost($host, $operationId)'); expect($content)->toContain("testRedisHost(array_merge(\$host, ['test_connectivity_only' => true]))"); expect($content)->toContain("executeRaw(['REPLICAOF', (string)\$primary['host'], (string)\$primary['port']])"); expect($content)->toContain("executeRaw(['CONFIG', 'REWRITE'])"); expect($content)->toContain('Redis replication was configured; waiting for the replica to catch up.'); expect($content)->toContain('$onlySyncBlockers'); expect($content)->toContain('redisProvisionProgress'); expect($content)->toContain('Redis replication is configured and syncing in the background.'); expect($content)->toContain('Redis replication is configured, but the replica is waiting for the primary link.'); }); it('keeps Redis promotion caught-up, durable, and metadata-safe', function (): void { $content = file_get_contents(app_path('classes/replication_manager.php')); expect($content)->toContain("application_write_freeze::freeze('Replication promotion in progress.'"); expect($content)->toContain("if (\$status['blockers'] !== [] || (float)\$status['replication_percent'] < 100.0)"); expect($content)->toContain("executeRaw(['REPLICAOF', 'NO', 'ONE'])"); expect($content)->toContain("executeRaw(['CONFIG', 'REWRITE'])"); expect($content)->toContain('switchPrimary(self::KIND_REDIS'); expect($content)->toContain('writeBootstrapSnapshot()'); }); it('does not require replica SQL threads before database provisioning configures them', function (): void { $content = file_get_contents(app_path('classes/replication_manager.php')); expect($content)->toContain("'test_connectivity_only' => true"); expect($content)->toContain("'connect_without_database' => \$usePreseededReplica"); expect($content)->toContain("'healthy' => \$status['blockers'] === []"); expect($content)->toContain("'message' => \$targetEngine === 'mariadb'"); expect($content)->toContain('Database replica status is not configured.'); expect($content)->toContain('Database replication IO thread is not running.'); expect($content)->toContain('Database replication SQL thread is not running.'); expect($content)->toContain("if (\$status !== [])"); }); it('keeps replication operation progress schema idempotent for existing installs', function (): void { $content = file_get_contents(app_path('classes/replication_schema_bootstrap.php')); expect($content)->toContain("ensureColumn('replication_operations', 'progress_percent'"); expect($content)->toContain("ensureColumn('replication_operations', 'message'"); expect($content)->toContain("ensureColumn('replication_operations', 'context_json'"); }); it('creates the generated replication user on the primary during provisioning', function (): void { $content = file_get_contents(app_path('classes/replication_manager.php')); expect($content)->toContain('$grantHosts = $this->databaseReplicationGrantHosts($host, $targetStatus);'); expect($content)->toContain('ensureDatabaseReplicationUser($primary, $replicationUser, $replicationPassword, $grantHosts)'); expect($content)->toContain('databaseDeniedAccountHostsFromText'); expect($content)->toContain('Access denied for user'); expect($content)->toContain('foreach ($grantHosts as $grantHost)'); expect($content)->toContain('shouldRepairDatabaseReplicationAccess'); expect($content)->toContain('repairDatabaseReplicationAccess'); expect($content)->toContain('shouldRepairDatabaseReplicationThreads'); expect($content)->toContain('repairDatabaseReplicationThreads'); expect($content)->toContain('refreshDatabaseReplicationConnection'); expect($content)->toContain("CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, MASTER_USER = '%s', MASTER_PASSWORD = '%s', MASTER_USE_GTID = slave_pos"); expect($content)->toContain("CHANGE REPLICATION SOURCE TO SOURCE_HOST = '%s', SOURCE_PORT = %d, SOURCE_USER = '%s', SOURCE_PASSWORD = '%s', SOURCE_AUTO_POSITION = 1"); expect($content)->toContain('restartDatabaseReplicationThreads'); expect($content)->toContain('databaseOnlyReplicationThreadBlockers'); expect($content)->toContain('databaseAccountHostGrantCandidates'); expect($content)->toContain('START SLAVE SQL_THREAD'); expect($content)->toContain('GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO'); }); it('extracts host-specific MariaDB replication account denials', function (): void { $extract = new ReflectionMethod(replication_manager::class, 'databaseDeniedAccountHostsFromText'); $extract->setAccessible(true); $normalize = new ReflectionMethod(replication_manager::class, 'normalizeDatabaseAccountHost'); $normalize->setAccessible(true); $candidates = new ReflectionMethod(replication_manager::class, 'databaseAccountHostGrantCandidates'); $candidates->setAccessible(true); expect($extract->invoke(null, "Access denied for user 'replication'@'10.0.1.13' (using password: YES)")) ->toBe(['10.0.1.13']); expect($extract->invoke(null, "Access denied for user 'replication'@'fd9c:738d:4130::d' (using password: YES)")) ->toBe(['fd9c:738d:4130::d']); expect($normalize->invoke(null, '10.0.1.13'))->toBe('10.0.1.13'); expect($normalize->invoke(null, 'bad host;drop'))->toBeNull(); expect($candidates->invoke(null, '10.0.1.13'))->toBe(['10.0.1.13', '10.0.1.%']); expect($candidates->invoke(null, 'fd9c:738d:4130::d'))->toBe(['fd9c:738d:4130::d', 'fd9c:738d:4130::%']); }); it('identifies stopped database replication threads as a restartable status', function (): void { $onlyThreadBlockers = new ReflectionMethod(replication_manager::class, 'databaseOnlyReplicationThreadBlockers'); $onlyThreadBlockers->setAccessible(true); expect($onlyThreadBlockers->invoke(null, [ 'Database replication IO and SQL threads must both be running.', 'Database replication IO thread is not running.', 'Database replication SQL thread is not running.', ]))->toBeTrue(); expect($onlyThreadBlockers->invoke(null, [ 'Database replication IO thread is not running.', "error reconnecting to master 'replication@23.88.23.183:5432'", ]))->toBeFalse(); }); it('supports MariaDB prerequisites without requiring Oracle MySQL variables', function (): void { $blockers = replication_manager::databasePrerequisiteBlockers([ 'server_version' => '11.8.6-MariaDB-ubu2404', 'log_bin' => 'ON', 'server_id' => '12', 'gtid_binlog_pos' => '0-12-42', ]); expect($blockers)->toBe([]); $quietServerBlockers = replication_manager::databasePrerequisiteBlockers([ 'server_version' => '11.8.6-MariaDB-ubu2404', 'log_bin' => 'ON', 'server_id' => '12', 'gtid_current_pos' => '', ]); expect($quietServerBlockers)->toBe([]); }); it('reports MariaDB-specific blockers when GTID or binary logging prerequisites are missing', function (): void { $blockers = replication_manager::databasePrerequisiteBlockers([ 'server_version' => '11.8.6-MariaDB-ubu2404', 'log_bin' => 'OFF', 'server_id' => '12', ]); expect($blockers)->toContain('MariaDB binary logging must be enabled.'); expect($blockers)->toContain('MariaDB GTID position must be available.'); expect($blockers)->not->toContain('Oracle MySQL 8.x is required for managed replication. Current server reports 11.8.6-MariaDB-ubu2404.'); });