copenhagentruckwash/api
1
0
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity
Files
448e0e50c24597026da0f764b611aa4232f9852c
api/services/nginx/app/tests/Unit/Replication/ReplicationManagerStatusTest.php
T
Jeppe B 2a6a86c9c3 Resolve backend Qodana critical and high findings (#314) 
Resolve recommended-profile Critical and High findings, retain narrow analyzer exceptions, and update the edge-broker WebSocket dependency to a non-vulnerable release.
2026-07-17 05:44:16 +02:00

687 lines
38 KiB
PHP
Raw Blame History

<?php
app_require('classes/replication_manager.php');
use classes\replication_manager;
it('computes MySQL GTID interval counts and coverage percentages', function (): void {
$source = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa:1-5:7-10,bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb:1-2';
$executed = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa:1-3:7-10,bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb:1';
expect(replication_manager::mysqlGtidIntervalCount($source))->toBe(11);
expect(replication_manager::mysqlGtidCoveragePercent($source, $executed))->toBe(72.73);
});
it('reports empty source GTID sets as caught up', function (): void {
expect(replication_manager::mysqlGtidCoveragePercent('', ''))->toBe(100.0);
});
it('computes Redis offset percentages safely', function (): void {
expect(replication_manager::redisOffsetPercent(1000, 750))->toBe(75.0);
expect(replication_manager::redisOffsetPercent(0, 0))->toBe(100.0);
expect(replication_manager::redisOffsetPercent(1000, 1250))->toBe(100.0);
expect(replication_manager::redisReplicationPercentFromInfo(
['master_repl_offset' => 1000],
['slave_repl_offset' => 750]
))->toBe(75.0);
expect(replication_manager::redisReplicationPercentFromInfo(
['master_repl_offset' => 1000],
['master_sync_in_progress' => '1', 'master_sync_total_bytes' => '1000', 'master_sync_left_bytes' => '250']
))->toBe(75.0);
expect(replication_manager::redisReplicationPercentFromInfo(
['master_repl_offset' => 1000],
['master_sync_in_progress' => '1']
))->toBe(5.0);
expect(replication_manager::redisProvisionProgress(0.0))->toBe(5.0);
expect(replication_manager::redisProvisionProgress(100.0, ['Redis replica link to primary is not up.']))->toBe(99.99);
expect(replication_manager::replicationHealthStatus(true, 'replica', 5.0, []))->toBe('degraded');
expect(replication_manager::replicationHealthStatus(true, 'replica', 5.0, [], false))->toBe('ok');
expect(replication_manager::replicationHealthStatus(true, 'replica', 100.0, []))->toBe('ok');
});
it('computes MariaDB GTID coverage by domain sequence', function (): void {
$source = '0-1-10,1-1-20';
$replica = '0-2-8,1-3-20';
expect(replication_manager::mariadbGtidCoveragePercent($source, $replica))->toBe(93.33);
expect(replication_manager::mariadbGtidCoveragePercent('', ''))->toBe(100.0);
});
it('normalizes public replication kind aliases', function (): void {
expect(replication_manager::normalizeKind('databases'))->toBe('database');
expect(replication_manager::normalizeKind('mysql'))->toBe('database');
expect(replication_manager::normalizeKind('redis'))->toBe('redis');
expect(replication_manager::normalizeKind('minio'))->toBe('minio');
expect(replication_manager::normalizeKind('s3'))->toBe('minio');
expect(replication_manager::normalizeKind('object-storage'))->toBe('minio');
});
it('generates replication-ready MariaDB compose templates without embedding secrets', function (): void {
$template = replication_manager::composeTemplate([
'kind' => 'database',
'role' => 'replica',
'service_name' => 'MariaDB Replica 2',
'database' => 'nnks_db',
'username' => 'nnks_db_user',
'host_port' => 5433,
'server_id' => 2,
]);
expect($template['kind'])->toBe('database');
expect($template['role'])->toBe('replica');
expect($template['service_name'])->toBe('mariadb-replica-2');
expect($template['compose'])->toContain('image: "mariadb:11"');
expect($template['compose'])->toContain('"--server-id=2"');
expect($template['compose'])->toContain('"--log-bin=/var/lib/mysql/mariadb-bin"');
expect($template['compose'])->toContain('"--binlog-format=ROW"');
expect($template['compose'])->toContain('"--gtid-strict-mode=ON"');
expect($template['compose'])->toContain('"--read-only=ON"');
expect($template['compose'])->toContain('"--replicate-ignore-table=nnks_db.logs"');
expect($template['compose'])->toContain('"--replicate-ignore-table=nnks_db.edge_gateway_log_entries"');
expect($template['compose'])->toContain('"--replicate-ignore-table=nnks_db.replication_status_snapshots"');
expect($template['compose'])->toContain('"--replicate-ignore-table=nnks_db.system_search_documents"');
expect($template['compose'])->toContain('"5433:3306"');
expect($template['compose'])->toContain('mariadb-replica-2-seed');
expect($template['compose'])->toContain('MARIADB_PRIMARY_ADMIN_PASSWORD');
expect($template['compose'])->toContain('mariadb-dump --host="$${MARIADB_PRIMARY_HOST}"');
expect($template['compose'])->toContain('--ignore-table="$${MARIADB_SEED_DATABASE}.logs"');
expect($template['compose'])->toContain('--ignore-table="$${MARIADB_SEED_DATABASE}.edge_gateway_log_entries"');
expect($template['compose'])->toContain('--ignore-table="$${MARIADB_SEED_DATABASE}.replication_status_snapshots"');
expect($template['compose'])->toContain('--ignore-table="$${MARIADB_SEED_DATABASE}.system_search_documents"');
expect($template['compose'])->toContain('--no-data "$${MARIADB_SEED_DATABASE}" "$${table}"');
expect($template['compose'])->toContain('touch "$${marker}"');
expect($template['compose'])->toContain('${MARIADB_ROOT_PASSWORD:?set MARIADB_ROOT_PASSWORD}');
expect($template['compose'])->not->toContain('<set-a-long-random-root-password>');
expect($template['env'])->toMatch('/MARIADB_ROOT_PASSWORD=[A-Za-z0-9_-]{32}/');
expect($template['env'])->toMatch('/MARIADB_PASSWORD=[A-Za-z0-9_-]{32}/');
expect($template['env'])->toContain('MARIADB_PRIMARY_ADMIN_PASSWORD=');
expect($template['credentials']['password'])->toMatch('/^[A-Za-z0-9_-]{32}$/');
expect($template['credentials']['admin_password'])->toMatch('/^[A-Za-z0-9_-]{32}$/');
expect($template['credentials']['port'])->toBe(5433);
expect($template['credentials']['allow_preseeded_replica'])->toBeTrue();
expect($template['seed_command'])->toContain('mariadb-dump');
expect($template['seed_command'])->toContain('--gtid');
expect($template['seed_command'])->toContain('--master-data=2');
expect($template['seed_command'])->toContain('--ignore-table=\'nnks_db.logs\'');
expect($template['seed_command'])->toContain('--ignore-table=\'nnks_db.edge_gateway_log_entries\'');
expect($template['seed_command'])->toContain('--ignore-table=\'nnks_db.replication_status_snapshots\'');
expect($template['seed_command'])->toContain('--ignore-table=\'nnks_db.system_search_documents\'');
});
it('can embed primary admin credentials in generated MariaDB replica env files', function (): void {
$template = replication_manager::composeTemplate([
'kind' => 'database',
'role' => 'replica',
'primary_admin_username' => 'primary-root',
'primary_admin_password' => 'primary-secret',
]);
expect($template['env'])->toContain('MARIADB_PRIMARY_ADMIN_USER=primary-root');
expect($template['env'])->toContain('MARIADB_PRIMARY_ADMIN_PASSWORD=primary-secret');
});
it('detects missing database tables before provisioning a preseeded replica', function (): void {
expect(replication_manager::missingDatabaseTables(
['customers', 'edge_gateways', 'orders'],
['customers', 'orders']
))->toBe(['edge_gateways']);
});
it('seeds MariaDB replicas in place instead of requiring container recreation', function (): void {
$content = file_get_contents(app_path('classes/replication_manager.php'));
expect($content)->toContain('advanceMariaDbReplicaSeed($operationId, $primary, $host, $target)');
expect($content)->toContain('MARIADB_SEED_BATCH_ROWS');
expect($content)->toContain('MARIADB_SEED_STEP_SECONDS');
expect($content)->toContain('activeOperationId($kind, $id, \'provision\')');
expect($content)->toContain('updateOperationProgress($operationId, $progress, $message, $context)');
expect($content)->toContain("application_write_freeze::freeze('MariaDB replica seed is copying data.'");
expect($content)->toContain('DROP DATABASE IF EXISTS');
expect($content)->toContain('CREATE DATABASE ');
expect($content)->toContain('SHOW CREATE TABLE');
expect($content)->toContain('SET GLOBAL gtid_slave_pos');
expect($content)->toContain('databasePrimaryKeyColumnNames($source, $primaryDatabase, $tableName)');
expect($content)->toContain('$target->begin_transaction()');
expect($content)->toContain("'connect_without_database' => \$usePreseededReplica");
});
it('keeps operational and derived tables schema-only during MariaDB seeding and replication', function (): void {
$content = file_get_contents(app_path('classes/replication_manager.php'));
expect($content)->toContain('MARIADB_SCHEMA_ONLY_TABLES');
expect($content)->toContain("'logs'");
expect($content)->toContain("'edge_gateway_log_entries'");
expect($content)->toContain("'replication_status_snapshots'");
expect($content)->toContain("'replication_operations'");
expect($content)->toContain("'replication_audit_logs'");
expect($content)->toContain("'system_search_documents'");
expect($content)->toContain("'skip_data' => \$skipData");
expect($content)->toContain('createMariaDbReplicaTable(');
expect($content)->toContain('SET GLOBAL replicate_ignore_table');
expect($content)->toContain('--replicate-ignore-table=');
expect($content)->toContain('mariaDbSchemaOnlyDumpIgnoreArgs');
expect($content)->toContain('mariaDbSchemaOnlySeedCommandIgnoreArgs');
expect($content)->toContain('databaseSchemaOnlyTablesWithRows');
expect($content)->toContain('schemaOnlyTablesContainRowsBlocker');
expect($content)->toContain('RESET SLAVE ALL');
expect($content)->toContain('mariaDbSeedContextRequiresFilterReset($context)');
});
it('allows failed replicas to be removed without allowing primary or healthy replica removal', function (): void {
expect(replication_manager::replicationHostCanBeRemoved(['role' => 'primary', 'status' => 'ok']))->toBeFalse();
expect(replication_manager::replicationHostCanBeRemoved(['role' => 'inactive', 'status' => 'inactive']))->toBeTrue();
expect(replication_manager::replicationHostCanBeRemoved(['role' => 'replica', 'status' => 'degraded']))->toBeTrue();
expect(replication_manager::replicationHostCanBeRemoved(['role' => 'replica', 'status' => 'ok']))->toBeFalse();
$content = file_get_contents(app_path('classes/replication_manager.php'));
expect($content)->toContain('coolify_manager::replicationHostCanBeRemoved');
expect($content)->toContain('coolify_manager::markTargetsRemovedForReplicationHost');
});
it('supports metadata-only replication host renames', function (): void {
$content = file_get_contents(app_path('classes/replication_manager.php'));
expect($content)->toContain('function renameHost(');
expect($content)->toContain('host_renamed');
expect($content)->toContain('coolify_manager::syncLabelForReplicationHost');
expect($content)->toContain('writeBootstrapSnapshot()');
expect($content)->toContain('Replication host label must be 128 characters or fewer.');
});
it('generates Redis replica compose templates with primary connection placeholders', function (): void {
$template = replication_manager::composeTemplate([
'kind' => 'redis',
'role' => 'replica',
'service_name' => 'Redis Replica',
'host_port' => 6380,
'primary_host' => 'redis-primary.internal',
'primary_port' => 6379,
]);
expect($template['kind'])->toBe('redis');
expect($template['compose'])->toContain('image: "redis:7"');
expect($template['compose'])->toContain('REDIS_PASSWORD: "${REDIS_PASSWORD:?set REDIS_PASSWORD}"');
expect($template['compose'])->toContain('REDIS_PRIMARY_HOST: "${REDIS_PRIMARY_HOST:?set REDIS_PRIMARY_HOST}"');
expect($template['compose'])->toContain('REDIS_PRIMARY_PORT: "${REDIS_PRIMARY_PORT:-6379}"');
expect($template['compose'])->toContain('${REDIS_PRIMARY_PASSWORD:?set REDIS_PRIMARY_PASSWORD}');
expect($template['compose'])->toContain('REDIS_PRIMARY_USERNAME: "${REDIS_PRIMARY_USERNAME:-}"');
expect($template['compose'])->toContain('if [ ! -f /data/redis.conf ]; then');
expect($template['compose'])->toContain('> /data/redis.conf');
expect($template['compose'])->toContain('exec redis-server /data/redis.conf');
expect($template['compose'])->toContain('echo "replicaof $$REDIS_PRIMARY_HOST $${REDIS_PRIMARY_PORT:-6379}"');
expect($template['compose'])->toContain('echo "masterauth $$REDIS_PRIMARY_PASSWORD"');
expect($template['compose'])->toContain('echo "masteruser $$REDIS_PRIMARY_USERNAME"');
expect($template['compose'])->toContain('redis-cli --no-auth-warning -a \"$${REDIS_PASSWORD}\" ping | grep PONG');
expect($template['compose'])->not->toContain($template['credentials']['password']);
expect($template['env'])->toMatch('/REDIS_PASSWORD=[A-Za-z0-9_-]{32}/');
expect($template['env'])->toContain('REDIS_PRIMARY_HOST=redis-primary.internal');
expect($template['env'])->toContain('REDIS_PRIMARY_PORT=6379');
expect($template['env'])->toContain("REDIS_PRIMARY_PASSWORD=\n");
expect($template['env'])->toContain("REDIS_PRIMARY_USERNAME=\n");
expect($template['credentials']['password'])->toMatch('/^[A-Za-z0-9_-]{32}$/');
});
it('generates MinIO replica compose templates without embedding secrets', function (): void {
$template = replication_manager::composeTemplate([
'kind' => 'minio',
'role' => 'replica',
'service_name' => 'MinIO Replica 1',
'host' => 'node2.truckwash.dk',
'scheme' => 'http',
'host_port' => 9010,
'console_port' => 9011,
'buckets' => ['attachments', 'uploads'],
]);
expect($template['kind'])->toBe('minio');
expect($template['engine'])->toBe('minio');
expect($template['service_name'])->toBe('minio-replica-1');
expect($template['host_port'])->toBe(9010);
expect($template['console_port'])->toBe(9011);
expect($template['compose'])->toContain('image: "minio/minio:latest"');
expect($template['compose'])->toContain('MINIO_ROOT_USER: "${MINIO_ROOT_USER:?set MINIO_ROOT_USER}"');
expect($template['compose'])->toContain('MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:?set MINIO_ROOT_PASSWORD}"');
expect($template['compose'])->toContain('MINIO_SERVER_URL: "${MINIO_SERVER_URL:-}"');
expect($template['compose'])->toContain('MINIO_BROWSER_REDIRECT_URL: "${MINIO_BROWSER_REDIRECT_URL:-}"');
expect($template['compose'])->toContain('"9010:9000"');
expect($template['compose'])->toContain('"9011:9001"');
expect($template['compose'])->toContain('mc mb --with-lock --ignore-existing');
expect($template['compose'])->toContain('mc version enable');
expect($template['compose'])->toContain('MINIO_PRIMARY_ENDPOINT');
expect($template['compose'])->not->toContain($template['credentials']['password']);
expect($template['env'])->toMatch('/MINIO_ROOT_USER=twminio[a-f0-9]{24}/');
expect($template['env'])->toMatch('/MINIO_ROOT_PASSWORD=[A-Za-z0-9_-]{32}/');
expect($template['env'])->toContain('MINIO_SERVER_URL=http://node2.truckwash.dk:9010');
expect($template['env'])->toContain('MINIO_BROWSER_REDIRECT_URL=http://node2.truckwash.dk:9011');
expect($template['env'])->toContain('MINIO_BUCKETS=attachments,uploads');
expect($template['env'])->toContain('MINIO_REPLICATION_TRANSFER_LIMIT=25Mi');
expect($template['env'])->toContain('MINIO_PRIMARY_ENDPOINT=');
expect($template['credentials']['username'])->toMatch('/^twminio[a-f0-9]{24}$/');
expect($template['credentials']['scheme'])->toBe('http');
expect($template['credentials']['buckets'])->toBe(['attachments', 'uploads']);
expect($template['credentials']['replication_transfer_limit'])->toBe('25Mi');
expect($template['credentials']['space_headroom_percent'])->toBe(20.0);
});
it('keeps MinIO backup replicas bounded to the recent backup window', function (): void {
$template = replication_manager::composeTemplate([
'kind' => 'minio',
'role' => 'replica',
'service_name' => 'minio-replica-1',
'buckets' => ['backups', 'uploads'],
]);
$content = file_get_contents(app_path('classes/replication_manager.php'));
expect(replication_manager::minioBackupReplicaRetentionDays())->toBe(30);
expect($template['compose'])->toContain('mc ilm rule add --expire-days "30" --noncurrent-expire-days "30"');
expect($template['env'])->toContain('MINIO_BACKUP_REPLICA_RETENTION_DAYS=30');
expect($template['steps'])->toContain('The backups bucket is retained on replicas for 30 days; other buckets are fully replicated.');
expect($content)->not->toContain('seedMinioReplicaBackupWindow');
expect($content)->not->toContain("'--newer-than'");
expect($content)->toContain("'--limit-upload'");
expect($content)->toContain("'--limit-download'");
expect($content)->toContain("? 'delete,delete-marker'");
expect($content)->toContain('putBucketLifecycleConfiguration');
expect($content)->toContain('listObjectVersions');
expect($content)->toContain('minioBackupReplicaRetentionConfigured');
expect(replication_manager::minioBackupRetentionBlockers([
'buckets' => [
['name' => 'backups', 'expired_objects' => 2],
],
]))->toBe([
'MinIO backup replica contains 2 backup objects older than 30 days. Run provisioning to prune retained backups.',
]);
});
it('prefills MinIO replica compose primary values from current config when available', function (): void {
$previousMinio = $GLOBALS['MINIO'] ?? null;
$GLOBALS['MINIO'] = [
'endpoint' => 'https://minio-primary.internal:9000',
'access_key' => 'primary-access',
'secret_key' => 'primary-secret',
];
try {
$template = replication_manager::composeTemplate([
'kind' => 'minio',
'role' => 'replica',
'service_name' => 'minio-replica-1',
]);
expect($template['env'])->toContain('MINIO_PRIMARY_ENDPOINT=https://minio-primary.internal:9000');
expect($template['env'])->toContain('MINIO_PRIMARY_ACCESS_KEY=primary-access');
expect($template['env'])->toContain('MINIO_PRIMARY_SECRET_KEY=primary-secret');
expect($template['compose'])->not->toContain('primary-secret');
} finally {
if ($previousMinio === null) {
unset($GLOBALS['MINIO']);
} else {
$GLOBALS['MINIO'] = $previousMinio;
}
}
});
it('computes MinIO free-space and catch-up math safely', function (): void {
expect(replication_manager::minioRequiredFreeBytes(1000))->toBe(1200);
expect(replication_manager::minioByteReplicationPercent(1000, 750))->toBe(75.0);
expect(replication_manager::minioByteReplicationPercent(0, 0))->toBe(100.0);
expect(replication_manager::minioProvisionProgress([
'replication_percent' => 3.2,
'raw' => ['storage' => ['measured' => true]],
]))->toBe(3.2);
expect(replication_manager::minioProvisionProgress([
'replication_percent' => 0,
'raw' => ['storage' => ['measured' => false]],
]))->toBe(5.0);
expect(replication_manager::minioProvisionProgress([
'replication_percent' => 2.5,
'raw' => ['progress_source' => 'minio_replicate_status'],
]))->toBe(2.5);
expect(replication_manager::minioProvisionProgress([
'replication_percent' => 100,
'raw' => ['storage' => ['measured' => true]],
]))->toBe(100.0);
expect(replication_manager::minioProvisionProgress([
'replication_percent' => 100,
'blockers' => ['MinIO replica has not caught up.'],
'raw' => ['storage' => ['measured' => true]],
]))->toBe(99.9);
expect(replication_manager::minioReplicationProgressFromStatusOutput([
'target' => [
'replicated' => ['size' => 750],
'pending' => ['size' => 250],
],
])['replication_percent'])->toBe(75.0);
expect(replication_manager::minioReplicationProgressFromStatusOutput([
'objects' => [
'completed' => 9,
'pending' => 1,
],
])['replication_percent'])->toBe(90.0);
expect(replication_manager::minioReplicationProgressFromStatusOutput([
'status' => 'complete',
])['replication_percent'])->toBe(100.0);
expect(replication_manager::minioReplicationProgressFromStatusOutput(
'{"replicatedSize": 750, "pendingSize": 250}'
)['replication_percent'])->toBe(75.0);
expect(replication_manager::minioReplicationProgressFromStatusOutput(
'{"replicaSize": 750, "pendingSize": 250}'
)['replication_percent'])->toBe(75.0);
expect(replication_manager::minioReplicationProgressFromStatusOutput(
'target-a: 100%, target-b: 5%'
)['replication_percent'])->toBe(5.0);
expect(replication_manager::minioReplicationProgressFromStatusOutput(
'target-a: 100%, target-b: 100%'
)['replication_percent'])->toBe(100.0);
expect(replication_manager::minioReplicationProgressFromStatusOutput([
'completedReplicationSize' => 1000,
'queued' => [
'curr' => ['count' => 0, 'bytes' => 0],
'avg' => ['count' => 42, 'bytes' => 25000000],
'peak' => ['count' => 100, 'bytes' => 50000000],
],
])['replication_percent'])->toBe(100.0);
expect(replication_manager::minioBucketCountsTowardCatchUp('uploads'))->toBeTrue();
expect(replication_manager::minioBucketCountsTowardCatchUp('backups'))->toBeFalse();
$boundedBackupProgress = replication_manager::minioCatchUpProgressFromBucketStatuses([
'uploads' => ['replication_percent' => 100.0, 'blockers' => [], 'stats' => []],
'backups' => ['replication_percent' => 99.74, 'blockers' => ['MinIO replica has not caught up.'], 'stats' => []],
]);
expect($boundedBackupProgress['replication_percent'])->toBe(100.0);
expect($boundedBackupProgress['blockers'])->toBe([]);
expect($boundedBackupProgress['ignored_buckets'])->toBe(['backups']);
$onlyBoundedBackupProgress = replication_manager::minioCatchUpProgressFromBucketStatuses([
'backups' => ['replication_percent' => 5.0, 'blockers' => ['MinIO replica has not caught up.'], 'stats' => []],
]);
expect($onlyBoundedBackupProgress['replication_percent'])->toBe(100.0);
expect($onlyBoundedBackupProgress['basis'])->toBe('bounded_retention_only');
$liveQueueProgress = replication_manager::minioCatchUpProgressFromBucketStatuses([
'uploads' => [
'replication_percent' => 99.98,
'blockers' => ['MinIO replica has not caught up.'],
'stats' => [
'completed_bytes' => 53149249190,
'pending_bytes' => 3439936,
'failed_bytes' => 0,
'total_bytes' => 0,
'completed_count' => 199368,
'pending_count' => 7,
'failed_count' => 0,
'total_count' => 0,
],
],
]);
expect($liveQueueProgress['replication_percent'])->toBe(100.0);
expect($liveQueueProgress['blockers'])->toBe([]);
expect($liveQueueProgress['live_tolerance']['within_tolerance'])->toBeTrue();
expect(replication_manager::minioSpaceBlockers(1199, 1200))->toContain('MinIO target does not have enough free space. Required 1200 bytes, available 1199 bytes.');
expect(replication_manager::minioSpaceBlockers(null, 1200))->toBe([]);
expect(replication_manager::minioSpaceBlockers(1200, 1200))->toBe([]);
expect(replication_manager::minioAvailableBytesFromAdminInfo([
'servers' => [
['drives' => [['availableSpace' => 4096]]],
],
]))->toBe(4096);
expect(replication_manager::normalizeMinioBuckets('Attachments, uploads backups'))->toBe(['attachments', 'uploads', 'backups']);
expect(replication_manager::minioDefaultReplicationTransferLimit())->toBe('25Mi');
expect(replication_manager::normalizeMinioTransferLimit('25MiB/s'))->toBe('25Mi');
expect(replication_manager::normalizeMinioTransferLimit('100 MB'))->toBe('100M');
expect(replication_manager::normalizeMinioTransferLimit('0'))->toBe('');
});
it('allows the MinIO client binary to be configured explicitly', function (): void {
$previous = getenv('MINIO_MC_BINARY');
putenv('MINIO_MC_BINARY=/opt/minio/mc');
try {
$method = new ReflectionMethod(replication_manager::class, 'minioClientBinary');
expect($method->invoke(null))->toBe('/opt/minio/mc');
} finally {
if ($previous === false) {
putenv('MINIO_MC_BINARY');
} else {
putenv('MINIO_MC_BINARY=' . $previous);
}
}
});
it('supports MinIO client runtime fallback configuration', function (): void {
$previousDownloadUrl = getenv('MINIO_MC_DOWNLOAD_URL');
$previousAutoInstall = getenv('MINIO_MC_AUTO_INSTALL');
$previousCommandTimeout = getenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS');
$previousDownloadTimeout = getenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS');
putenv('MINIO_MC_DOWNLOAD_URL=https://example.test/mc');
putenv('MINIO_MC_AUTO_INSTALL=0');
putenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS=3');
putenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS=4');
try {
$downloadUrl = new ReflectionMethod(replication_manager::class, 'minioClientDownloadUrl');
$autoInstall = new ReflectionMethod(replication_manager::class, 'minioClientAutoInstallEnabled');
$commandTimeout = new ReflectionMethod(replication_manager::class, 'minioClientCommandTimeoutSeconds');
$downloadTimeout = new ReflectionMethod(replication_manager::class, 'minioClientDownloadTimeoutSeconds');
$commandLabel = new ReflectionMethod(replication_manager::class, 'minioClientCommandLabel');
expect($downloadUrl->invoke(null))->toBe('https://example.test/mc');
expect($autoInstall->invoke(null))->toBeFalse();
expect($commandTimeout->invoke(null))->toBe(3);
expect($downloadTimeout->invoke(null))->toBe(4);
expect($commandLabel->invoke(null, [
'alias',
'set',
'target',
'http://minio.example.test:9010',
'access-key',
'secret-key',
]))->toContain('[redacted]');
expect($commandLabel->invoke(null, [
'alias',
'set',
'target',
'http://minio.example.test:9010',
'access-key',
'secret-key',
]))->not->toContain('secret-key');
} finally {
if ($previousDownloadUrl === false) {
putenv('MINIO_MC_DOWNLOAD_URL');
} else {
putenv('MINIO_MC_DOWNLOAD_URL=' . $previousDownloadUrl);
}
if ($previousAutoInstall === false) {
putenv('MINIO_MC_AUTO_INSTALL');
} else {
putenv('MINIO_MC_AUTO_INSTALL=' . $previousAutoInstall);
}
if ($previousCommandTimeout === false) {
putenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS');
} else {
putenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS=' . $previousCommandTimeout);
}
if ($previousDownloadTimeout === false) {
putenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS');
} else {
putenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS=' . $previousDownloadTimeout);
}
}
});
it('wires MinIO replication through routes and bootstrap snapshots', function (): void {
$manager = file_get_contents(app_path('classes/replication_manager.php'));
$routes = file_get_contents(app_path('routes/superuserReplicationRoute.php'));
$openapi = file_get_contents(app_path('openapi.yaml'));
expect($manager)->toContain('private const KIND_MINIO');
expect($manager)->toContain('provisionMinioHost($host, $operationId)');
expect($manager)->toContain('promoteMinioHost($host)');
expect($manager)->toContain('testMinioHost($host)');
expect($manager)->toContain('minioTargetFreeBytes($host)');
expect($manager)->toContain('minioRequiredFreeBytes');
expect($manager)->toContain('minioReplicationConfiguredForHosts($primary, $host)');
expect($manager)->toContain("'--priority',");
expect($manager)->toContain('minioReplicationTransferLimitArgs');
expect($manager)->toContain('MINIO_PROGRESS_SCAN_INTERVAL_SECONDS');
expect($manager)->toContain('MINIO_INCOMPLETE_PROGRESS_MAX_PERCENT');
expect($manager)->toContain('MINIO_MC_COMMAND_TIMEOUT_SECONDS');
expect($manager)->toContain('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS');
expect($manager)->toContain('proc_terminate($process');
expect($manager)->toContain("'connect_timeout' => self::MINIO_S3_CONNECT_TIMEOUT_SECONDS");
expect($manager)->toContain("'retries' => 0");
expect($manager)->toContain('&& $forceStorageScan;');
expect($manager)->not->toContain('$isPrimary || $forceStorageScan');
expect($manager)->toContain('sanitizePublicLastStatus');
expect($manager)->toContain('MinIO primary object-scan timeouts do not indicate primary availability failure.');
expect($manager)->toContain('minioProvisionProgress($status)');
expect($manager)->toContain('MinIO replica is syncing. Copied');
expect($manager)->toContain("['mb', '--with-lock', '--ignore-existing', 'target/' . \$bucket]");
expect($manager)->toContain('repairMinioTargetBucketObjectLockIfEmpty($configDir, $target, $bucket, $message)');
expect($manager)->toContain('minioBucketHasObjects($target, $bucket)');
expect($manager)->toContain("'skip_storage_scan' => true");
expect($manager)->toContain('lastStatusReplicationPercent($host, 5.0)');
expect($manager)->toContain('completeReadyMinioProvisionOperation');
expect($manager)->toContain('MinIO replication target is caught up.');
expect($manager)->toContain('shouldAdvanceActiveProvisionDuringRefresh');
expect($manager)->toContain('provisionHost((string)$host[\'kind\'], (int)$host[\'id\'])');
expect($manager)->toContain('stale targets do not keep a healthy current target below 100%');
$normalizedManager = str_replace("\r\n", "\n", $manager);
expect($normalizedManager)->toContain("'replicate',\n 'status',\n 'source/' . \$bucket");
expect($normalizedManager)->toContain("'replicate',\n 'status',\n '--json',\n 'source/' . \$bucket");
expect($manager)->toContain('private function minioBucketStats(');
expect($manager)->toContain("'minio' => [");
expect($routes)->toContain("/superuser/replication/minio");
expect($openapi)->toContain('enum: [database, redis, minio]');
expect($openapi)->toContain('endpoint:');
expect($openapi)->toContain('space_headroom_percent:');
});
it('provisions Redis replicas after a connectivity-only preflight and reports sync progress', function (): void {
$content = file_get_contents(app_path('classes/replication_manager.php'));
expect($content)->toContain('provisionRedisHost($host, $operationId)');
expect($content)->toContain("testRedisHost(array_merge(\$host, ['test_connectivity_only' => true]))");
expect($content)->toContain("executeRaw(['REPLICAOF', (string)\$primary['host'], (string)\$primary['port']])");
expect($content)->toContain("executeRaw(['CONFIG', 'REWRITE'])");
expect($content)->toContain('Redis replication was configured; waiting for the replica to catch up.');
expect($content)->toContain('$onlySyncBlockers');
expect($content)->toContain('redisProvisionProgress');
expect($content)->toContain('Redis replication is configured and syncing in the background.');
expect($content)->toContain('Redis replication is configured, but the replica is waiting for the primary link.');
});
it('keeps Redis promotion caught-up, durable, and metadata-safe', function (): void {
$content = file_get_contents(app_path('classes/replication_manager.php'));
expect($content)->toContain("application_write_freeze::freeze('Replication promotion in progress.'");
expect($content)->toContain("if (\$status['blockers'] !== [] || (float)\$status['replication_percent'] < 100.0)");
expect($content)->toContain("executeRaw(['REPLICAOF', 'NO', 'ONE'])");
expect($content)->toContain("executeRaw(['CONFIG', 'REWRITE'])");
expect($content)->toContain('switchPrimary(self::KIND_REDIS');
expect($content)->toContain('writeBootstrapSnapshot()');
});
it('does not require replica SQL threads before database provisioning configures them', function (): void {
$content = file_get_contents(app_path('classes/replication_manager.php'));
expect($content)->toContain("'test_connectivity_only' => true");
expect($content)->toContain("'connect_without_database' => \$usePreseededReplica");
expect($content)->toContain("'healthy' => \$status['blockers'] === []");
expect($content)->toContain("'message' => \$targetEngine === 'mariadb'");
expect($content)->toContain('Database replica status is not configured.');
expect($content)->toContain('Database replication IO thread is not running.');
expect($content)->toContain('Database replication SQL thread is not running.');
expect($content)->toContain("if (\$status !== [])");
});
it('keeps replication operation progress schema idempotent for existing installs', function (): void {
$content = file_get_contents(app_path('classes/replication_schema_bootstrap.php'));
expect($content)->toContain("ensureColumn('replication_operations', 'progress_percent'");
expect($content)->toContain("ensureColumn('replication_operations', 'message'");
expect($content)->toContain("ensureColumn('replication_operations', 'context_json'");
});
it('creates the generated replication user on the primary during provisioning', function (): void {
$content = file_get_contents(app_path('classes/replication_manager.php'));
expect($content)->toContain('$grantHosts = $this->databaseReplicationGrantHosts($host, $targetStatus);');
expect($content)->toContain('ensureDatabaseReplicationUser($primary, $replicationUser, $replicationPassword, $grantHosts)');
expect($content)->toContain('databaseDeniedAccountHostsFromText');
expect($content)->toContain('Access denied for user');
expect($content)->toContain('foreach ($grantHosts as $grantHost)');
expect($content)->toContain('shouldRepairDatabaseReplicationAccess');
expect($content)->toContain('repairDatabaseReplicationAccess');
expect($content)->toContain('shouldRepairDatabaseReplicationThreads');
expect($content)->toContain('repairDatabaseReplicationThreads');
expect($content)->toContain('refreshDatabaseReplicationConnection');
expect($content)->toContain("CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, MASTER_USER = '%s', MASTER_PASSWORD = '%s', MASTER_USE_GTID = slave_pos");
expect($content)->toContain("CHANGE REPLICATION SOURCE TO SOURCE_HOST = '%s', SOURCE_PORT = %d, SOURCE_USER = '%s', SOURCE_PASSWORD = '%s', SOURCE_AUTO_POSITION = 1");
expect($content)->toContain('restartDatabaseReplicationThreads');
expect($content)->toContain('databaseOnlyReplicationThreadBlockers');
expect($content)->toContain('databaseAccountHostGrantCandidates');
expect($content)->toContain('START SLAVE SQL_THREAD');
expect($content)->toContain('GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO');
});
it('extracts host-specific MariaDB replication account denials', function (): void {
$extract = new ReflectionMethod(replication_manager::class, 'databaseDeniedAccountHostsFromText');
$normalize = new ReflectionMethod(replication_manager::class, 'normalizeDatabaseAccountHost');
$candidates = new ReflectionMethod(replication_manager::class, 'databaseAccountHostGrantCandidates');
expect($extract->invoke(null, "Access denied for user 'replication'@'10.0.1.13' (using password: YES)"))
->toBe(['10.0.1.13']);
expect($extract->invoke(null, "Access denied for user 'replication'@'fd9c:738d:4130::d' (using password: YES)"))
->toBe(['fd9c:738d:4130::d']);
expect($normalize->invoke(null, '10.0.1.13'))->toBe('10.0.1.13');
expect($normalize->invoke(null, 'bad host;drop'))->toBeNull();
expect($candidates->invoke(null, '10.0.1.13'))->toBe(['10.0.1.13', '10.0.1.%']);
expect($candidates->invoke(null, 'fd9c:738d:4130::d'))->toBe(['fd9c:738d:4130::d', 'fd9c:738d:4130::%']);
});
it('identifies stopped database replication threads as a restartable status', function (): void {
$onlyThreadBlockers = new ReflectionMethod(replication_manager::class, 'databaseOnlyReplicationThreadBlockers');
expect($onlyThreadBlockers->invoke(null, [
'Database replication IO and SQL threads must both be running.',
'Database replication IO thread is not running.',
'Database replication SQL thread is not running.',
]))->toBeTrue();
expect($onlyThreadBlockers->invoke(null, [
'Database replication IO thread is not running.',
"error reconnecting to master 'replication@23.88.23.183:5432'",
]))->toBeFalse();
});
it('supports MariaDB prerequisites without requiring Oracle MySQL variables', function (): void {
$blockers = replication_manager::databasePrerequisiteBlockers([
'server_version' => '11.8.6-MariaDB-ubu2404',
'log_bin' => 'ON',
'server_id' => '12',
'gtid_binlog_pos' => '0-12-42',
]);
expect($blockers)->toBe([]);
$quietServerBlockers = replication_manager::databasePrerequisiteBlockers([
'server_version' => '11.8.6-MariaDB-ubu2404',
'log_bin' => 'ON',
'server_id' => '12',
'gtid_current_pos' => '',
]);
expect($quietServerBlockers)->toBe([]);
});
it('reports MariaDB-specific blockers when GTID or binary logging prerequisites are missing', function (): void {
$blockers = replication_manager::databasePrerequisiteBlockers([
'server_version' => '11.8.6-MariaDB-ubu2404',
'log_bin' => 'OFF',
'server_id' => '12',
]);
expect($blockers)->toContain('MariaDB binary logging must be enabled.');
expect($blockers)->toContain('MariaDB GTID position must be available.');
expect($blockers)->not->toContain('Oracle MySQL 8.x is required for managed replication. Current server reports 11.8.6-MariaDB-ubu2404.');
});
Reference in New Issue View Git Blame Copy Permalink