2850 lines
113 KiB
PHP
2850 lines
113 KiB
PHP
<?php
|
|
|
|
namespace classes;
|
|
|
|
use RuntimeException;
|
|
use Throwable;
|
|
|
|
class coolify_manager
|
|
{
|
|
private const KINDS = ['database', 'redis', 'minio'];
|
|
private const RESOURCE_TYPE_SERVICE = 'service';
|
|
private const DEFAULT_PUBLIC_GATEWAY_HOST = 'api-v2.truckwash.io';
|
|
private const REQUIRED_LOAD_BALANCER_SERVICES = [
|
|
['protocol' => 'http', 'listen_port' => 80, 'destination_port' => 80],
|
|
['protocol' => 'tcp', 'listen_port' => 443, 'destination_port' => 443],
|
|
];
|
|
|
|
/** @var callable|null */
|
|
private $clientFactory;
|
|
/** @var callable|null */
|
|
private $hetznerClientFactory;
|
|
private bool $schemaEnsured = false;
|
|
|
|
public function __construct(?callable $clientFactory = null, ?callable $hetznerClientFactory = null)
|
|
{
|
|
$this->clientFactory = $clientFactory;
|
|
$this->hetznerClientFactory = $hetznerClientFactory;
|
|
}
|
|
|
|
public function summary(): array
|
|
{
|
|
$this->ensureSchema();
|
|
|
|
return [
|
|
'generated_at' => date('c'),
|
|
'instances' => $this->listInstances(),
|
|
'targets' => $this->listTargets(),
|
|
'availability' => $this->availabilitySummary(),
|
|
'load_balancer' => $this->loadBalancerSummary(),
|
|
];
|
|
}
|
|
|
|
public function listInstances(): array
|
|
{
|
|
if (!coolify_schema_bootstrap::tablesExist()) {
|
|
return [];
|
|
}
|
|
|
|
return array_map(
|
|
fn(array $instance): array => $this->publicInstance($instance),
|
|
$this->selectRows('SELECT * FROM coolify_instances WHERE deleted_at IS NULL ORDER BY id')
|
|
);
|
|
}
|
|
|
|
public function createInstance(array $input, ?int $actorUserId = null): array
|
|
{
|
|
$this->ensureSchema();
|
|
$label = trim((string)($input['label'] ?? 'Coolify'));
|
|
$baseUrl = trim((string)($input['base_url'] ?? $input['url'] ?? ''));
|
|
$apiToken = (string)($input['api_token'] ?? $input['token'] ?? '');
|
|
if ($label === '' || $baseUrl === '' || trim($apiToken) === '') {
|
|
throw new RuntimeException('Coolify label, base URL, and API token are required.');
|
|
}
|
|
|
|
$this->execute(
|
|
"INSERT INTO coolify_instances (
|
|
label, base_url, api_token_secret, default_project_uuid, default_environment_uuid,
|
|
default_environment_name, default_server_uuid, default_destination_uuid
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
|
'ssssssss',
|
|
[
|
|
$label,
|
|
rtrim($baseUrl, '/'),
|
|
replication_secret_box::encrypt($apiToken),
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
]
|
|
);
|
|
|
|
$id = $this->insertId();
|
|
$this->setModuleEnabled(true);
|
|
$this->audit(null, $id, null, 'instance_created', $actorUserId, 'info', [
|
|
'label' => $label,
|
|
'base_url' => $baseUrl,
|
|
]);
|
|
|
|
return $this->publicInstance($this->getInstance($id));
|
|
}
|
|
|
|
public function testInstance(int $instanceId, ?int $actorUserId = null): array
|
|
{
|
|
$this->ensureSchema();
|
|
$instance = $this->getInstance($instanceId);
|
|
$startedAt = microtime(true);
|
|
|
|
try {
|
|
$client = $this->clientForInstance($instance);
|
|
$health = $client->healthcheck();
|
|
$version = [];
|
|
try {
|
|
$version = $client->version();
|
|
} catch (Throwable) {
|
|
}
|
|
|
|
$result = [
|
|
'ok' => true,
|
|
'status' => 'ok',
|
|
'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2),
|
|
'health' => $health,
|
|
'version' => $version,
|
|
'checked_at' => date('c'),
|
|
];
|
|
|
|
$this->execute(
|
|
"UPDATE coolify_instances SET status = 'ok', last_checked_at = NOW(), last_error = NULL WHERE id = ?",
|
|
'i',
|
|
[$instanceId]
|
|
);
|
|
$this->audit(null, $instanceId, null, 'instance_tested', $actorUserId, 'info', $result);
|
|
|
|
return [
|
|
'instance' => $this->publicInstance($this->getInstance($instanceId)),
|
|
'test' => $result,
|
|
];
|
|
} catch (Throwable $throwable) {
|
|
$result = [
|
|
'ok' => false,
|
|
'status' => 'down',
|
|
'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2),
|
|
'error' => $throwable->getMessage(),
|
|
'checked_at' => date('c'),
|
|
];
|
|
$this->execute(
|
|
"UPDATE coolify_instances SET status = 'down', last_checked_at = NOW(), last_error = ? WHERE id = ?",
|
|
'si',
|
|
[$throwable->getMessage(), $instanceId]
|
|
);
|
|
$this->audit(null, $instanceId, null, 'instance_test_failed', $actorUserId, 'warning', $result);
|
|
|
|
return [
|
|
'instance' => $this->publicInstance($this->getInstance($instanceId)),
|
|
'test' => $result,
|
|
];
|
|
}
|
|
}
|
|
|
|
public function discoverInstancePlacement(int $instanceId): array
|
|
{
|
|
$this->ensureSchema();
|
|
$instance = $this->getInstance($instanceId);
|
|
$client = $this->clientForInstance($instance);
|
|
$errors = [];
|
|
|
|
$servers = [];
|
|
try {
|
|
$servers = array_map(
|
|
fn(array $server): array => $this->publicPlacementServer($server),
|
|
$this->coolifyCollection($client->listServers())
|
|
);
|
|
} catch (Throwable $throwable) {
|
|
$errors['servers'] = $throwable->getMessage();
|
|
}
|
|
|
|
$projects = [];
|
|
$environments = [];
|
|
try {
|
|
$projects = array_map(
|
|
fn(array $project): array => $this->publicPlacementProject($project),
|
|
$this->coolifyCollection($client->listProjects())
|
|
);
|
|
|
|
foreach ($projects as $project) {
|
|
$projectUuid = (string)($project['uuid'] ?? '');
|
|
if ($projectUuid === '') {
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
foreach ($this->coolifyCollection($client->listProjectEnvironments($projectUuid)) as $environment) {
|
|
$environments[] = $this->publicPlacementEnvironment($environment, $project);
|
|
}
|
|
} catch (Throwable $throwable) {
|
|
$errors['environments'][$projectUuid] = $throwable->getMessage();
|
|
}
|
|
}
|
|
} catch (Throwable $throwable) {
|
|
$errors['projects'] = $throwable->getMessage();
|
|
}
|
|
|
|
return [
|
|
'generated_at' => date('c'),
|
|
'instance' => $this->publicInstance($instance),
|
|
'servers' => array_values(array_filter($servers, static fn(array $server): bool => (string)($server['uuid'] ?? '') !== '')),
|
|
'projects' => array_values(array_filter($projects, static fn(array $project): bool => (string)($project['uuid'] ?? '') !== '')),
|
|
'environments' => array_values(array_filter($environments, static fn(array $environment): bool => (string)($environment['name'] ?? $environment['uuid'] ?? '') !== '')),
|
|
'destination_discovery_supported' => false,
|
|
'errors' => $errors,
|
|
];
|
|
}
|
|
|
|
public function listTargets(?string $kind = null): array
|
|
{
|
|
if (!coolify_schema_bootstrap::tablesExist()) {
|
|
return [];
|
|
}
|
|
|
|
$types = '';
|
|
$params = [];
|
|
$where = ['t.deleted_at IS NULL'];
|
|
if ($kind !== null && trim($kind) !== '') {
|
|
$where[] = 't.kind = ?';
|
|
$types .= 's';
|
|
$params[] = replication_manager::normalizeKind($kind);
|
|
}
|
|
|
|
$targets = $this->selectRows(
|
|
"SELECT t.*, i.label AS instance_label, i.base_url AS instance_base_url, h.label AS replication_label,
|
|
h.host AS replication_host, h.port AS replication_port, h.role AS replication_role,
|
|
h.status AS replication_status, h.last_status_json AS replication_last_status_json,
|
|
h.last_checked_at AS replication_last_checked_at
|
|
FROM coolify_targets t
|
|
INNER JOIN coolify_instances i ON i.id = t.instance_id
|
|
LEFT JOIN replication_hosts h ON h.id = t.replication_host_id
|
|
WHERE " . implode(' AND ', $where) . '
|
|
ORDER BY FIELD(t.kind, \'database\', \'redis\', \'minio\'), t.id',
|
|
$types,
|
|
$params
|
|
);
|
|
|
|
return array_map(fn(array $target): array => $this->publicTarget($target), $targets);
|
|
}
|
|
|
|
public function createTarget(array $input, ?int $actorUserId = null): array
|
|
{
|
|
$this->ensureSchema();
|
|
$kind = replication_manager::normalizeKind((string)($input['kind'] ?? ''));
|
|
$role = strtolower(trim((string)($input['role'] ?? 'replica')));
|
|
if ($role !== 'replica') {
|
|
throw new RuntimeException('Coolify-managed targets must be deployed as replicas first to avoid planned downtime.');
|
|
}
|
|
|
|
$instanceId = (int)($input['instance_id'] ?? 0);
|
|
if ($instanceId <= 0) {
|
|
$instanceId = $this->defaultInstanceId();
|
|
}
|
|
$isolatedStack = $this->isIsolatedStackTargetRequest($input);
|
|
$instance = $this->getInstance($instanceId);
|
|
$input = $this->applyCoolifyDeploymentDefaults($input, $instance);
|
|
$input = $this->applyCoolifyPortDefaults($kind, $input, $instance);
|
|
|
|
$composeInput = $this->composeInputFromRequest($kind, $input, $instance);
|
|
$template = replication_manager::composeTemplate($composeInput);
|
|
$hostPayload = $this->hostPayloadFromTemplate($kind, $input, $template);
|
|
$hostPayload['options'] = array_replace(
|
|
is_array($hostPayload['options'] ?? null) ? $hostPayload['options'] : [],
|
|
[
|
|
'deployment_provider' => 'coolify',
|
|
'coolify_instance_id' => $instanceId,
|
|
]
|
|
);
|
|
|
|
$replicationHost = (new replication_manager())->addHost($kind, $hostPayload, $actorUserId);
|
|
$replicationHostId = (int)$replicationHost['id'];
|
|
$label = trim((string)($input['label'] ?? $replicationHost['label'] ?? $template['service_name'] ?? 'Coolify target'));
|
|
$resourceName = self::resourceName($kind, (string)($template['service_name'] ?? $label), $replicationHostId);
|
|
$targetOptions = $this->targetOptions($input, $template, $composeInput);
|
|
|
|
$this->execute(
|
|
"INSERT INTO coolify_targets (
|
|
instance_id, replication_host_id, kind, label, role, server_uuid, project_uuid,
|
|
environment_uuid, environment_name, destination_uuid, resource_name, deployment_status,
|
|
availability_state, desired_compose_hash, options_json
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', 'degraded', ?, ?)",
|
|
'iisssssssssss',
|
|
[
|
|
$instanceId,
|
|
$replicationHostId,
|
|
$kind,
|
|
$label,
|
|
$role,
|
|
$this->targetMapping($input, $instance, 'server_uuid'),
|
|
$this->targetMapping($input, $instance, 'project_uuid'),
|
|
$this->targetMapping($input, $instance, 'environment_uuid'),
|
|
$this->targetMapping($input, $instance, 'environment_name') ?: 'production',
|
|
$this->targetMapping($input, $instance, 'destination_uuid'),
|
|
$resourceName,
|
|
$this->composeHash($template),
|
|
self::jsonEncode($targetOptions),
|
|
]
|
|
);
|
|
|
|
$targetId = $this->insertId();
|
|
$this->attachTargetToReplicationHost($kind, $replicationHostId, $targetId, $instanceId);
|
|
if (!$isolatedStack) {
|
|
$this->ensureFailoverEnabled($kind);
|
|
}
|
|
$this->audit($targetId, $instanceId, $replicationHostId, 'target_created', $actorUserId, 'info', [
|
|
'kind' => $kind,
|
|
'role' => $role,
|
|
'resource_name' => $resourceName,
|
|
'isolated_stack' => $isolatedStack,
|
|
]);
|
|
|
|
$target = $this->getTarget($targetId);
|
|
$deploy = $this->toBool($input['deploy'] ?? false, false);
|
|
if ($deploy) {
|
|
try {
|
|
$this->deployTarget($targetId, $actorUserId);
|
|
} catch (Throwable $throwable) {
|
|
$this->markTargetFailure($targetId, 'reconcile_failed', $throwable->getMessage(), [
|
|
'stage' => 'create_target_deploy',
|
|
]);
|
|
}
|
|
}
|
|
|
|
return [
|
|
'target' => $this->publicTarget($this->getTarget($targetId)),
|
|
'host' => $replicationHost,
|
|
];
|
|
}
|
|
|
|
public function reconcileTarget(int $targetId, ?int $actorUserId = null): array
|
|
{
|
|
$this->ensureSchema();
|
|
$target = $this->getTarget($targetId);
|
|
$host = $this->replicationHost((int)$target['replication_host_id']);
|
|
if (self::blocksPrimaryMutation($host, 'deploy')) {
|
|
return $this->blockedTargetOperation($target, $host, 'deploy', $actorUserId);
|
|
}
|
|
|
|
$operationId = $this->startOperation($targetId, (int)$target['instance_id'], 'reconcile', $actorUserId);
|
|
|
|
try {
|
|
$instance = $this->getInstance((int)$target['instance_id']);
|
|
$client = $this->clientForInstance($instance);
|
|
$host = $this->syncReplicationHostPortsForTarget($target, $host);
|
|
$host = $this->syncReplicationHostEndpointForTarget($target, $host, $instance);
|
|
$template = $this->composeTemplateForTarget($target, $host);
|
|
$env = self::parseEnvFile((string)($template['env'] ?? ''));
|
|
$hash = $this->composeHash($template);
|
|
$payload = $this->servicePayload($target, $template, false);
|
|
$resourceUuid = trim((string)($target['resource_uuid'] ?? ''));
|
|
$action = 'in_sync';
|
|
$apiResult = [];
|
|
$shouldStart = in_array((string)($target['deployment_status'] ?? ''), ['pending', 'reconcile_failed', 'created', 'deploying', 'provision_blocked'], true);
|
|
|
|
if ($resourceUuid === '') {
|
|
$apiResult = $client->createService($payload);
|
|
$resourceUuid = (string)($apiResult['uuid'] ?? '');
|
|
if ($resourceUuid === '') {
|
|
throw new RuntimeException('Coolify did not return a service UUID.');
|
|
}
|
|
$this->recordCreatedResource($targetId, $resourceUuid, $hash);
|
|
$action = 'created';
|
|
$shouldStart = true;
|
|
} elseif ($hash !== (string)($target['desired_compose_hash'] ?? '')) {
|
|
$apiResult = $client->updateService($resourceUuid, $this->servicePayload($target, $template, true));
|
|
$action = 'updated';
|
|
$shouldStart = true;
|
|
} else {
|
|
try {
|
|
$apiResult = $client->getService($resourceUuid);
|
|
} catch (Throwable) {
|
|
$apiResult = [];
|
|
}
|
|
$action = $shouldStart ? 'start_requested' : 'in_sync';
|
|
}
|
|
|
|
if ($env !== []) {
|
|
$client->updateServiceEnvsBulk($resourceUuid, $env);
|
|
}
|
|
$startResult = null;
|
|
if ($shouldStart) {
|
|
$startResult = $this->startOrRestartService($client, $resourceUuid, $action === 'updated');
|
|
}
|
|
|
|
$context = [
|
|
'action' => $action,
|
|
'resource_uuid' => $resourceUuid,
|
|
'compose_hash' => $hash,
|
|
'coolify' => self::redactCoolifyResponse($apiResult),
|
|
'start' => self::redactCoolifyResponse(is_array($startResult) ? $startResult : []),
|
|
];
|
|
$availabilityState = $this->availabilityStateForHost($host);
|
|
$this->execute(
|
|
"UPDATE coolify_targets
|
|
SET resource_uuid = ?, deployment_status = ?, availability_state = ?, desired_compose_hash = ?,
|
|
last_reconcile_status = ?, last_reconcile_json = ?, last_reconciled_at = NOW()
|
|
WHERE id = ?",
|
|
'ssssssi',
|
|
[
|
|
$resourceUuid,
|
|
$action === 'in_sync' ? 'in_sync' : 'deploying',
|
|
$availabilityState,
|
|
$hash,
|
|
$action,
|
|
self::jsonEncode($context),
|
|
$targetId,
|
|
]
|
|
);
|
|
$this->finishOperation($operationId, 'completed', 'Coolify reconcile completed.', []);
|
|
$this->audit($targetId, (int)$target['instance_id'], (int)$target['replication_host_id'], 'target_reconciled', $actorUserId, 'info', $context);
|
|
|
|
return [
|
|
'ok' => true,
|
|
'status' => $action,
|
|
'target' => $this->publicTarget($this->getTarget($targetId)),
|
|
'context' => $context,
|
|
];
|
|
} catch (Throwable $throwable) {
|
|
$this->finishOperation($operationId, 'failed', null, [$throwable->getMessage()]);
|
|
$this->markTargetFailure($targetId, 'reconcile_failed', $throwable->getMessage());
|
|
$this->audit($targetId, (int)$target['instance_id'], (int)$target['replication_host_id'], 'target_reconcile_failed', $actorUserId, 'error', [
|
|
'error' => $throwable->getMessage(),
|
|
]);
|
|
throw $throwable;
|
|
}
|
|
}
|
|
|
|
public function deployTarget(int $targetId, ?int $actorUserId = null): array
|
|
{
|
|
$reconcile = $this->reconcileTarget($targetId, $actorUserId);
|
|
$target = $this->getTarget($targetId);
|
|
$hostId = (int)($target['replication_host_id'] ?? 0);
|
|
$provision = null;
|
|
|
|
if ($hostId > 0) {
|
|
if ($this->targetSkipsReplicationProvisioning($target)) {
|
|
$provision = [
|
|
'ok' => true,
|
|
'skipped' => true,
|
|
'status' => 'isolated_stack_empty_data_service',
|
|
'message' => 'Isolated stack data services are intentionally not attached to production replication.',
|
|
];
|
|
|
|
return [
|
|
'ok' => true,
|
|
'reconcile' => $reconcile,
|
|
'provision' => $provision,
|
|
'target' => $this->publicTarget($this->getTarget($targetId)),
|
|
];
|
|
}
|
|
|
|
$reconcileAction = (string)($reconcile['status'] ?? $reconcile['context']['action'] ?? '');
|
|
if (in_array($reconcileAction, ['created', 'updated'], true)) {
|
|
$provision = (string)($target['kind'] ?? '') === 'minio'
|
|
? (new replication_manager())->provisionHost((string)$target['kind'], $hostId, $actorUserId, true)
|
|
: $this->deferredProvisionResult(null);
|
|
$this->setTargetProvisionState($targetId, $hostId, 'deploying', 'provision_deferred');
|
|
} else {
|
|
$provision = $this->attemptTargetProvision($targetId, $target, $hostId, $actorUserId, true);
|
|
}
|
|
}
|
|
|
|
return [
|
|
'ok' => ($provision['ok'] ?? true) !== false,
|
|
'reconcile' => $reconcile,
|
|
'provision' => $provision,
|
|
'target' => $this->publicTarget($this->getTarget($targetId)),
|
|
];
|
|
}
|
|
|
|
private function attemptTargetProvision(
|
|
int $targetId,
|
|
array $target,
|
|
int $hostId,
|
|
?int $actorUserId,
|
|
bool $deferLongRunning = false
|
|
): array
|
|
{
|
|
try {
|
|
$provision = (new replication_manager())->provisionHost(
|
|
(string)$target['kind'],
|
|
$hostId,
|
|
$actorUserId,
|
|
$deferLongRunning
|
|
);
|
|
if (($provision['ok'] ?? false) === false && $this->isTransientProvisionBlock($provision)) {
|
|
$this->setTargetProvisionState($targetId, $hostId, 'deploying', 'provision_deferred');
|
|
return $this->deferredProvisionResult($provision);
|
|
}
|
|
|
|
$completed = (($provision['ok'] ?? false) === true)
|
|
&& (($provision['operation']['status'] ?? null) !== 'running');
|
|
$this->setTargetProvisionState(
|
|
$targetId,
|
|
$hostId,
|
|
$completed ? 'provisioned' : ((($provision['ok'] ?? false) === true) ? 'deploying' : 'provision_blocked'),
|
|
$completed ? 'provisioned' : ((($provision['ok'] ?? false) === true) ? 'provisioning' : 'provision_blocked')
|
|
);
|
|
|
|
return $provision;
|
|
} catch (Throwable $throwable) {
|
|
$this->markTargetFailure($targetId, 'provision_blocked', $throwable->getMessage());
|
|
return [
|
|
'ok' => false,
|
|
'message' => $throwable->getMessage(),
|
|
'blockers' => [$throwable->getMessage()],
|
|
];
|
|
}
|
|
}
|
|
|
|
private function setTargetProvisionState(int $targetId, int $hostId, string $deploymentStatus, string $lastReconcileStatus): void
|
|
{
|
|
$this->execute(
|
|
"UPDATE coolify_targets
|
|
SET availability_state = ?, deployment_status = ?, last_reconcile_status = ?, last_reconciled_at = NOW()
|
|
WHERE id = ?",
|
|
'sssi',
|
|
[
|
|
$this->availabilityStateForHost($this->replicationHost($hostId, true)),
|
|
$deploymentStatus,
|
|
$lastReconcileStatus,
|
|
$targetId,
|
|
]
|
|
);
|
|
}
|
|
|
|
private function deferredProvisionResult(?array $provision): array
|
|
{
|
|
$blockers = array_values(array_unique(array_filter(array_map(
|
|
static fn(mixed $blocker): string => trim((string)$blocker),
|
|
is_array($provision['blockers'] ?? null) ? $provision['blockers'] : []
|
|
))));
|
|
|
|
return array_replace($provision ?? [], [
|
|
'ok' => true,
|
|
'deferred' => true,
|
|
'status' => 'waiting_for_coolify',
|
|
'message' => 'Coolify deployment has started. Replication provisioning will continue after the service port becomes reachable.',
|
|
'blockers' => $blockers,
|
|
]);
|
|
}
|
|
|
|
private function isTransientProvisionBlock(array $provision): bool
|
|
{
|
|
$blockers = is_array($provision['blockers'] ?? null) ? $provision['blockers'] : [];
|
|
if ($blockers === []) {
|
|
return false;
|
|
}
|
|
|
|
$matched = false;
|
|
foreach ($blockers as $blocker) {
|
|
$message = strtolower(trim((string)$blocker));
|
|
if ($message === '') {
|
|
continue;
|
|
}
|
|
$isTransient = false;
|
|
foreach ([
|
|
'connection refused',
|
|
'connection timed out',
|
|
'timed out',
|
|
'timeout',
|
|
'failed to connect',
|
|
'could not connect',
|
|
'no route to host',
|
|
'network is unreachable',
|
|
'connection reset',
|
|
'temporarily unavailable',
|
|
'temporary failure',
|
|
'name or service not known',
|
|
] as $needle) {
|
|
if (str_contains($message, $needle)) {
|
|
$isTransient = true;
|
|
$matched = true;
|
|
break;
|
|
}
|
|
}
|
|
if (!$isTransient) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return $matched;
|
|
}
|
|
|
|
public function restartTarget(int $targetId, ?int $actorUserId = null): array
|
|
{
|
|
$this->ensureSchema();
|
|
$target = $this->getTarget($targetId);
|
|
$host = $this->replicationHost((int)$target['replication_host_id']);
|
|
if (self::blocksPrimaryMutation($host, 'restart')) {
|
|
return $this->blockedTargetOperation($target, $host, 'restart', $actorUserId);
|
|
}
|
|
|
|
$resourceUuid = trim((string)($target['resource_uuid'] ?? ''));
|
|
if ($resourceUuid === '') {
|
|
throw new RuntimeException('Coolify target has no resource UUID yet. Reconcile it first.');
|
|
}
|
|
|
|
$operationId = $this->startOperation($targetId, (int)$target['instance_id'], 'restart', $actorUserId);
|
|
try {
|
|
$result = $this->clientForInstance($this->getInstance((int)$target['instance_id']))->restartService($resourceUuid);
|
|
$this->execute(
|
|
"UPDATE coolify_targets SET deployment_status = 'restarting', last_reconcile_status = 'restart_requested', last_reconciled_at = NOW() WHERE id = ?",
|
|
'i',
|
|
[$targetId]
|
|
);
|
|
$this->finishOperation($operationId, 'completed', 'Coolify restart requested.', []);
|
|
$this->audit($targetId, (int)$target['instance_id'], (int)$target['replication_host_id'], 'target_restart_requested', $actorUserId, 'warning', [
|
|
'resource_uuid' => $resourceUuid,
|
|
'coolify' => self::redactCoolifyResponse($result),
|
|
]);
|
|
|
|
return [
|
|
'ok' => true,
|
|
'target' => $this->publicTarget($this->getTarget($targetId)),
|
|
'coolify' => self::redactCoolifyResponse($result),
|
|
];
|
|
} catch (Throwable $throwable) {
|
|
$this->finishOperation($operationId, 'failed', null, [$throwable->getMessage()]);
|
|
$this->markTargetFailure($targetId, 'restart_failed', $throwable->getMessage());
|
|
throw $throwable;
|
|
}
|
|
}
|
|
|
|
public function failoverTarget(int $targetId, ?int $actorUserId = null): array
|
|
{
|
|
$this->ensureSchema();
|
|
$target = $this->getTarget($targetId);
|
|
$host = $this->replicationHost((int)$target['replication_host_id']);
|
|
$kind = (string)$target['kind'];
|
|
|
|
if (($host['role'] ?? '') === 'primary') {
|
|
$result = (new replication_manager())->runAutomaticFailoverMonitor($actorUserId);
|
|
} else {
|
|
$result = (new replication_manager())->promoteHost($kind, (int)$host['id'], $actorUserId);
|
|
}
|
|
|
|
$this->execute(
|
|
"UPDATE coolify_targets SET availability_state = ?, last_reconcile_status = 'failover_checked', last_reconciled_at = NOW() WHERE id = ?",
|
|
'si',
|
|
[$this->availabilityStateForHost($this->replicationHost((int)$host['id'], true)), $targetId]
|
|
);
|
|
$this->audit($targetId, (int)$target['instance_id'], (int)$target['replication_host_id'], 'target_failover_requested', $actorUserId, 'critical', [
|
|
'result' => $result,
|
|
]);
|
|
|
|
return [
|
|
'ok' => true,
|
|
'target' => $this->publicTarget($this->getTarget($targetId)),
|
|
'failover' => $result,
|
|
];
|
|
}
|
|
|
|
public function deleteTarget(int $targetId, array $input, ?int $actorUserId = null): array
|
|
{
|
|
$this->ensureSchema();
|
|
$target = $this->getTarget($targetId);
|
|
$host = $this->replicationHost((int)$target['replication_host_id'], true);
|
|
if (($host['role'] ?? '') === 'primary') {
|
|
throw new RuntimeException('Coolify cannot delete an active primary target. Promote a healthy replica first.');
|
|
}
|
|
|
|
$confirmation = trim((string)($input['confirm'] ?? $input['confirmation'] ?? ''));
|
|
$expected = 'delete-coolify-target-' . $targetId;
|
|
if ($confirmation !== $expected) {
|
|
throw new RuntimeException('Destructive confirmation is required. Send confirm="' . $expected . '".');
|
|
}
|
|
|
|
$deleteResource = $this->toBool($input['delete_resource'] ?? false, false);
|
|
$resourceUuid = trim((string)($target['resource_uuid'] ?? ''));
|
|
$coolifyResult = null;
|
|
if ($deleteResource && $resourceUuid !== '') {
|
|
$coolifyResult = $this->clientForInstance($this->getInstance((int)$target['instance_id']))->deleteService($resourceUuid);
|
|
}
|
|
|
|
$hostRemoved = false;
|
|
$canRemoveHost = replication_manager::replicationHostCanBeRemoved($host)
|
|
|| self::targetAllowsReplicaRemoval($target);
|
|
if ((int)($host['id'] ?? 0) > 0 && $canRemoveHost) {
|
|
(new replication_manager())->removeHost((string)$target['kind'], (int)$host['id'], $actorUserId, false);
|
|
$hostRemoved = true;
|
|
}
|
|
|
|
$this->execute(
|
|
"UPDATE coolify_targets SET deleted_at = NOW(), deployment_status = 'removed', availability_state = 'degraded' WHERE id = ?",
|
|
'i',
|
|
[$targetId]
|
|
);
|
|
$this->audit($targetId, (int)$target['instance_id'], (int)$target['replication_host_id'], 'target_deleted', $actorUserId, 'warning', [
|
|
'delete_resource' => $deleteResource,
|
|
'host_removed' => $hostRemoved,
|
|
'coolify' => self::redactCoolifyResponse(is_array($coolifyResult) ? $coolifyResult : []),
|
|
]);
|
|
|
|
return [
|
|
'ok' => true,
|
|
'id' => $targetId,
|
|
'host_removed' => $hostRemoved,
|
|
'coolify' => self::redactCoolifyResponse(is_array($coolifyResult) ? $coolifyResult : []),
|
|
];
|
|
}
|
|
|
|
public function runAvailabilityMaintenance(?int $actorUserId = null): array
|
|
{
|
|
$this->ensureSchema();
|
|
$failover = (new replication_manager())->runAutomaticFailoverMonitor($actorUserId);
|
|
$updated = [];
|
|
foreach ($this->selectRows('SELECT id, kind, replication_host_id, resource_uuid, deployment_status FROM coolify_targets WHERE deleted_at IS NULL') as $target) {
|
|
$hostId = (int)($target['replication_host_id'] ?? 0);
|
|
if ($hostId <= 0) {
|
|
continue;
|
|
}
|
|
try {
|
|
$provision = null;
|
|
$host = $this->replicationHost($hostId, true);
|
|
if ($this->shouldRetryProvisioning($target, $host)
|
|
|| $this->hasRunningReplicationProvisionOperation((string)$target['kind'], $hostId)) {
|
|
$provision = $this->attemptTargetProvision((int)$target['id'], $target, $hostId, $actorUserId);
|
|
}
|
|
$state = $this->availabilityStateForHost($this->replicationHost($hostId, true));
|
|
$this->execute('UPDATE coolify_targets SET availability_state = ? WHERE id = ?', 'si', [$state, (int)$target['id']]);
|
|
$updated[] = [
|
|
'id' => (int)$target['id'],
|
|
'availability_state' => $state,
|
|
'deployment_status' => $this->getTargetDeploymentStatus((int)$target['id']),
|
|
'provision' => $provision,
|
|
];
|
|
} catch (Throwable) {
|
|
}
|
|
}
|
|
|
|
return [
|
|
'ok' => true,
|
|
'failover' => $failover,
|
|
'targets' => $updated,
|
|
];
|
|
}
|
|
|
|
public function listLoadBalancerGateways(bool $includeDeleted = false): array
|
|
{
|
|
$this->ensureSchema();
|
|
$where = $includeDeleted ? '1=1' : 'deleted_at IS NULL';
|
|
return array_map(
|
|
fn(array $gateway): array => $this->publicGateway($gateway),
|
|
$this->selectRows(
|
|
"SELECT * FROM coolify_instance_gateways WHERE $where ORDER BY priority ASC, id ASC"
|
|
)
|
|
);
|
|
}
|
|
|
|
public function saveLoadBalancerGateway(array $input, ?int $actorUserId = null): array
|
|
{
|
|
$this->ensureSchema();
|
|
|
|
$id = (int)($input['id'] ?? 0);
|
|
$hostname = trim((string)($input['hostname'] ?? ''));
|
|
$targetIp = trim((string)($input['target_ip'] ?? $input['ip'] ?? ''));
|
|
$enabled = $this->toBool($input['enabled'] ?? true, true) ? 1 : 0;
|
|
$priority = max(0, (int)($input['priority'] ?? 100));
|
|
$instanceId = (int)($input['instance_id'] ?? 0);
|
|
$instanceIdValue = $instanceId > 0 ? $instanceId : null;
|
|
|
|
if ($hostname === '' || $targetIp === '') {
|
|
throw new RuntimeException('Gateway hostname and target IP are required.');
|
|
}
|
|
|
|
if (filter_var($targetIp, FILTER_VALIDATE_IP) === false) {
|
|
throw new RuntimeException('Gateway target IP must be a valid IPv4 or IPv6 address.');
|
|
}
|
|
|
|
if ($id > 0) {
|
|
$this->execute(
|
|
"UPDATE coolify_instance_gateways
|
|
SET instance_id = ?, hostname = ?, target_ip = ?, enabled = ?, priority = ?, deleted_at = NULL
|
|
WHERE id = ?",
|
|
'issiii',
|
|
[$instanceIdValue, $hostname, $targetIp, $enabled, $priority, $id]
|
|
);
|
|
$action = 'load_balancer_gateway_updated';
|
|
} else {
|
|
$this->execute(
|
|
"INSERT INTO coolify_instance_gateways (instance_id, hostname, target_ip, enabled, priority)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
ON DUPLICATE KEY UPDATE
|
|
instance_id = VALUES(instance_id),
|
|
hostname = VALUES(hostname),
|
|
enabled = VALUES(enabled),
|
|
priority = VALUES(priority),
|
|
deleted_at = NULL",
|
|
'issii',
|
|
[$instanceIdValue, $hostname, $targetIp, $enabled, $priority]
|
|
);
|
|
$id = $this->insertId();
|
|
if ($id <= 0) {
|
|
$row = $this->selectOne('SELECT id FROM coolify_instance_gateways WHERE target_ip = ? LIMIT 1', 's', [$targetIp]);
|
|
$id = (int)($row['id'] ?? 0);
|
|
}
|
|
$action = 'load_balancer_gateway_saved';
|
|
}
|
|
|
|
$gateway = $this->getGateway($id);
|
|
$this->audit(null, $instanceIdValue, null, $action, $actorUserId, 'info', [
|
|
'gateway_id' => $id,
|
|
'hostname' => $hostname,
|
|
'target_ip' => $targetIp,
|
|
'enabled' => (bool)$enabled,
|
|
'priority' => $priority,
|
|
]);
|
|
|
|
return $this->publicGateway($gateway);
|
|
}
|
|
|
|
public function testLoadBalancerGateway(int $gatewayId, ?int $actorUserId = null): array
|
|
{
|
|
$this->ensureSchema();
|
|
$gateway = $this->getGateway($gatewayId);
|
|
$publicHost = $this->coolifyConfigValue('public_gateway_host', self::DEFAULT_PUBLIC_GATEWAY_HOST);
|
|
$result = $this->probeGatewayTarget((string)$gateway['target_ip'], $publicHost);
|
|
$state = ($result['ok'] ?? false) === true ? 'ok' : 'down';
|
|
|
|
$this->execute(
|
|
"UPDATE coolify_instance_gateways
|
|
SET health_state = ?, last_probe_json = ?, last_probed_at = NOW()
|
|
WHERE id = ?",
|
|
'ssi',
|
|
[$state, self::jsonEncode($result), $gatewayId]
|
|
);
|
|
$this->audit(null, isset($gateway['instance_id']) ? (int)$gateway['instance_id'] : null, null, 'load_balancer_gateway_tested', $actorUserId, $state === 'ok' ? 'info' : 'warning', [
|
|
'gateway_id' => $gatewayId,
|
|
'target_ip' => $gateway['target_ip'] ?? null,
|
|
'result' => $result,
|
|
]);
|
|
|
|
return [
|
|
'gateway' => $this->publicGateway($this->getGateway($gatewayId)),
|
|
'test' => $result,
|
|
];
|
|
}
|
|
|
|
public function loadBalancerSummary(): array
|
|
{
|
|
$this->ensureSchema();
|
|
$config = $this->loadBalancerConfig();
|
|
$gateways = $this->listLoadBalancerGateways();
|
|
$base = [
|
|
'configured' => $config['load_balancer_id'] !== '' && $config['token_set'],
|
|
'status' => 'not_configured',
|
|
'config' => $this->publicLoadBalancerConfig($config),
|
|
'gateways' => $gateways,
|
|
'load_balancer' => null,
|
|
'drift' => [],
|
|
'last_error' => null,
|
|
];
|
|
|
|
if (!$base['configured']) {
|
|
return $base;
|
|
}
|
|
|
|
try {
|
|
$loadBalancer = $this->hetznerClient($config['token'])->getLoadBalancer($config['load_balancer_id']);
|
|
$drift = $this->planLoadBalancerReconcile($loadBalancer, $gateways);
|
|
$this->syncGatewayLoadBalancerStates($gateways, $drift['actual_target_ips']);
|
|
|
|
return array_replace($base, [
|
|
'status' => $drift['has_drift'] ? 'degraded' : 'ok',
|
|
'gateways' => $this->listLoadBalancerGateways(),
|
|
'load_balancer' => $this->publicLoadBalancer($loadBalancer),
|
|
'drift' => $drift,
|
|
]);
|
|
} catch (Throwable $throwable) {
|
|
return array_replace($base, [
|
|
'status' => 'down',
|
|
'last_error' => $throwable->getMessage(),
|
|
]);
|
|
}
|
|
}
|
|
|
|
public function reconcileLoadBalancer(bool $dryRun = true, ?int $actorUserId = null): array
|
|
{
|
|
$this->ensureSchema();
|
|
$config = $this->loadBalancerConfig();
|
|
if ($config['load_balancer_id'] === '' || !$config['token_set']) {
|
|
throw new RuntimeException('Hetzner Load Balancer ID and API token are required.');
|
|
}
|
|
|
|
$client = $this->hetznerClient($config['token']);
|
|
$loadBalancer = $client->getLoadBalancer($config['load_balancer_id']);
|
|
$gateways = $this->listLoadBalancerGateways(true);
|
|
$plan = $this->planLoadBalancerReconcile($loadBalancer, $gateways);
|
|
$canMutate = !$dryRun && $config['automation_enabled'] && $config['automation_mode'] === 'enforce';
|
|
$applied = [];
|
|
$skipped = [];
|
|
$errors = [];
|
|
|
|
foreach ($plan['actions'] as $action) {
|
|
$type = (string)($action['type'] ?? '');
|
|
if ($type === 'skip_remove_target') {
|
|
$skipped[] = $action;
|
|
continue;
|
|
}
|
|
|
|
if (!$canMutate) {
|
|
$skipped[] = array_replace($action, ['reason' => $action['reason'] ?? 'report_only']);
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
if ($type === 'add_target') {
|
|
$client->addIpTarget($config['load_balancer_id'], (string)$action['target_ip']);
|
|
} elseif ($type === 'remove_target') {
|
|
$client->removeIpTarget($config['load_balancer_id'], (string)$action['target_ip']);
|
|
} elseif ($type === 'add_service') {
|
|
$client->addService(
|
|
$config['load_balancer_id'],
|
|
(string)$action['protocol'],
|
|
(int)$action['listen_port'],
|
|
(int)$action['destination_port']
|
|
);
|
|
} else {
|
|
$skipped[] = array_replace($action, ['reason' => 'unknown_action']);
|
|
continue;
|
|
}
|
|
$applied[] = $action;
|
|
} catch (hetzner_cloud_api_exception $exception) {
|
|
if (($action['type'] ?? '') === 'add_target' && $exception->apiCode() === 'target_already_defined') {
|
|
$applied[] = array_replace($action, ['already_defined' => true]);
|
|
continue;
|
|
}
|
|
$errors[] = array_replace($action, [
|
|
'error' => $exception->getMessage(),
|
|
'api_code' => $exception->apiCode(),
|
|
]);
|
|
} catch (Throwable $throwable) {
|
|
$errors[] = array_replace($action, ['error' => $throwable->getMessage()]);
|
|
}
|
|
}
|
|
|
|
$this->audit(null, null, null, $canMutate ? 'load_balancer_reconcile_applied' : 'load_balancer_reconcile_planned', $actorUserId, $errors === [] ? 'info' : 'warning', [
|
|
'dry_run' => $dryRun,
|
|
'can_mutate' => $canMutate,
|
|
'automation_enabled' => $config['automation_enabled'],
|
|
'automation_mode' => $config['automation_mode'],
|
|
'load_balancer_id' => $config['load_balancer_id'],
|
|
'actions' => $plan['actions'],
|
|
'applied' => $applied,
|
|
'skipped' => $skipped,
|
|
'errors' => $errors,
|
|
]);
|
|
|
|
$freshLoadBalancer = $loadBalancer;
|
|
if ($canMutate && $errors === []) {
|
|
$freshLoadBalancer = $client->getLoadBalancer($config['load_balancer_id']);
|
|
}
|
|
$freshPlan = $this->planLoadBalancerReconcile($freshLoadBalancer, $this->listLoadBalancerGateways(true));
|
|
$this->syncGatewayLoadBalancerStates($this->listLoadBalancerGateways(), $freshPlan['actual_target_ips']);
|
|
|
|
return [
|
|
'ok' => $errors === [],
|
|
'dry_run' => $dryRun,
|
|
'mutated' => $canMutate,
|
|
'config' => $this->publicLoadBalancerConfig($config),
|
|
'load_balancer' => $this->publicLoadBalancer($freshLoadBalancer),
|
|
'drift' => $freshPlan,
|
|
'planned' => $plan['actions'],
|
|
'applied' => $applied,
|
|
'skipped' => $skipped,
|
|
'errors' => $errors,
|
|
'gateways' => $this->listLoadBalancerGateways(),
|
|
];
|
|
}
|
|
|
|
public function loadBalancerAutomationEnabled(): bool
|
|
{
|
|
$this->ensureSchema();
|
|
$config = $this->loadBalancerConfig();
|
|
return $config['automation_enabled']
|
|
&& $config['load_balancer_id'] !== ''
|
|
&& $config['token_set'];
|
|
}
|
|
|
|
private function shouldRetryProvisioning(array $target, ?array $host = null): bool
|
|
{
|
|
if (trim((string)($target['resource_uuid'] ?? '')) === '') {
|
|
return false;
|
|
}
|
|
|
|
if (in_array((string)($target['deployment_status'] ?? ''), ['created', 'deploying', 'provision_blocked'], true)) {
|
|
return true;
|
|
}
|
|
|
|
return $host !== null && self::replicationHostStillNeedsProvisioning($host);
|
|
}
|
|
|
|
private function hasRunningReplicationProvisionOperation(string $kind, int $hostId): bool
|
|
{
|
|
if ($hostId <= 0) {
|
|
return false;
|
|
}
|
|
|
|
return $this->selectOne(
|
|
"SELECT id FROM replication_operations
|
|
WHERE kind = ? AND host_id = ? AND operation = 'provision' AND status = 'running'
|
|
LIMIT 1",
|
|
'si',
|
|
[$kind, $hostId]
|
|
) !== null;
|
|
}
|
|
|
|
private static function replicationHostStillNeedsProvisioning(array $host): bool
|
|
{
|
|
if ((string)($host['role'] ?? '') === 'primary') {
|
|
return false;
|
|
}
|
|
|
|
$status = self::jsonDecode($host['last_status_json'] ?? null);
|
|
$effectiveStatus = (string)($status['status'] ?? $host['status'] ?? 'unknown');
|
|
$percent = round((float)($status['replication_percent'] ?? 0), 2);
|
|
$blockers = array_values(array_filter($status['blockers'] ?? []));
|
|
|
|
return $effectiveStatus !== 'ok' || $percent < 100.0 || $blockers !== [];
|
|
}
|
|
|
|
private function getTargetDeploymentStatus(int $targetId): string
|
|
{
|
|
try {
|
|
$target = $this->selectOne('SELECT deployment_status FROM coolify_targets WHERE id = ? LIMIT 1', 'i', [$targetId]);
|
|
return (string)($target['deployment_status'] ?? 'unknown');
|
|
} catch (Throwable) {
|
|
return 'unknown';
|
|
}
|
|
}
|
|
|
|
public static function parseEnvFile(string $env): array
|
|
{
|
|
$values = [];
|
|
foreach (preg_split('/\r\n|\r|\n/', $env) ?: [] as $line) {
|
|
$line = trim($line);
|
|
if ($line === '' || str_starts_with($line, '#') || !str_contains($line, '=')) {
|
|
continue;
|
|
}
|
|
[$key, $value] = explode('=', $line, 2);
|
|
$key = trim($key);
|
|
if ($key === '') {
|
|
continue;
|
|
}
|
|
$values[$key] = trim($value);
|
|
}
|
|
return $values;
|
|
}
|
|
|
|
public static function blocksPrimaryMutation(array $host, string $operation): bool
|
|
{
|
|
return in_array($operation, ['deploy', 'restart', 'delete', 'stop', 'replace'], true)
|
|
&& (string)($host['role'] ?? '') === 'primary';
|
|
}
|
|
|
|
public static function targetAllowsReplicaRemoval(?array $target): bool
|
|
{
|
|
if ($target === null || (string)($target['role'] ?? $target['replication_role'] ?? '') === 'primary') {
|
|
return false;
|
|
}
|
|
|
|
$deploymentStatus = (string)($target['deployment_status'] ?? '');
|
|
$lastReconcileStatus = (string)($target['last_reconcile_status'] ?? '');
|
|
if (in_array($deploymentStatus, ['reconcile_failed', 'removed', 'delete_failed'], true)
|
|
|| in_array($lastReconcileStatus, ['reconcile_failed', 'delete_failed'], true)) {
|
|
return true;
|
|
}
|
|
|
|
$lastReconcile = self::jsonDecode($target['last_reconcile_json'] ?? null);
|
|
$message = strtolower((string)($lastReconcile['message'] ?? $lastReconcile['error'] ?? ''));
|
|
return $message !== '' && (str_contains($message, 'not found') || str_contains($message, '404'));
|
|
}
|
|
|
|
public static function replicationHostCanBeRemoved(array $host): bool
|
|
{
|
|
if ((string)($host['role'] ?? '') === 'primary') {
|
|
return false;
|
|
}
|
|
|
|
$hostId = (int)($host['id'] ?? 0);
|
|
if ($hostId <= 0) {
|
|
return false;
|
|
}
|
|
|
|
try {
|
|
if (!coolify_schema_bootstrap::tablesExist()) {
|
|
return false;
|
|
}
|
|
|
|
$manager = new self();
|
|
$target = $manager->selectOne(
|
|
'SELECT * FROM coolify_targets WHERE replication_host_id = ? AND deleted_at IS NULL ORDER BY id DESC LIMIT 1',
|
|
'i',
|
|
[$hostId]
|
|
);
|
|
if ($target === null) {
|
|
return self::hostHasCoolifyMetadata($host);
|
|
}
|
|
|
|
return self::targetAllowsReplicaRemoval($target);
|
|
} catch (Throwable) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public static function markTargetsRemovedForReplicationHost(int $hostId, ?int $actorUserId = null): void
|
|
{
|
|
if ($hostId <= 0) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
if (!coolify_schema_bootstrap::tablesExist()) {
|
|
return;
|
|
}
|
|
|
|
$manager = new self();
|
|
$targets = $manager->selectRows(
|
|
'SELECT id, instance_id, replication_host_id FROM coolify_targets WHERE replication_host_id = ? AND deleted_at IS NULL',
|
|
'i',
|
|
[$hostId]
|
|
);
|
|
if ($targets === []) {
|
|
return;
|
|
}
|
|
|
|
$manager->execute(
|
|
"UPDATE coolify_targets
|
|
SET deleted_at = NOW(), deployment_status = 'removed', availability_state = 'degraded'
|
|
WHERE replication_host_id = ? AND deleted_at IS NULL",
|
|
'i',
|
|
[$hostId]
|
|
);
|
|
|
|
foreach ($targets as $target) {
|
|
$manager->audit(
|
|
(int)$target['id'],
|
|
(int)$target['instance_id'],
|
|
(int)$target['replication_host_id'],
|
|
'target_removed_with_replication_host',
|
|
$actorUserId,
|
|
'warning',
|
|
['host_id' => $hostId]
|
|
);
|
|
}
|
|
} catch (Throwable) {
|
|
// Removing the replication host should not be blocked by optional Coolify metadata cleanup.
|
|
}
|
|
}
|
|
|
|
public static function deploymentMetadataForReplicationHost(int $hostId): ?array
|
|
{
|
|
if ($hostId <= 0) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
global $db;
|
|
if (!coolify_schema_bootstrap::tablesExist()) {
|
|
return null;
|
|
}
|
|
$stmt = $db->prepare(
|
|
"SELECT t.*, i.label AS instance_label, i.base_url AS instance_base_url
|
|
FROM coolify_targets t
|
|
INNER JOIN coolify_instances i ON i.id = t.instance_id
|
|
WHERE t.replication_host_id = ? AND t.deleted_at IS NULL
|
|
ORDER BY t.id DESC LIMIT 1"
|
|
);
|
|
if ($stmt === false) {
|
|
return null;
|
|
}
|
|
$stmt->bind_param('i', $hostId);
|
|
$stmt->execute();
|
|
$result = $stmt->get_result();
|
|
$target = $result ? $result->fetch_assoc() : null;
|
|
if (!is_array($target)) {
|
|
return null;
|
|
}
|
|
|
|
return [
|
|
'target_id' => (int)$target['id'],
|
|
'instance_id' => (int)$target['instance_id'],
|
|
'instance_label' => (string)($target['instance_label'] ?? ''),
|
|
'base_url' => (string)($target['instance_base_url'] ?? ''),
|
|
'server_uuid' => $target['server_uuid'] ?? null,
|
|
'project_uuid' => $target['project_uuid'] ?? null,
|
|
'environment_uuid' => $target['environment_uuid'] ?? null,
|
|
'environment_name' => $target['environment_name'] ?? null,
|
|
'destination_uuid' => $target['destination_uuid'] ?? null,
|
|
'resource_uuid' => $target['resource_uuid'] ?? null,
|
|
'resource_type' => (string)($target['resource_type'] ?? self::RESOURCE_TYPE_SERVICE),
|
|
'resource_name' => $target['resource_name'] ?? null,
|
|
'deployment_status' => (string)($target['deployment_status'] ?? 'unknown'),
|
|
'last_reconcile_status' => $target['last_reconcile_status'] ?? null,
|
|
'last_reconciled_at' => $target['last_reconciled_at'] ?? null,
|
|
'availability_state' => (string)($target['availability_state'] ?? 'degraded'),
|
|
];
|
|
} catch (Throwable) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public static function syncDeploymentStateForReplicationHost(int $hostId): void
|
|
{
|
|
if ($hostId <= 0) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
if (!coolify_schema_bootstrap::tablesExist()) {
|
|
return;
|
|
}
|
|
|
|
(new self())->syncTargetsForReplicationHost($hostId);
|
|
} catch (Throwable) {
|
|
// Replication health checks must not fail just because Coolify metadata cannot be updated.
|
|
}
|
|
}
|
|
|
|
public static function syncLabelForReplicationHost(int $hostId, string $label): void
|
|
{
|
|
if ($hostId <= 0 || trim($label) === '') {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
if (!coolify_schema_bootstrap::tablesExist()) {
|
|
return;
|
|
}
|
|
|
|
(new self())->execute(
|
|
'UPDATE coolify_targets SET label = ? WHERE replication_host_id = ? AND deleted_at IS NULL',
|
|
'si',
|
|
[$label, $hostId]
|
|
);
|
|
} catch (Throwable) {
|
|
// Renaming a replication host should not fail because optional Coolify metadata is unavailable.
|
|
}
|
|
}
|
|
|
|
private function syncTargetsForReplicationHost(int $hostId): void
|
|
{
|
|
$host = $this->replicationHost($hostId, true);
|
|
$availabilityState = $this->availabilityStateForHost($host);
|
|
$hostIsReady = $this->replicationHostIsReady($host);
|
|
|
|
foreach ($this->selectRows(
|
|
'SELECT id, deployment_status FROM coolify_targets WHERE replication_host_id = ? AND deleted_at IS NULL',
|
|
'i',
|
|
[$hostId]
|
|
) as $target) {
|
|
$deploymentStatus = (string)($target['deployment_status'] ?? 'unknown');
|
|
$nextDeploymentStatus = $deploymentStatus;
|
|
if ($hostIsReady && in_array($deploymentStatus, ['pending', 'created', 'deploying', 'provision_blocked', 'restarting'], true)) {
|
|
$nextDeploymentStatus = 'provisioned';
|
|
}
|
|
|
|
$this->execute(
|
|
'UPDATE coolify_targets SET availability_state = ?, deployment_status = ? WHERE id = ?',
|
|
'ssi',
|
|
[$availabilityState, $nextDeploymentStatus, (int)$target['id']]
|
|
);
|
|
}
|
|
}
|
|
|
|
private function ensureSchema(): void
|
|
{
|
|
if ($this->schemaEnsured) {
|
|
return;
|
|
}
|
|
|
|
coolify_schema_bootstrap::ensureTables();
|
|
$this->schemaEnsured = true;
|
|
}
|
|
|
|
private function composeInputFromRequest(string $kind, array $input, array $instance): array
|
|
{
|
|
$composeRole = $this->isIsolatedStackTargetRequest($input) ? 'primary' : 'replica';
|
|
$hostPort = (int)($input['host_port'] ?? $input['port'] ?? match ($kind) {
|
|
'database' => 3307,
|
|
'redis' => 6380,
|
|
default => 9010,
|
|
});
|
|
|
|
$base = [
|
|
'kind' => $kind,
|
|
'role' => $composeRole,
|
|
'service_name' => $input['service_name'] ?? $input['resource_name'] ?? null,
|
|
'host_port' => $hostPort,
|
|
];
|
|
|
|
if ($kind === 'database') {
|
|
$base['database'] = (string)($input['database'] ?? $input['database_name'] ?? 'nnks_db');
|
|
$base['username'] = (string)($input['username'] ?? 'nnks_db_user');
|
|
$base['server_id'] = (int)($input['server_id'] ?? max(2, time() % 4294967295));
|
|
[$base['primary_host'], $base['primary_port']] = $this->primaryAddress('database');
|
|
} elseif ($kind === 'redis') {
|
|
[$base['primary_host'], $base['primary_port']] = $this->primaryAddress('redis');
|
|
} else {
|
|
$base['host'] = (string)($input['host'] ?? '');
|
|
$base['scheme'] = (string)($input['scheme'] ?? 'http');
|
|
$base['console_port'] = (int)($input['console_port'] ?? ($hostPort + 1));
|
|
$base['buckets'] = $this->normalizeBuckets($input['buckets'] ?? null);
|
|
$base['replication_transfer_limit'] = (string)($input['replication_transfer_limit']
|
|
?? ($input['options']['replication_transfer_limit'] ?? ''));
|
|
[$base['primary_host'], $base['primary_port']] = $this->primaryAddress('minio');
|
|
}
|
|
|
|
return $base;
|
|
}
|
|
|
|
private function hostPayloadFromTemplate(string $kind, array $input, array $template): array
|
|
{
|
|
$credentials = is_array($template['credentials'] ?? null) ? $template['credentials'] : [];
|
|
$host = trim((string)($input['host'] ?? $input['endpoint'] ?? ''));
|
|
if ($host === '') {
|
|
throw new RuntimeException('Target host is required so replication can reach the Coolify-managed container.');
|
|
}
|
|
$isolatedStack = $this->isIsolatedStackTargetRequest($input);
|
|
|
|
$payload = [
|
|
'label' => trim((string)($input['label'] ?? $credentials['label'] ?? $template['service_name'] ?? '')),
|
|
'host' => $host,
|
|
'port' => (int)($credentials['port'] ?? $input['port'] ?? $template['host_port'] ?? 0),
|
|
'username' => (string)($credentials['username'] ?? $input['username'] ?? ''),
|
|
'password' => (string)($credentials['password'] ?? $input['password'] ?? ''),
|
|
];
|
|
|
|
if ($kind === 'database') {
|
|
$payload['database'] = (string)($credentials['database'] ?? $input['database'] ?? $input['database_name'] ?? '');
|
|
$payload['admin_username'] = (string)($credentials['admin_username'] ?? $input['admin_username'] ?? 'root');
|
|
$payload['admin_password'] = (string)($credentials['admin_password'] ?? $input['admin_password'] ?? '');
|
|
$payload['replication_username'] = (string)($credentials['replication_username'] ?? $input['replication_username'] ?? 'replication');
|
|
$payload['replication_password'] = (string)($credentials['replication_password'] ?? $input['replication_password'] ?? '');
|
|
$payload['ssl_mode'] = (string)($credentials['ssl_mode'] ?? $input['ssl_mode'] ?? 'DISABLED');
|
|
$payload['options'] = [
|
|
'allow_preseeded_replica' => !$isolatedStack,
|
|
'isolated_stack' => $isolatedStack,
|
|
'skip_replication_provisioning' => $isolatedStack,
|
|
'production_data_attached' => false,
|
|
];
|
|
} elseif ($kind === 'redis') {
|
|
$payload['database'] = (int)($credentials['database'] ?? $input['database'] ?? 0);
|
|
$payload['options'] = [
|
|
'isolated_stack' => $isolatedStack,
|
|
'skip_replication_provisioning' => $isolatedStack,
|
|
'production_data_attached' => false,
|
|
];
|
|
} else {
|
|
$payload['scheme'] = (string)($credentials['scheme'] ?? $input['scheme'] ?? 'http');
|
|
$payload['buckets'] = $credentials['buckets'] ?? $this->normalizeBuckets($input['buckets'] ?? null);
|
|
$payload['console_port'] = (int)($credentials['console_port'] ?? $input['console_port'] ?? 9001);
|
|
$payload['replication_transfer_limit'] = (string)($credentials['replication_transfer_limit']
|
|
?? $input['replication_transfer_limit']
|
|
?? ($input['options']['replication_transfer_limit'] ?? ''));
|
|
$payload['options'] = [
|
|
'scheme' => $payload['scheme'],
|
|
'buckets' => $payload['buckets'],
|
|
'console_port' => $payload['console_port'],
|
|
'replication_transfer_limit' => $payload['replication_transfer_limit'],
|
|
'space_headroom_percent' => (float)($credentials['space_headroom_percent'] ?? 20.0),
|
|
'isolated_stack' => $isolatedStack,
|
|
'skip_replication_provisioning' => $isolatedStack,
|
|
'production_data_attached' => false,
|
|
];
|
|
}
|
|
|
|
return $payload;
|
|
}
|
|
|
|
private function composeTemplateForTarget(array $target, array $host): array
|
|
{
|
|
$kind = (string)$target['kind'];
|
|
$options = self::jsonDecode($target['options_json'] ?? null);
|
|
$credentials = $this->hostCredentials($host);
|
|
$input = is_array($options['compose_input'] ?? null) ? $options['compose_input'] : [];
|
|
$input['kind'] = $kind;
|
|
$input['role'] = $this->targetComposeRole($target, $options);
|
|
$input['service_name'] = $target['resource_name'] ?? $target['label'] ?? null;
|
|
$input['host_port'] = (int)($host['port'] ?? $input['host_port'] ?? 0);
|
|
|
|
if ($kind === 'database') {
|
|
$input['database'] = (string)($host['database_name'] ?? $input['database'] ?? '');
|
|
$input['username'] = (string)($host['username'] ?? $input['username'] ?? '');
|
|
$input['password'] = $credentials['password'];
|
|
$input['admin_username'] = $credentials['admin_username'] ?: 'root';
|
|
$input['admin_password'] = $credentials['admin_password'];
|
|
$input['replication_username'] = $credentials['replication_username'] ?: 'replication';
|
|
$input['replication_password'] = $credentials['replication_password'];
|
|
[$input['primary_host'], $input['primary_port']] = $this->primaryAddress('database');
|
|
$primaryCredentials = $this->primaryCredentials('database');
|
|
if ($primaryCredentials !== []) {
|
|
$input['primary_admin_username'] = $primaryCredentials['admin_username']
|
|
?: ($primaryCredentials['username'] ?: 'root');
|
|
$input['primary_admin_password'] = $primaryCredentials['admin_password']
|
|
?: $primaryCredentials['password'];
|
|
}
|
|
} elseif ($kind === 'redis') {
|
|
$input['password'] = $credentials['password'];
|
|
[$input['primary_host'], $input['primary_port']] = $this->primaryAddress('redis');
|
|
} else {
|
|
$hostOptions = self::jsonDecode($host['options_json'] ?? null);
|
|
$input['host'] = (string)($host['host'] ?? $input['host'] ?? '');
|
|
$input['scheme'] = (string)($hostOptions['scheme'] ?? $input['scheme'] ?? 'http');
|
|
$input['username'] = $credentials['username'];
|
|
$input['password'] = $credentials['password'];
|
|
$input['buckets'] = $hostOptions['buckets'] ?? $input['buckets'] ?? [];
|
|
$input['console_port'] = (int)($hostOptions['console_port'] ?? $input['console_port'] ?? 9001);
|
|
$input['replication_transfer_limit'] = (string)($hostOptions['replication_transfer_limit']
|
|
?? $input['replication_transfer_limit']
|
|
?? '');
|
|
[$input['primary_host'], $input['primary_port']] = $this->primaryAddress('minio');
|
|
}
|
|
|
|
return replication_manager::composeTemplate($input);
|
|
}
|
|
|
|
private function primaryCredentials(string $kind): array
|
|
{
|
|
$primary = $this->selectOne(
|
|
"SELECT * FROM replication_hosts WHERE kind = ? AND role = 'primary' AND deleted_at IS NULL ORDER BY id DESC LIMIT 1",
|
|
's',
|
|
[$kind]
|
|
);
|
|
if ($primary === null) {
|
|
return [];
|
|
}
|
|
|
|
return $this->hostCredentials($primary);
|
|
}
|
|
|
|
private function startOrRestartService(coolify_api_client $client, string $resourceUuid, bool $restartIfRunning): array
|
|
{
|
|
try {
|
|
return $client->startService($resourceUuid);
|
|
} catch (Throwable $throwable) {
|
|
if (!str_contains(strtolower($throwable->getMessage()), 'already running')) {
|
|
throw $throwable;
|
|
}
|
|
|
|
if ($restartIfRunning) {
|
|
return array_replace(
|
|
['already_running' => true, 'action' => 'restart_requested'],
|
|
$client->restartService($resourceUuid)
|
|
);
|
|
}
|
|
|
|
return [
|
|
'already_running' => true,
|
|
'action' => 'start_noop',
|
|
'message' => 'Service is already running.',
|
|
];
|
|
}
|
|
}
|
|
|
|
private function recordCreatedResource(int $targetId, string $resourceUuid, string $hash): void
|
|
{
|
|
$this->execute(
|
|
"UPDATE coolify_targets
|
|
SET resource_uuid = ?, deployment_status = 'created', desired_compose_hash = ?,
|
|
last_reconcile_status = 'created', last_reconciled_at = NOW()
|
|
WHERE id = ?",
|
|
'ssi',
|
|
[$resourceUuid, $hash, $targetId]
|
|
);
|
|
}
|
|
|
|
private function servicePayload(array $target, array $template, bool $update): array
|
|
{
|
|
$payload = [
|
|
'name' => (string)($target['resource_name'] ?? $target['label']),
|
|
'description' => 'Truckwash managed ' . $target['kind'] . ' replication target. Do not stop the active primary here.',
|
|
'instant_deploy' => false,
|
|
'docker_compose_raw' => $this->encodedDockerCompose($template),
|
|
'force_domain_override' => false,
|
|
];
|
|
|
|
if (!$update) {
|
|
$payload = array_replace($payload, [
|
|
'project_uuid' => $target['project_uuid'] ?? null,
|
|
'environment_name' => $target['environment_name'] ?: 'production',
|
|
'environment_uuid' => $target['environment_uuid'] ?? null,
|
|
'server_uuid' => $target['server_uuid'] ?? null,
|
|
'destination_uuid' => $target['destination_uuid'] ?? null,
|
|
]);
|
|
}
|
|
|
|
return array_filter($payload, static fn($value): bool => $value !== null && $value !== '');
|
|
}
|
|
|
|
private function encodedDockerCompose(array $template): string
|
|
{
|
|
return base64_encode((string)($template['compose'] ?? ''));
|
|
}
|
|
|
|
private function targetOptions(array $input, array $template, array $composeInput): array
|
|
{
|
|
$isolatedStack = $this->isIsolatedStackTargetRequest($input);
|
|
$options = [
|
|
'compose_input' => $composeInput,
|
|
'compose_role' => (string)($composeInput['role'] ?? 'replica'),
|
|
'compose_service_name' => (string)($template['service_name'] ?? ''),
|
|
'engine' => (string)($template['engine'] ?? ''),
|
|
'coolify_docs' => [
|
|
'services_endpoint' => '/api/v1/services',
|
|
'envs_bulk_endpoint' => '/api/v1/services/{uuid}/envs/bulk',
|
|
],
|
|
];
|
|
|
|
if ($isolatedStack) {
|
|
$options['isolated_stack'] = true;
|
|
$options['skip_replication_provisioning'] = true;
|
|
$options['production_data_attached'] = false;
|
|
}
|
|
|
|
return $options;
|
|
}
|
|
|
|
private function isIsolatedStackTargetRequest(array $input): bool
|
|
{
|
|
$options = is_array($input['options'] ?? null) ? $input['options'] : [];
|
|
|
|
return $this->toBool(
|
|
$input['isolated_stack']
|
|
?? $input['isolated_empty_service']
|
|
?? $input['skip_replication_provisioning']
|
|
?? $options['isolated_stack']
|
|
?? $options['skip_replication_provisioning']
|
|
?? false,
|
|
false
|
|
);
|
|
}
|
|
|
|
private function targetComposeRole(array $target, array $options): string
|
|
{
|
|
$composeInput = is_array($options['compose_input'] ?? null) ? $options['compose_input'] : [];
|
|
$composeRole = strtolower(trim((string)($options['compose_role'] ?? $composeInput['role'] ?? '')));
|
|
if (in_array($composeRole, ['primary', 'replica'], true)) {
|
|
return $composeRole;
|
|
}
|
|
|
|
return $this->toBool($options['isolated_stack'] ?? $options['skip_replication_provisioning'] ?? false, false)
|
|
? 'primary'
|
|
: 'replica';
|
|
}
|
|
|
|
private function targetSkipsReplicationProvisioning(array $target): bool
|
|
{
|
|
$options = self::jsonDecode($target['options_json'] ?? null);
|
|
|
|
return $this->toBool($options['skip_replication_provisioning'] ?? $options['isolated_stack'] ?? false, false);
|
|
}
|
|
|
|
private function attachTargetToReplicationHost(string $kind, int $hostId, int $targetId, int $instanceId): void
|
|
{
|
|
$host = $this->replicationHost($hostId, true);
|
|
$options = self::jsonDecode($host['options_json'] ?? null);
|
|
$options['deployment_provider'] = 'coolify';
|
|
$options['coolify_instance_id'] = $instanceId;
|
|
$options['coolify_target_id'] = $targetId;
|
|
$this->execute(
|
|
'UPDATE replication_hosts SET options_json = ? WHERE id = ? AND kind = ?',
|
|
'sis',
|
|
[self::jsonEncode($options), $hostId, $kind]
|
|
);
|
|
}
|
|
|
|
private function blockedTargetOperation(array $target, array $host, string $operation, ?int $actorUserId): array
|
|
{
|
|
$context = [
|
|
'operation' => $operation,
|
|
'reason' => 'active_primary_guard',
|
|
'message' => 'Coolify will not mutate the active primary. Promote a healthy replica first.',
|
|
];
|
|
$this->execute(
|
|
"UPDATE coolify_targets SET availability_state = 'destructive_action_required', last_reconcile_status = ?, last_reconcile_json = ?, last_reconciled_at = NOW() WHERE id = ?",
|
|
'ssi',
|
|
['blocked', self::jsonEncode($context), (int)$target['id']]
|
|
);
|
|
$this->audit((int)$target['id'], (int)$target['instance_id'], (int)$target['replication_host_id'], 'target_operation_blocked', $actorUserId, 'warning', $context);
|
|
|
|
return [
|
|
'ok' => false,
|
|
'status' => 'destructive_action_required',
|
|
'message' => $context['message'],
|
|
'target' => $this->publicTarget($this->getTarget((int)$target['id'])),
|
|
'host' => [
|
|
'id' => (int)($host['id'] ?? 0),
|
|
'role' => (string)($host['role'] ?? ''),
|
|
'status' => (string)($host['status'] ?? ''),
|
|
],
|
|
];
|
|
}
|
|
|
|
private function availabilitySummary(): array
|
|
{
|
|
$summary = [];
|
|
foreach (self::KINDS as $kind) {
|
|
$targets = $this->listTargets($kind);
|
|
$states = array_map(static fn(array $target): string => (string)($target['availability_state'] ?? 'degraded'), $targets);
|
|
$summary[$kind] = [
|
|
'status' => in_array('protected', $states, true) ? 'protected' : ($targets === [] ? 'not_configured' : 'degraded'),
|
|
'targets' => count($targets),
|
|
'protected' => count(array_filter($states, static fn(string $state): bool => $state === 'protected' || $state === 'failover_ready')),
|
|
'blocked' => count(array_filter($states, static fn(string $state): bool => str_contains($state, 'blocked') || $state === 'destructive_action_required')),
|
|
];
|
|
}
|
|
return $summary;
|
|
}
|
|
|
|
private function loadBalancerConfig(): array
|
|
{
|
|
$mode = $this->coolifyConfigValue('lb_automation_mode', 'report_only');
|
|
$mode = in_array($mode, ['report_only', 'enforce'], true) ? $mode : 'report_only';
|
|
$token = $this->hetznerCloudToken();
|
|
$tokenSource = trim((string)(getenv('HETZNER_CLOUD_API_TOKEN') ?: '')) !== '' ? 'env' : 'config';
|
|
|
|
return [
|
|
'automation_enabled' => $this->coolifyConfigBool('lb_automation_enabled', false),
|
|
'automation_mode' => $mode,
|
|
'load_balancer_id' => $this->coolifyConfigValue('hetzner_load_balancer_id', ''),
|
|
'public_gateway_host' => $this->coolifyConfigValue('public_gateway_host', self::DEFAULT_PUBLIC_GATEWAY_HOST),
|
|
'token' => $token,
|
|
'token_set' => trim($token) !== '',
|
|
'token_source' => trim($token) !== '' ? $tokenSource : null,
|
|
'required_services' => self::REQUIRED_LOAD_BALANCER_SERVICES,
|
|
];
|
|
}
|
|
|
|
private function publicLoadBalancerConfig(array $config): array
|
|
{
|
|
unset($config['token']);
|
|
return $config;
|
|
}
|
|
|
|
private function coolifyConfigValue(string $variable, string $default = ''): string
|
|
{
|
|
$row = $this->selectOne(
|
|
"SELECT value FROM module_config WHERE module = 'Coolify' AND variable = ? LIMIT 1",
|
|
's',
|
|
[$variable]
|
|
);
|
|
$value = trim((string)($row['value'] ?? ''));
|
|
return $value !== '' ? $value : $default;
|
|
}
|
|
|
|
private function coolifyConfigBool(string $variable, bool $default = false): bool
|
|
{
|
|
$row = $this->selectOne(
|
|
"SELECT value FROM module_config WHERE module = 'Coolify' AND variable = ? LIMIT 1",
|
|
's',
|
|
[$variable]
|
|
);
|
|
if ($row === null) {
|
|
return $default;
|
|
}
|
|
return $this->toBool($row['value'] ?? null, $default);
|
|
}
|
|
|
|
private function hetznerCloudToken(): string
|
|
{
|
|
$envToken = trim((string)(getenv('HETZNER_CLOUD_API_TOKEN') ?: ''));
|
|
if ($envToken !== '') {
|
|
return $envToken;
|
|
}
|
|
|
|
$stored = $this->coolifyConfigValue('hetzner_cloud_api_token', '');
|
|
if ($stored === '') {
|
|
return '';
|
|
}
|
|
|
|
return replication_secret_box::decrypt($stored);
|
|
}
|
|
|
|
private function hetznerClient(string $token): object
|
|
{
|
|
if ($this->hetznerClientFactory !== null) {
|
|
$client = call_user_func($this->hetznerClientFactory, $token);
|
|
foreach (['getLoadBalancer', 'addIpTarget', 'removeIpTarget', 'addService'] as $method) {
|
|
if (!is_object($client) || !method_exists($client, $method)) {
|
|
throw new RuntimeException('Hetzner client factory returned an invalid client.');
|
|
}
|
|
}
|
|
return $client;
|
|
}
|
|
|
|
return new hetzner_cloud_client($token);
|
|
}
|
|
|
|
private function planLoadBalancerReconcile(array $loadBalancer, array $gateways): array
|
|
{
|
|
$actualTargetIps = self::loadBalancerIpTargets($loadBalancer);
|
|
$actualServices = self::loadBalancerServices($loadBalancer);
|
|
$enabledIps = [];
|
|
$actions = [];
|
|
$missingTargets = [];
|
|
$disabledPresentTargets = [];
|
|
$missingServices = [];
|
|
|
|
foreach ($gateways as $gateway) {
|
|
$targetIp = trim((string)($gateway['target_ip'] ?? ''));
|
|
if ($targetIp === '') {
|
|
continue;
|
|
}
|
|
|
|
if (empty($gateway['deleted_at']) && !empty($gateway['enabled'])) {
|
|
$enabledIps[] = $targetIp;
|
|
if (!in_array($targetIp, $actualTargetIps, true)) {
|
|
$missingTargets[] = $targetIp;
|
|
$actions[] = [
|
|
'type' => 'add_target',
|
|
'target_ip' => $targetIp,
|
|
'hostname' => $gateway['hostname'] ?? null,
|
|
];
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (in_array($targetIp, $actualTargetIps, true)) {
|
|
$disabledPresentTargets[] = $targetIp;
|
|
$actions[] = [
|
|
'type' => 'remove_target',
|
|
'target_ip' => $targetIp,
|
|
'hostname' => $gateway['hostname'] ?? null,
|
|
];
|
|
}
|
|
}
|
|
|
|
foreach (self::REQUIRED_LOAD_BALANCER_SERVICES as $requiredService) {
|
|
if (self::hasLoadBalancerService($actualServices, $requiredService)) {
|
|
continue;
|
|
}
|
|
$missingServices[] = $requiredService;
|
|
$actions[] = array_replace(['type' => 'add_service'], $requiredService);
|
|
}
|
|
|
|
$actions = $this->guardLastLoadBalancerTarget($actions, $actualTargetIps);
|
|
|
|
return [
|
|
'has_drift' => $actions !== [],
|
|
'actions' => array_values($actions),
|
|
'missing_targets' => array_values($missingTargets),
|
|
'disabled_present_targets' => array_values($disabledPresentTargets),
|
|
'missing_services' => array_values($missingServices),
|
|
'actual_target_ips' => $actualTargetIps,
|
|
'expected_target_ips' => array_values(array_unique($enabledIps)),
|
|
'actual_services' => $actualServices,
|
|
'required_services' => self::REQUIRED_LOAD_BALANCER_SERVICES,
|
|
];
|
|
}
|
|
|
|
private function guardLastLoadBalancerTarget(array $actions, array $actualTargetIps): array
|
|
{
|
|
$remainingTargets = count($actualTargetIps);
|
|
$guarded = [];
|
|
|
|
foreach ($actions as $action) {
|
|
if (($action['type'] ?? '') !== 'remove_target') {
|
|
$guarded[] = $action;
|
|
continue;
|
|
}
|
|
|
|
if ($remainingTargets <= 1) {
|
|
$guarded[] = array_replace($action, [
|
|
'type' => 'skip_remove_target',
|
|
'reason' => 'last_reachable_target_guard',
|
|
]);
|
|
continue;
|
|
}
|
|
|
|
$remainingTargets--;
|
|
$guarded[] = $action;
|
|
}
|
|
|
|
return $guarded;
|
|
}
|
|
|
|
private function syncGatewayLoadBalancerStates(array $gateways, array $actualTargetIps): void
|
|
{
|
|
foreach ($gateways as $gateway) {
|
|
$targetIp = trim((string)($gateway['target_ip'] ?? ''));
|
|
if ($targetIp === '') {
|
|
continue;
|
|
}
|
|
$enabled = !empty($gateway['enabled']);
|
|
$present = in_array($targetIp, $actualTargetIps, true);
|
|
$state = match (true) {
|
|
$enabled && $present => 'in_lb',
|
|
$enabled && !$present => 'missing',
|
|
!$enabled && $present => 'disabled_present',
|
|
default => 'disabled_absent',
|
|
};
|
|
$this->execute(
|
|
'UPDATE coolify_instance_gateways SET lb_state = ?, last_reconciled_at = NOW() WHERE id = ?',
|
|
'si',
|
|
[$state, (int)$gateway['id']]
|
|
);
|
|
}
|
|
}
|
|
|
|
private static function loadBalancerIpTargets(array $loadBalancer): array
|
|
{
|
|
$ips = [];
|
|
foreach (($loadBalancer['targets'] ?? []) as $target) {
|
|
if (!is_array($target)) {
|
|
continue;
|
|
}
|
|
$type = strtolower((string)($target['type'] ?? ''));
|
|
$ip = '';
|
|
if ($type === 'ip') {
|
|
$ipPayload = is_array($target['ip'] ?? null) ? $target['ip'] : [];
|
|
$ip = (string)($ipPayload['ip'] ?? '');
|
|
} elseif (isset($target['server']['public_net']['ipv4']['ip'])) {
|
|
$ip = (string)$target['server']['public_net']['ipv4']['ip'];
|
|
}
|
|
$ip = trim($ip);
|
|
if ($ip !== '') {
|
|
$ips[] = $ip;
|
|
}
|
|
}
|
|
|
|
return array_values(array_unique($ips));
|
|
}
|
|
|
|
private static function loadBalancerServices(array $loadBalancer): array
|
|
{
|
|
$services = [];
|
|
foreach (($loadBalancer['services'] ?? []) as $service) {
|
|
if (!is_array($service)) {
|
|
continue;
|
|
}
|
|
$services[] = [
|
|
'protocol' => strtolower((string)($service['protocol'] ?? '')),
|
|
'listen_port' => (int)($service['listen_port'] ?? 0),
|
|
'destination_port' => (int)($service['destination_port'] ?? 0),
|
|
'proxyprotocol' => (bool)($service['proxyprotocol'] ?? false),
|
|
];
|
|
}
|
|
return $services;
|
|
}
|
|
|
|
private static function hasLoadBalancerService(array $services, array $required): bool
|
|
{
|
|
foreach ($services as $service) {
|
|
if ((string)$service['protocol'] === (string)$required['protocol']
|
|
&& (int)$service['listen_port'] === (int)$required['listen_port']
|
|
&& (int)$service['destination_port'] === (int)$required['destination_port']
|
|
&& empty($service['proxyprotocol'])) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private function publicLoadBalancer(array $loadBalancer): array
|
|
{
|
|
return [
|
|
'id' => isset($loadBalancer['id']) ? (int)$loadBalancer['id'] : null,
|
|
'name' => (string)($loadBalancer['name'] ?? ''),
|
|
'ipv4' => $loadBalancer['public_net']['ipv4']['ip'] ?? null,
|
|
'ipv6' => $loadBalancer['public_net']['ipv6']['ip'] ?? null,
|
|
'location' => $loadBalancer['location']['name'] ?? null,
|
|
'algorithm' => $loadBalancer['algorithm']['type'] ?? null,
|
|
'targets' => self::loadBalancerIpTargets($loadBalancer),
|
|
'services' => self::loadBalancerServices($loadBalancer),
|
|
];
|
|
}
|
|
|
|
private function publicGateway(array $gateway): array
|
|
{
|
|
return [
|
|
'id' => (int)$gateway['id'],
|
|
'instance_id' => isset($gateway['instance_id']) ? (int)$gateway['instance_id'] : null,
|
|
'hostname' => (string)$gateway['hostname'],
|
|
'target_ip' => (string)$gateway['target_ip'],
|
|
'enabled' => (bool)$gateway['enabled'],
|
|
'priority' => (int)$gateway['priority'],
|
|
'health_state' => (string)($gateway['health_state'] ?? 'unknown'),
|
|
'lb_state' => (string)($gateway['lb_state'] ?? 'unknown'),
|
|
'last_probe' => self::jsonDecode($gateway['last_probe_json'] ?? null),
|
|
'last_probed_at' => $gateway['last_probed_at'] ?? null,
|
|
'last_reconciled_at' => $gateway['last_reconciled_at'] ?? null,
|
|
'deleted_at' => $gateway['deleted_at'] ?? null,
|
|
'created_at' => $gateway['created_at'] ?? null,
|
|
'updated_at' => $gateway['updated_at'] ?? null,
|
|
];
|
|
}
|
|
|
|
private function getGateway(int $id): array
|
|
{
|
|
$gateway = $this->selectOne(
|
|
'SELECT * FROM coolify_instance_gateways WHERE id = ? AND deleted_at IS NULL LIMIT 1',
|
|
'i',
|
|
[$id]
|
|
);
|
|
if ($gateway === null) {
|
|
throw new RuntimeException('Coolify gateway target was not found.');
|
|
}
|
|
return $gateway;
|
|
}
|
|
|
|
private function probeGatewayTarget(string $targetIp, string $publicHost): array
|
|
{
|
|
$startedAt = microtime(true);
|
|
$url = 'https://' . $publicHost . '/ping';
|
|
$curl = curl_init($url);
|
|
if ($curl === false) {
|
|
throw new RuntimeException('Could not initialize gateway probe.');
|
|
}
|
|
|
|
curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
|
|
curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 3);
|
|
curl_setopt($curl, CURLOPT_TIMEOUT, 5);
|
|
curl_setopt($curl, CURLOPT_NOSIGNAL, true);
|
|
curl_setopt($curl, CURLOPT_HTTPHEADER, ['Accept: application/json']);
|
|
curl_setopt($curl, CURLOPT_RESOLVE, [$publicHost . ':443:' . $targetIp]);
|
|
curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0);
|
|
curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false);
|
|
|
|
$raw = curl_exec($curl);
|
|
$error = curl_error($curl);
|
|
$status = (int)curl_getinfo($curl, CURLINFO_HTTP_CODE);
|
|
curl_close($curl);
|
|
|
|
return [
|
|
'ok' => $raw !== false && $status >= 200 && $status < 300,
|
|
'status_code' => $status ?: null,
|
|
'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2),
|
|
'host' => $publicHost,
|
|
'target_ip' => $targetIp,
|
|
'path' => '/ping',
|
|
'error' => $raw === false ? $error : null,
|
|
'checked_at' => date('c'),
|
|
];
|
|
}
|
|
|
|
private function availabilityStateForHost(array $host): string
|
|
{
|
|
$role = (string)($host['role'] ?? '');
|
|
$status = self::jsonDecode($host['last_status_json'] ?? null);
|
|
$effectiveStatus = (string)($status['status'] ?? $host['status'] ?? 'unknown');
|
|
$percent = round((float)($status['replication_percent'] ?? ($role === 'primary' ? 100 : 0)), 2);
|
|
$blockers = is_array($status['blockers'] ?? null) ? $status['blockers'] : [];
|
|
|
|
if ($role === 'primary') {
|
|
return $this->hasHealthyReplica((string)$host['kind'], (int)$host['id']) ? 'protected' : 'degraded';
|
|
}
|
|
if ($effectiveStatus === 'ok' && $percent >= 100.0 && $blockers === []) {
|
|
return 'failover_ready';
|
|
}
|
|
if (in_array($effectiveStatus, ['down', 'removed'], true)) {
|
|
return 'degraded';
|
|
}
|
|
return 'failover_blocked';
|
|
}
|
|
|
|
private function replicationHostIsReady(array $host): bool
|
|
{
|
|
$status = self::jsonDecode($host['last_status_json'] ?? null);
|
|
$effectiveStatus = (string)($status['status'] ?? $host['status'] ?? 'unknown');
|
|
$percent = round((float)($status['replication_percent'] ?? (($host['role'] ?? '') === 'primary' ? 100 : 0)), 2);
|
|
$blockers = is_array($status['blockers'] ?? null) ? $status['blockers'] : [];
|
|
|
|
return $effectiveStatus === 'ok' && $percent >= 100.0 && $blockers === [];
|
|
}
|
|
|
|
private function hasHealthyReplica(string $kind, int $primaryId): bool
|
|
{
|
|
foreach ($this->selectRows(
|
|
"SELECT * FROM replication_hosts WHERE kind = ? AND role = 'replica' AND deleted_at IS NULL AND id <> ?",
|
|
'si',
|
|
[$kind, $primaryId]
|
|
) as $host) {
|
|
$status = self::jsonDecode($host['last_status_json'] ?? null);
|
|
if (($status['status'] ?? '') === 'ok'
|
|
&& round((float)($status['replication_percent'] ?? 0), 2) >= 100.0
|
|
&& (is_array($status['blockers'] ?? null) ? $status['blockers'] : []) === []) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private function coolifyCollection(array $response): array
|
|
{
|
|
if (self::isListArray($response)) {
|
|
return array_values(array_filter($response, 'is_array'));
|
|
}
|
|
|
|
foreach (['data', 'items', 'servers', 'projects', 'environments', 'resources'] as $key) {
|
|
if (!is_array($response[$key] ?? null)) {
|
|
continue;
|
|
}
|
|
|
|
$collection = $response[$key];
|
|
if (self::isListArray($collection)) {
|
|
return array_values(array_filter($collection, 'is_array'));
|
|
}
|
|
|
|
return array_values(array_filter($collection, 'is_array'));
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
private function publicPlacementServer(array $server): array
|
|
{
|
|
$settings = is_array($server['settings'] ?? null) ? $server['settings'] : [];
|
|
$publicHost = self::publicServerHostFromCoolifyServer($server, null, false)
|
|
?? self::resolvedPublicDnsServerHostFromCoolifyServer($server);
|
|
|
|
return [
|
|
'id' => isset($server['id']) ? (int)$server['id'] : null,
|
|
'uuid' => $this->placementString($server['uuid'] ?? ''),
|
|
'name' => $this->placementString($server['name'] ?? $server['uuid'] ?? ''),
|
|
'description' => $this->placementString($server['description'] ?? ''),
|
|
'ip' => $this->placementString($server['ip'] ?? $server['public_ip'] ?? $server['address'] ?? ''),
|
|
'public_host' => $publicHost,
|
|
'user' => $this->placementString($server['user'] ?? ''),
|
|
'port' => isset($server['port']) ? (int)$server['port'] : null,
|
|
'proxy_type' => $this->placementString($server['proxy_type'] ?? ''),
|
|
'swarm_cluster' => $this->placementString($server['swarm_cluster'] ?? ''),
|
|
'is_reachable' => array_key_exists('is_reachable', $settings) ? (bool)$settings['is_reachable'] : null,
|
|
'is_usable' => array_key_exists('is_usable', $settings) ? (bool)$settings['is_usable'] : null,
|
|
];
|
|
}
|
|
|
|
private function publicPlacementProject(array $project): array
|
|
{
|
|
return [
|
|
'id' => isset($project['id']) ? (int)$project['id'] : null,
|
|
'uuid' => $this->placementString($project['uuid'] ?? ''),
|
|
'name' => $this->placementString($project['name'] ?? $project['uuid'] ?? ''),
|
|
'description' => $this->placementString($project['description'] ?? ''),
|
|
];
|
|
}
|
|
|
|
private function publicPlacementEnvironment(array $environment, array $project): array
|
|
{
|
|
return [
|
|
'id' => isset($environment['id']) ? (int)$environment['id'] : null,
|
|
'uuid' => $this->placementString($environment['uuid'] ?? ''),
|
|
'name' => $this->placementString($environment['name'] ?? $environment['uuid'] ?? ''),
|
|
'description' => $this->placementString($environment['description'] ?? ''),
|
|
'project_id' => isset($environment['project_id']) ? (int)$environment['project_id'] : null,
|
|
'project_uuid' => $this->placementString($project['uuid'] ?? ''),
|
|
'project_name' => $this->placementString($project['name'] ?? ''),
|
|
];
|
|
}
|
|
|
|
private function placementString(mixed $value): string
|
|
{
|
|
return trim((string)($value ?? ''));
|
|
}
|
|
|
|
private function publicInstance(array $instance): array
|
|
{
|
|
return [
|
|
'id' => (int)$instance['id'],
|
|
'label' => (string)$instance['label'],
|
|
'base_url' => (string)$instance['base_url'],
|
|
'api_token_set' => trim((string)($instance['api_token_secret'] ?? '')) !== '',
|
|
'default_project_uuid' => $instance['default_project_uuid'] ?? null,
|
|
'default_environment_uuid' => $instance['default_environment_uuid'] ?? null,
|
|
'default_environment_name' => $instance['default_environment_name'] ?? null,
|
|
'default_server_uuid' => $instance['default_server_uuid'] ?? null,
|
|
'default_destination_uuid' => $instance['default_destination_uuid'] ?? null,
|
|
'status' => (string)($instance['status'] ?? 'unknown'),
|
|
'last_checked_at' => $instance['last_checked_at'] ?? null,
|
|
'last_error' => $instance['last_error'] ?? null,
|
|
'created_at' => $instance['created_at'] ?? null,
|
|
'updated_at' => $instance['updated_at'] ?? null,
|
|
];
|
|
}
|
|
|
|
private function publicTarget(array $target): array
|
|
{
|
|
$replication = [
|
|
'host_id' => isset($target['replication_host_id']) ? (int)$target['replication_host_id'] : null,
|
|
'label' => $target['replication_label'] ?? null,
|
|
'host' => $target['replication_host'] ?? null,
|
|
'port' => isset($target['replication_port']) ? (int)$target['replication_port'] : null,
|
|
'role' => $target['replication_role'] ?? null,
|
|
'status' => $target['replication_status'] ?? null,
|
|
'last_status' => self::jsonDecode($target['replication_last_status_json'] ?? null),
|
|
'last_checked_at' => $target['replication_last_checked_at'] ?? null,
|
|
];
|
|
|
|
return [
|
|
'id' => (int)$target['id'],
|
|
'instance_id' => (int)$target['instance_id'],
|
|
'instance_label' => (string)($target['instance_label'] ?? ''),
|
|
'kind' => (string)$target['kind'],
|
|
'label' => (string)$target['label'],
|
|
'role' => (string)$target['role'],
|
|
'server_uuid' => $target['server_uuid'] ?? null,
|
|
'project_uuid' => $target['project_uuid'] ?? null,
|
|
'environment_uuid' => $target['environment_uuid'] ?? null,
|
|
'environment_name' => $target['environment_name'] ?? null,
|
|
'destination_uuid' => $target['destination_uuid'] ?? null,
|
|
'resource_uuid' => $target['resource_uuid'] ?? null,
|
|
'resource_type' => (string)($target['resource_type'] ?? self::RESOURCE_TYPE_SERVICE),
|
|
'resource_name' => $target['resource_name'] ?? null,
|
|
'deployment_status' => (string)($target['deployment_status'] ?? 'unknown'),
|
|
'availability_state' => (string)($target['availability_state'] ?? 'degraded'),
|
|
'last_reconcile_status' => $target['last_reconcile_status'] ?? null,
|
|
'last_reconcile' => self::jsonDecode($target['last_reconcile_json'] ?? null),
|
|
'last_reconciled_at' => $target['last_reconciled_at'] ?? null,
|
|
'replication' => $replication,
|
|
'created_at' => $target['created_at'] ?? null,
|
|
'updated_at' => $target['updated_at'] ?? null,
|
|
];
|
|
}
|
|
|
|
private function clientForInstance(array $instance): coolify_api_client
|
|
{
|
|
$token = replication_secret_box::decrypt($instance['api_token_secret'] ?? '');
|
|
if ($this->clientFactory !== null) {
|
|
$client = call_user_func($this->clientFactory, $instance, $token);
|
|
if (!$client instanceof coolify_api_client) {
|
|
throw new RuntimeException('Coolify client factory returned an invalid client.');
|
|
}
|
|
return $client;
|
|
}
|
|
return new coolify_api_client((string)$instance['base_url'], $token);
|
|
}
|
|
|
|
private function getInstance(int $id): array
|
|
{
|
|
$instance = $this->selectOne('SELECT * FROM coolify_instances WHERE id = ? AND deleted_at IS NULL LIMIT 1', 'i', [$id]);
|
|
if ($instance === null) {
|
|
throw new RuntimeException('Coolify instance was not found.');
|
|
}
|
|
return $instance;
|
|
}
|
|
|
|
private function getTarget(int $id): array
|
|
{
|
|
$target = $this->selectOne(
|
|
"SELECT t.*, i.label AS instance_label, i.base_url AS instance_base_url, h.label AS replication_label,
|
|
h.host AS replication_host, h.port AS replication_port, h.role AS replication_role,
|
|
h.status AS replication_status, h.last_status_json AS replication_last_status_json,
|
|
h.last_checked_at AS replication_last_checked_at
|
|
FROM coolify_targets t
|
|
INNER JOIN coolify_instances i ON i.id = t.instance_id
|
|
LEFT JOIN replication_hosts h ON h.id = t.replication_host_id
|
|
WHERE t.id = ? AND t.deleted_at IS NULL LIMIT 1",
|
|
'i',
|
|
[$id]
|
|
);
|
|
if ($target === null) {
|
|
throw new RuntimeException('Coolify target was not found.');
|
|
}
|
|
return $target;
|
|
}
|
|
|
|
private function replicationHost(int $id, bool $includeDeleted = false): array
|
|
{
|
|
$sql = 'SELECT * FROM replication_hosts WHERE id = ?';
|
|
if (!$includeDeleted) {
|
|
$sql .= ' AND deleted_at IS NULL';
|
|
}
|
|
$host = $this->selectOne($sql . ' LIMIT 1', 'i', [$id]);
|
|
if ($host === null) {
|
|
throw new RuntimeException('Linked replication host was not found.');
|
|
}
|
|
return $host;
|
|
}
|
|
|
|
private function hostCredentials(array $host): array
|
|
{
|
|
return [
|
|
'username' => (string)($host['username'] ?? ''),
|
|
'password' => replication_secret_box::decrypt($host['password_secret'] ?? ''),
|
|
'admin_username' => (string)($host['admin_username'] ?? ''),
|
|
'admin_password' => replication_secret_box::decrypt($host['admin_password_secret'] ?? ''),
|
|
'replication_username' => (string)($host['replication_username'] ?? ''),
|
|
'replication_password' => replication_secret_box::decrypt($host['replication_password_secret'] ?? ''),
|
|
];
|
|
}
|
|
|
|
private function primaryAddress(string $kind): array
|
|
{
|
|
$primary = $this->selectOne(
|
|
"SELECT * FROM replication_hosts WHERE kind = ? AND role = 'primary' AND deleted_at IS NULL ORDER BY id DESC LIMIT 1",
|
|
's',
|
|
[$kind]
|
|
);
|
|
if ($primary === null) {
|
|
return match ($kind) {
|
|
'database' => ['<primary-host>', 3306],
|
|
'redis' => ['redis-primary', 6379],
|
|
default => ['http://minio-primary:9000', 9000],
|
|
};
|
|
}
|
|
|
|
if ($kind === 'minio') {
|
|
$options = self::jsonDecode($primary['options_json'] ?? null);
|
|
$endpoint = (string)($options['endpoint'] ?? (($options['scheme'] ?? 'http') . '://' . $primary['host'] . ':' . $primary['port']));
|
|
return [$endpoint, (int)$primary['port']];
|
|
}
|
|
|
|
return [(string)$primary['host'], (int)$primary['port']];
|
|
}
|
|
|
|
private function defaultInstanceId(): int
|
|
{
|
|
$instance = $this->selectOne('SELECT id FROM coolify_instances WHERE deleted_at IS NULL ORDER BY id LIMIT 1');
|
|
if ($instance === null) {
|
|
throw new RuntimeException('No Coolify instance is configured.');
|
|
}
|
|
return (int)$instance['id'];
|
|
}
|
|
|
|
private function applyCoolifyDeploymentDefaults(array $input, array $instance): array
|
|
{
|
|
$serverUuid = $this->targetMapping($input, $instance, 'server_uuid');
|
|
if ($serverUuid === null) {
|
|
return $input;
|
|
}
|
|
|
|
$serverHost = $this->resolveCoolifyServerHost(
|
|
$instance,
|
|
$serverUuid,
|
|
(int)($input['host_port'] ?? $input['port'] ?? 0),
|
|
0
|
|
);
|
|
if ($serverHost !== null) {
|
|
$input['host'] = $serverHost;
|
|
}
|
|
|
|
return $input;
|
|
}
|
|
|
|
private function applyCoolifyPortDefaults(string $kind, array $input, array $instance): array
|
|
{
|
|
$serverUuid = $this->targetMapping($input, $instance, 'server_uuid');
|
|
if ($serverUuid === null) {
|
|
return $input;
|
|
}
|
|
|
|
$port = (int)($input['host_port'] ?? $input['port'] ?? match ($kind) {
|
|
'database' => 3307,
|
|
'redis' => 6380,
|
|
default => 9010,
|
|
});
|
|
$consolePort = $kind === 'minio' ? (int)($input['console_port'] ?? ($port + 1)) : null;
|
|
[$nextPort, $nextConsolePort] = $this->nextAvailablePublicPorts(
|
|
$kind,
|
|
$port,
|
|
$consolePort,
|
|
$this->usedPublicPortsForCoolifyServer($serverUuid, 0)
|
|
);
|
|
|
|
$input['host_port'] = $nextPort;
|
|
$input['port'] = $nextPort;
|
|
if ($kind === 'minio' && $nextConsolePort !== null) {
|
|
$input['console_port'] = $nextConsolePort;
|
|
}
|
|
|
|
return $input;
|
|
}
|
|
|
|
private function resolveCoolifyServerHost(array $instance, string $serverUuid, ?int $port = null, int $excludeHostId = 0): ?string
|
|
{
|
|
try {
|
|
foreach ($this->coolifyCollection($this->clientForInstance($instance)->listServers()) as $server) {
|
|
if ($this->placementString($server['uuid'] ?? '') !== $serverUuid) {
|
|
continue;
|
|
}
|
|
|
|
$publicHost = self::publicServerHostFromCoolifyServer($server, $port, false);
|
|
if ($publicHost !== null) {
|
|
return $publicHost;
|
|
}
|
|
|
|
$knownHost = $this->knownPublicHostForCoolifyServer($serverUuid, $port, $excludeHostId);
|
|
if ($knownHost !== null) {
|
|
return $knownHost;
|
|
}
|
|
|
|
return self::resolvedPublicDnsServerHostFromCoolifyServer($server);
|
|
}
|
|
} catch (Throwable) {
|
|
}
|
|
|
|
return $this->knownPublicHostForCoolifyServer($serverUuid, $port, $excludeHostId);
|
|
}
|
|
|
|
private function syncReplicationHostEndpointForTarget(array $target, array $host, array $instance): array
|
|
{
|
|
$serverUuid = trim((string)($target['server_uuid'] ?? ''));
|
|
$hostId = (int)($host['id'] ?? 0);
|
|
if ($serverUuid === '' || $hostId <= 0) {
|
|
return $host;
|
|
}
|
|
|
|
$port = (int)($host['port'] ?? 0);
|
|
$publicHost = $this->resolveCoolifyServerHost($instance, $serverUuid, $port, $hostId);
|
|
if ($publicHost === null || $publicHost === trim((string)($host['host'] ?? ''))) {
|
|
return $host;
|
|
}
|
|
|
|
$options = self::jsonDecode($host['options_json'] ?? null);
|
|
if ((string)($target['kind'] ?? '') === 'minio') {
|
|
$scheme = strtolower(trim((string)($options['scheme'] ?? 'http'))) ?: 'http';
|
|
$options['endpoint'] = $scheme . '://' . $publicHost . ':' . $port;
|
|
}
|
|
|
|
$optionsJson = self::jsonEncode($options);
|
|
$this->execute(
|
|
'UPDATE replication_hosts SET host = ?, options_json = ? WHERE id = ?',
|
|
'ssi',
|
|
[$publicHost, $optionsJson, $hostId]
|
|
);
|
|
|
|
$host['host'] = $publicHost;
|
|
$host['options_json'] = $optionsJson;
|
|
return $host;
|
|
}
|
|
|
|
private function syncReplicationHostPortsForTarget(array $target, array $host): array
|
|
{
|
|
$serverUuid = trim((string)($target['server_uuid'] ?? ''));
|
|
$hostId = (int)($host['id'] ?? 0);
|
|
$kind = (string)($target['kind'] ?? '');
|
|
if ($serverUuid === '' || $hostId <= 0 || (string)($host['role'] ?? '') === 'primary') {
|
|
return $host;
|
|
}
|
|
|
|
$options = self::jsonDecode($host['options_json'] ?? null);
|
|
$port = (int)($host['port'] ?? 0);
|
|
$consolePort = $kind === 'minio' ? (int)($options['console_port'] ?? ($port + 1)) : null;
|
|
if ($port <= 0) {
|
|
return $host;
|
|
}
|
|
|
|
$usedPorts = $this->usedPublicPortsForCoolifyServer($serverUuid, $hostId);
|
|
[$nextPort, $nextConsolePort] = $this->nextAvailablePublicPorts($kind, $port, $consolePort, $usedPorts);
|
|
if ($nextPort === $port && ($kind !== 'minio' || $nextConsolePort === $consolePort)) {
|
|
return $host;
|
|
}
|
|
|
|
if ($kind === 'minio') {
|
|
$scheme = strtolower(trim((string)($options['scheme'] ?? 'http'))) ?: 'http';
|
|
$options['console_port'] = $nextConsolePort;
|
|
$options['endpoint'] = $scheme . '://' . (string)$host['host'] . ':' . $nextPort;
|
|
}
|
|
|
|
$optionsJson = self::jsonEncode($options);
|
|
$this->execute(
|
|
'UPDATE replication_hosts SET port = ?, options_json = ? WHERE id = ?',
|
|
'isi',
|
|
[$nextPort, $optionsJson, $hostId]
|
|
);
|
|
|
|
$host['port'] = $nextPort;
|
|
$host['options_json'] = $optionsJson;
|
|
return $host;
|
|
}
|
|
|
|
private function usedPublicPortsForCoolifyServer(string $serverUuid, int $excludeHostId): array
|
|
{
|
|
$rows = $this->selectRows(
|
|
"SELECT h.port, h.options_json, t.last_reconcile_json
|
|
FROM coolify_targets t
|
|
INNER JOIN replication_hosts h ON h.id = t.replication_host_id
|
|
WHERE t.server_uuid = ? AND h.id <> ? AND t.deleted_at IS NULL AND h.deleted_at IS NULL
|
|
LIMIT 100",
|
|
'si',
|
|
[$serverUuid, $excludeHostId]
|
|
);
|
|
|
|
$ports = [];
|
|
foreach ($rows as $row) {
|
|
$port = (int)($row['port'] ?? 0);
|
|
if ($port > 0) {
|
|
$ports[$port] = true;
|
|
}
|
|
$options = self::jsonDecode($row['options_json'] ?? null);
|
|
$consolePort = (int)($options['console_port'] ?? 0);
|
|
if ($consolePort > 0) {
|
|
$ports[$consolePort] = true;
|
|
}
|
|
foreach (self::coolifyApplicationPortsFromContext(self::jsonDecode($row['last_reconcile_json'] ?? null)) as $applicationPort) {
|
|
$ports[$applicationPort] = true;
|
|
}
|
|
}
|
|
|
|
return array_keys($ports);
|
|
}
|
|
|
|
private function nextAvailablePublicPorts(string $kind, int $port, ?int $consolePort, array $usedPorts): array
|
|
{
|
|
$used = array_fill_keys(array_map('intval', $usedPorts), true);
|
|
if ($kind !== 'minio') {
|
|
while (isset($used[$port]) && $port < 65535) {
|
|
$port++;
|
|
}
|
|
return [$port, null];
|
|
}
|
|
|
|
$consolePort = $consolePort !== null && $consolePort > 0 ? $consolePort : ($port + 1);
|
|
while ((isset($used[$port]) || isset($used[$consolePort])) && $consolePort < 65535) {
|
|
$port += 2;
|
|
$consolePort = $port + 1;
|
|
}
|
|
|
|
return [$port, $consolePort];
|
|
}
|
|
|
|
private static function coolifyApplicationPortsFromContext(array $context): array
|
|
{
|
|
$ports = [];
|
|
$applications = $context['coolify']['applications'] ?? [];
|
|
if (!is_array($applications)) {
|
|
return [];
|
|
}
|
|
|
|
foreach ($applications as $application) {
|
|
if (!is_array($application)) {
|
|
continue;
|
|
}
|
|
foreach (preg_split('/\s*,\s*/', (string)($application['ports'] ?? '')) ?: [] as $mapping) {
|
|
if (preg_match('/^(\d+)\s*:/', trim($mapping), $matches) === 1) {
|
|
$ports[] = (int)$matches[1];
|
|
}
|
|
}
|
|
}
|
|
|
|
return array_values(array_unique(array_filter($ports)));
|
|
}
|
|
|
|
private function knownPublicHostForCoolifyServer(string $serverUuid, ?int $port = null, int $excludeHostId = 0): ?string
|
|
{
|
|
if ($serverUuid === '') {
|
|
return null;
|
|
}
|
|
|
|
$where = 't.server_uuid = ? AND t.deleted_at IS NULL AND h.deleted_at IS NULL';
|
|
$types = 's';
|
|
$params = [$serverUuid];
|
|
if ($excludeHostId > 0) {
|
|
$where .= ' AND h.id <> ?';
|
|
$types .= 'i';
|
|
$params[] = $excludeHostId;
|
|
}
|
|
|
|
$rows = $this->selectRows(
|
|
"SELECT h.host, h.status, h.last_status_json
|
|
FROM coolify_targets t
|
|
INNER JOIN replication_hosts h ON h.id = t.replication_host_id
|
|
WHERE $where
|
|
ORDER BY (h.status = 'ok') DESC, h.last_checked_at DESC, h.updated_at DESC, h.id DESC
|
|
LIMIT 20",
|
|
$types,
|
|
$params
|
|
);
|
|
|
|
$fallback = null;
|
|
foreach ($rows as $row) {
|
|
$host = self::publicServerHostCandidate($row['host'] ?? null);
|
|
if ($host === null) {
|
|
continue;
|
|
}
|
|
$lastStatus = self::jsonDecode($row['last_status_json'] ?? null);
|
|
$isHealthy = (string)($row['status'] ?? '') === 'ok' || (string)($lastStatus['status'] ?? '') === 'ok';
|
|
if ($fallback === null && $isHealthy) {
|
|
$fallback = $host;
|
|
}
|
|
if ($port !== null && $port > 0 && self::tcpPortIsOpen($host, $port)) {
|
|
return $host;
|
|
}
|
|
}
|
|
|
|
return $fallback;
|
|
}
|
|
|
|
public static function publicServerHostFromCoolifyServer(array $server, ?int $port = null, bool $includeDisplayName = true): ?string
|
|
{
|
|
$candidates = [];
|
|
foreach ([
|
|
'public_host',
|
|
'publicHost',
|
|
'public_ip',
|
|
'publicIp',
|
|
'public_ipv4',
|
|
'publicIpv4',
|
|
'public_ipv6',
|
|
'publicIpv6',
|
|
'address',
|
|
'hostname',
|
|
'fqdn',
|
|
'domain',
|
|
'ip',
|
|
] as $key) {
|
|
$host = self::publicServerHostCandidate($server[$key] ?? null);
|
|
if ($host !== null && !in_array($host, $candidates, true)) {
|
|
$candidates[] = $host;
|
|
}
|
|
}
|
|
|
|
if ($includeDisplayName) {
|
|
$host = self::publicServerHostCandidate($server['name'] ?? null);
|
|
if ($host !== null && !in_array($host, $candidates, true)) {
|
|
$candidates[] = $host;
|
|
}
|
|
}
|
|
|
|
if ($port !== null && $port > 0) {
|
|
foreach ($candidates as $host) {
|
|
if (self::tcpPortIsOpen($host, $port)) {
|
|
return $host;
|
|
}
|
|
}
|
|
}
|
|
|
|
return $candidates[0] ?? null;
|
|
}
|
|
|
|
public static function publicDnsServerNameFromCoolifyServer(array $server): ?string
|
|
{
|
|
$host = self::publicServerHostCandidate($server['name'] ?? null);
|
|
if ($host === null || !self::isPublicDnsName($host)) {
|
|
return null;
|
|
}
|
|
|
|
return $host;
|
|
}
|
|
|
|
private static function resolvedPublicDnsServerHostFromCoolifyServer(array $server): ?string
|
|
{
|
|
$host = self::publicDnsServerNameFromCoolifyServer($server);
|
|
if ($host === null) {
|
|
return null;
|
|
}
|
|
|
|
foreach (@gethostbynamel($host) ?: [] as $address) {
|
|
$address = self::publicServerHostCandidate($address);
|
|
if ($address !== null) {
|
|
return $address;
|
|
}
|
|
}
|
|
|
|
return $host;
|
|
}
|
|
|
|
private static function isPublicDnsName(string $host): bool
|
|
{
|
|
$host = strtolower(trim($host, '.'));
|
|
return str_contains($host, '.')
|
|
&& preg_match('/^[a-z0-9][a-z0-9.-]*[a-z0-9]$/', $host) === 1
|
|
&& preg_match('/[a-z]/', $host) === 1
|
|
&& !str_contains($host, '..');
|
|
}
|
|
|
|
private static function tcpPortIsOpen(string $host, int $port): bool
|
|
{
|
|
if ($port <= 0 || $port > 65535) {
|
|
return false;
|
|
}
|
|
|
|
$errno = 0;
|
|
$errstr = '';
|
|
$socket = @fsockopen($host, $port, $errno, $errstr, 0.4);
|
|
if (is_resource($socket)) {
|
|
fclose($socket);
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private static function publicServerHostCandidate(mixed $value): ?string
|
|
{
|
|
$host = trim((string)($value ?? ''));
|
|
if ($host === '') {
|
|
return null;
|
|
}
|
|
|
|
if (str_contains($host, '://')) {
|
|
$parsed = parse_url($host, PHP_URL_HOST);
|
|
$host = is_string($parsed) ? $parsed : $host;
|
|
}
|
|
|
|
$host = trim($host);
|
|
if (str_contains($host, '/')) {
|
|
$host = strtok($host, '/') ?: '';
|
|
}
|
|
if (str_contains($host, ':') && substr_count($host, ':') === 1) {
|
|
$host = explode(':', $host, 2)[0];
|
|
}
|
|
|
|
$host = trim($host, " \t\n\r\0\x0B[]");
|
|
if ($host === '' || preg_match('/\s/', $host) === 1 || self::isDockerLocalOrLoopbackHost($host)) {
|
|
return null;
|
|
}
|
|
|
|
return $host;
|
|
}
|
|
|
|
private static function isDockerLocalOrLoopbackHost(string $host): bool
|
|
{
|
|
$normalized = strtolower(trim($host, '[]'));
|
|
if (in_array($normalized, [
|
|
'localhost',
|
|
'host.docker.internal',
|
|
'host.containers.internal',
|
|
'docker.for.win.localhost',
|
|
'docker.for.mac.localhost',
|
|
'0.0.0.0',
|
|
'::',
|
|
'::1',
|
|
'0:0:0:0:0:0:0:1',
|
|
], true)) {
|
|
return true;
|
|
}
|
|
|
|
return str_starts_with($normalized, '127.')
|
|
|| str_starts_with($normalized, '169.254.')
|
|
|| str_starts_with($normalized, 'fe80:');
|
|
}
|
|
|
|
private function targetMapping(array $input, array $instance, string $key): ?string
|
|
{
|
|
$defaultKey = 'default_' . $key;
|
|
return $this->nullableString($input[$key] ?? $instance[$defaultKey] ?? null);
|
|
}
|
|
|
|
private function nullableString(mixed $value): ?string
|
|
{
|
|
$value = trim((string)($value ?? ''));
|
|
return $value === '' ? null : $value;
|
|
}
|
|
|
|
private function normalizeBuckets(mixed $value): array
|
|
{
|
|
if (is_array($value)) {
|
|
return array_values(array_filter(array_map('strval', $value)));
|
|
}
|
|
return array_values(array_filter(array_map('trim', preg_split('/[,\s]+/', (string)$value) ?: [])));
|
|
}
|
|
|
|
private static function resourceName(string $kind, string $name, int $hostId): string
|
|
{
|
|
$name = strtolower(trim($name));
|
|
$name = preg_replace('/[^a-z0-9-]+/', '-', $name) ?: '';
|
|
$name = trim($name, '-');
|
|
if ($name === '') {
|
|
$name = 'truckwash-' . $kind . '-replica';
|
|
}
|
|
return substr($name . '-' . $hostId, 0, 120);
|
|
}
|
|
|
|
private function composeHash(array $template): string
|
|
{
|
|
return hash('sha256', (string)($template['compose'] ?? '') . "\n---env---\n" . (string)($template['env'] ?? ''));
|
|
}
|
|
|
|
private function startOperation(?int $targetId, ?int $instanceId, string $operation, ?int $actorUserId): int
|
|
{
|
|
$this->execute(
|
|
"INSERT INTO coolify_operations (target_id, instance_id, operation, status, actor_user_id)
|
|
VALUES (?, ?, ?, 'running', ?)",
|
|
'iisi',
|
|
[$targetId, $instanceId, $operation, $actorUserId]
|
|
);
|
|
return $this->insertId();
|
|
}
|
|
|
|
private function finishOperation(int $operationId, string $status, ?string $message, array $errors): void
|
|
{
|
|
$this->execute(
|
|
"UPDATE coolify_operations SET status = ?, message = ?, error_message = ?, completed_at = NOW() WHERE id = ?",
|
|
'sssi',
|
|
[$status, $message, implode("\n", $errors), $operationId]
|
|
);
|
|
}
|
|
|
|
private function markTargetFailure(int $targetId, string $status, string $message, array $context = []): void
|
|
{
|
|
$payload = array_replace($context, ['message' => $message, 'status' => $status]);
|
|
$this->execute(
|
|
"UPDATE coolify_targets
|
|
SET deployment_status = ?, availability_state = 'degraded', last_reconcile_status = ?, last_reconcile_json = ?, last_reconciled_at = NOW()
|
|
WHERE id = ?",
|
|
'sssi',
|
|
[$status, $status, self::jsonEncode($payload), $targetId]
|
|
);
|
|
}
|
|
|
|
private function audit(?int $targetId, ?int $instanceId, ?int $hostId, string $action, ?int $actorUserId, string $severity, array $context): void
|
|
{
|
|
$this->execute(
|
|
"INSERT INTO coolify_audit_logs (target_id, instance_id, replication_host_id, action, actor_user_id, severity, context_json)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
'iiisiss',
|
|
[$targetId, $instanceId, $hostId, $action, $actorUserId, $severity, self::jsonEncode($context)]
|
|
);
|
|
}
|
|
|
|
private function setModuleEnabled(bool $enabled): void
|
|
{
|
|
$value = $enabled ? 'true' : 'false';
|
|
$row = $this->selectOne("SELECT value FROM module_config WHERE module = 'Coolify' AND variable = 'enabled' LIMIT 1");
|
|
if ($row === null) {
|
|
$this->execute("INSERT INTO module_config (module, variable, value, type) VALUES ('Coolify', 'enabled', ?, 'bool')", 's', [$value]);
|
|
return;
|
|
}
|
|
$this->execute("UPDATE module_config SET value = ? WHERE module = 'Coolify' AND variable = 'enabled'", 's', [$value]);
|
|
}
|
|
|
|
private function ensureFailoverEnabled(string $kind): void
|
|
{
|
|
$this->setModuleConfigValue('Failover', 'enabled', 'true', 'bool');
|
|
$this->setModuleConfigValue('Failover', $kind . '_enabled', 'true', 'bool');
|
|
}
|
|
|
|
private function setModuleConfigValue(string $module, string $variable, string $value, string $type): void
|
|
{
|
|
$row = $this->selectOne(
|
|
'SELECT value FROM module_config WHERE module = ? AND variable = ? LIMIT 1',
|
|
'ss',
|
|
[$module, $variable]
|
|
);
|
|
if ($row === null) {
|
|
$this->execute(
|
|
'INSERT INTO module_config (module, variable, value, type) VALUES (?, ?, ?, ?)',
|
|
'ssss',
|
|
[$module, $variable, $value, $type]
|
|
);
|
|
return;
|
|
}
|
|
$this->execute(
|
|
'UPDATE module_config SET value = ?, type = ? WHERE module = ? AND variable = ?',
|
|
'ssss',
|
|
[$value, $type, $module, $variable]
|
|
);
|
|
}
|
|
|
|
private function selectOne(string $sql, string $types = '', array $params = []): ?array
|
|
{
|
|
$rows = $this->selectRows($sql, $types, $params);
|
|
return $rows[0] ?? null;
|
|
}
|
|
|
|
private function selectRows(string $sql, string $types = '', array $params = []): array
|
|
{
|
|
global $db;
|
|
if ($types === '') {
|
|
$result = $db->query($sql);
|
|
return $result ? $result->fetch_all(MYSQLI_ASSOC) : [];
|
|
}
|
|
|
|
$stmt = $db->prepare($sql);
|
|
if ($stmt === false) {
|
|
throw new RuntimeException('Could not prepare Coolify query.');
|
|
}
|
|
$stmt->bind_param($types, ...$params);
|
|
$stmt->execute();
|
|
$result = $stmt->get_result();
|
|
return $result ? $result->fetch_all(MYSQLI_ASSOC) : [];
|
|
}
|
|
|
|
private function execute(string $sql, string $types = '', array $params = []): void
|
|
{
|
|
global $db;
|
|
if ($types === '') {
|
|
$db->query($sql);
|
|
return;
|
|
}
|
|
|
|
$stmt = $db->prepare($sql);
|
|
if ($stmt === false) {
|
|
throw new RuntimeException('Could not prepare Coolify statement.');
|
|
}
|
|
$stmt->bind_param($types, ...$params);
|
|
$stmt->execute();
|
|
}
|
|
|
|
private function insertId(): int
|
|
{
|
|
global $db;
|
|
return (int)$db->insert_id();
|
|
}
|
|
|
|
private function toBool(mixed $value, bool $default): bool
|
|
{
|
|
if (is_bool($value)) {
|
|
return $value;
|
|
}
|
|
if ($value === null) {
|
|
return $default;
|
|
}
|
|
$normalized = strtolower(trim((string)$value));
|
|
if (in_array($normalized, ['1', 'true', 'yes', 'on'], true)) {
|
|
return true;
|
|
}
|
|
if (in_array($normalized, ['0', 'false', 'no', 'off'], true)) {
|
|
return false;
|
|
}
|
|
return $default;
|
|
}
|
|
|
|
private static function hostHasCoolifyMetadata(array $host): bool
|
|
{
|
|
$options = isset($host['options']) && is_array($host['options'])
|
|
? $host['options']
|
|
: self::jsonDecode($host['options_json'] ?? null);
|
|
|
|
return (string)($options['deployment_provider'] ?? '') === 'coolify'
|
|
|| isset($options['coolify_target_id'])
|
|
|| isset($options['coolify_instance_id']);
|
|
}
|
|
|
|
private static function redactCoolifyResponse(array $response): array
|
|
{
|
|
foreach (['token', 'api_token', 'password', 'secret', 'real_value'] as $key) {
|
|
if (array_key_exists($key, $response)) {
|
|
$response[$key] = '[redacted]';
|
|
}
|
|
}
|
|
return $response;
|
|
}
|
|
|
|
private static function jsonEncode(mixed $value): string
|
|
{
|
|
$json = json_encode($value, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
|
|
if ($json === false) {
|
|
throw new RuntimeException('Could not encode Coolify JSON payload.');
|
|
}
|
|
return $json;
|
|
}
|
|
|
|
private static function jsonDecode(mixed $value): array
|
|
{
|
|
if (!is_string($value) || trim($value) === '') {
|
|
return [];
|
|
}
|
|
$decoded = json_decode($value, true);
|
|
return is_array($decoded) ? $decoded : [];
|
|
}
|
|
|
|
private static function isListArray(array $value): bool
|
|
{
|
|
if ($value === []) {
|
|
return true;
|
|
}
|
|
|
|
return array_keys($value) === range(0, count($value) - 1);
|
|
}
|
|
}
|