collectRuntime($warnings); $dependencies = $this->collectDependencies($warnings); $modules = $this->collectModules($force, $warnings); $sessions = (new system_session_activity_tracker())->getSnapshot(); $statuses = [ $runtime['cpu']['status'] ?? 'ok', $runtime['memory']['status'] ?? 'ok', $runtime['disk']['status'] ?? 'ok', $dependencies['database']['status'] ?? 'down', $dependencies['redis']['status'] ?? 'down', $dependencies['minio']['status'] ?? 'down', ]; foreach ($modules as $module) { if (($module['enabled'] ?? false) === true) { $statuses[] = (string)($module['status'] ?? 'configured'); } } $warningEntries = $this->normalizeWarningEntries($warnings); return [ 'overall_status' => self::reduceOverallStatus($statuses), 'generated_at' => date('c'), 'refresh_after_seconds' => self::REFRESH_AFTER_SECONDS, 'runtime' => $runtime, 'dependencies' => $dependencies, 'modules' => $modules, 'sessions' => $sessions, 'warnings' => array_map( static fn(array $warningEntry): string => (string)($warningEntry['message'] ?? ''), $warningEntries ), 'warning_entries' => $warningEntries, ]; } public function probeDatabase(): array { global $db; $startedAt = microtime(true); try { $result = $db->query("SELECT 1 AS ok, DATABASE() AS database_name, VERSION() AS server_version"); $row = $result->fetch_assoc(); return [ 'status' => 'ok', 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), 'database' => (string)($row['database_name'] ?? $db->getDatabase()), 'server_version' => (string)($row['server_version'] ?? ''), 'checked_at' => date('c'), ]; } catch (Throwable $throwable) { return [ 'status' => 'down', 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), 'database' => method_exists($db, 'getDatabase') ? (string)$db->getDatabase() : '', 'server_version' => null, 'error' => $throwable->getMessage(), 'checked_at' => date('c'), ]; } } public function probeRedis(): array { $startedAt = microtime(true); try { if (!defined('redis')) { throw new \RuntimeException('Redis is not initialized'); } $client = redis->get_client(); $ping = (string)$client->ping(); return [ 'status' => (stripos($ping, 'PONG') !== false || stripos($ping, 'OK') !== false) ? 'ok' : 'degraded', 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), 'database' => isset($GLOBALS['REDIS_CONFIG']['database']) ? (int)$GLOBALS['REDIS_CONFIG']['database'] : null, 'checked_at' => date('c'), ]; } catch (Throwable $throwable) { return [ 'status' => 'down', 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), 'database' => isset($GLOBALS['REDIS_CONFIG']['database']) ? (int)$GLOBALS['REDIS_CONFIG']['database'] : null, 'error' => $throwable->getMessage(), 'checked_at' => date('c'), ]; } } public function probeMinio(): array { $startedAt = microtime(true); $buckets = $this->minioBuckets(); try { $client = new S3Client([ 'version' => 'latest', 'region' => 'us-east-1', 'endpoint' => (string)($GLOBALS['MINIO']['endpoint'] ?? ''), 'use_path_style_endpoint' => true, 'credentials' => [ 'key' => (string)($GLOBALS['MINIO']['access_key'] ?? ''), 'secret' => (string)($GLOBALS['MINIO']['secret_key'] ?? ''), ], ]); $client->listBuckets(); $bucketStatuses = []; $overall = 'ok'; foreach ($buckets as $bucket) { try { $exists = (bool)$client->doesBucketExist($bucket); $bucketStatuses[] = [ 'name' => $bucket, 'status' => $exists ? 'ok' : 'down', ]; if (!$exists) { $overall = 'degraded'; } } catch (Throwable $throwable) { $bucketStatuses[] = [ 'name' => $bucket, 'status' => 'down', 'error' => $throwable->getMessage(), ]; $overall = 'degraded'; } } return [ 'status' => $overall, 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), 'endpoint' => (string)($GLOBALS['MINIO']['endpoint'] ?? ''), 'buckets' => $bucketStatuses, 'checked_at' => date('c'), ]; } catch (Throwable $throwable) { return [ 'status' => 'down', 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), 'endpoint' => (string)($GLOBALS['MINIO']['endpoint'] ?? ''), 'buckets' => array_map(static fn(string $bucket): array => ['name' => $bucket, 'status' => 'unknown'], $buckets), 'error' => $throwable->getMessage(), 'checked_at' => date('c'), ]; } } public static function reduceOverallStatus(array $statuses): string { $normalized = array_map(static fn($status): string => strtolower(trim((string)$status)), $statuses); if (in_array('down', $normalized, true)) { return 'down'; } foreach ($normalized as $status) { if (in_array($status, ['degraded', 'not_configured'], true)) { return 'degraded'; } } return 'ok'; } public static function shouldReuseCachedModuleProbe(?array $cachedProbe, bool $force, ?int $referenceTimestamp = null, int $ttlSeconds = self::MODULE_PROBE_TTL_SECONDS): bool { if ($force || !is_array($cachedProbe)) { return false; } $checkedAt = $cachedProbe['checked_at'] ?? null; if (!is_string($checkedAt) || trim($checkedAt) === '') { return false; } $checkedTimestamp = strtotime($checkedAt); if ($checkedTimestamp === false) { return false; } $referenceTimestamp = $referenceTimestamp ?? time(); return ($referenceTimestamp - $checkedTimestamp) < max(1, $ttlSeconds); } public static function statusFromUsagePercent(?float $usagePercent, float $degradedThreshold = 85.0, float $downThreshold = 98.0): string { if ($usagePercent === null) { return 'down'; } if ($usagePercent >= $downThreshold) { return 'down'; } if ($usagePercent >= $degradedThreshold) { return 'degraded'; } return 'ok'; } protected function moduleReason(string $key, array $params, string $message): array { return [ 'status_reason_key' => $key, 'status_reason_params' => $params, 'status_reason' => $message, ]; } protected function pushWarning(array &$warnings, string $key, array $params, string $message): void { $warnings[] = [ 'key' => $key, 'params' => $params, 'message' => $message, ]; } protected function normalizeWarningEntries(array $warnings): array { $entries = []; $seen = []; foreach ($warnings as $warning) { $entry = $this->normalizeWarningEntry($warning); if ($entry === null) { continue; } $signature = md5(json_encode($entry, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)); if (isset($seen[$signature])) { continue; } $seen[$signature] = true; $entries[] = $entry; } return $entries; } protected function normalizeWarningEntry(mixed $warning): ?array { if (is_string($warning)) { $message = trim($warning); if ($message === '') { return null; } return [ 'key' => null, 'params' => [], 'message' => $message, ]; } if (!is_array($warning)) { return null; } $message = trim((string)($warning['message'] ?? '')); if ($message === '') { return null; } return [ 'key' => isset($warning['key']) ? (string)$warning['key'] : null, 'params' => isset($warning['params']) && is_array($warning['params']) ? $warning['params'] : [], 'message' => $message, ]; } private function collectRuntime(array &$warnings): array { return [ 'cpu' => $this->probeCpu($warnings), 'memory' => $this->probeMemory($warnings), 'disk' => $this->probeDisk(), ]; } private function collectDependencies(array &$warnings): array { $database = $this->probeDatabase(); $redis = $this->probeRedis(); $minio = $this->probeMinio(); if (($redis['status'] ?? '') === 'down') { $this->pushWarning( $warnings, 'redis_cache_bypass', [], 'Redis is unavailable; module probe caching is bypassed.' ); } if (($minio['status'] ?? '') === 'degraded') { $this->pushWarning( $warnings, 'minio_missing_buckets', [], 'MinIO is reachable, but one or more expected buckets are missing or inaccessible.' ); } return [ 'database' => $database, 'redis' => $redis, 'minio' => $minio, ]; } private function probeCpu(array &$warnings): array { $checkedAt = date('c'); if (is_readable('/proc/stat')) { $first = $this->readProcStatTotals('/proc/stat'); usleep(100000); $second = $this->readProcStatTotals('/proc/stat'); if ($first !== null && $second !== null) { $totalDelta = $second['total'] - $first['total']; $idleDelta = $second['idle'] - $first['idle']; if ($totalDelta > 0) { $usagePercent = round((1 - ($idleDelta / $totalDelta)) * 100, 2); return [ 'status' => self::statusFromUsagePercent($usagePercent), 'usage_percent' => $usagePercent, 'source' => 'proc_stat', 'checked_at' => $checkedAt, ]; } } } if (is_readable('/proc/loadavg')) { $loadParts = explode(' ', trim((string)file_get_contents('/proc/loadavg'))); $load = isset($loadParts[0]) ? (float)$loadParts[0] : null; $cpuCount = (int)trim((string)@shell_exec('nproc 2>/dev/null')); if ($cpuCount <= 0) { $cpuCount = 1; } if ($load !== null) { $usagePercent = round(min(100, max(0, ($load / $cpuCount) * 100)), 2); $this->pushWarning( $warnings, 'cpu_loadavg_fallback', [], 'CPU usage fell back to load average because /proc/stat sampling was unavailable.' ); return [ 'status' => self::statusFromUsagePercent($usagePercent), 'usage_percent' => $usagePercent, 'source' => 'loadavg', 'checked_at' => $checkedAt, ]; } } $this->pushWarning( $warnings, 'cpu_unavailable', [], 'CPU usage metrics are unavailable in this runtime.' ); return [ 'status' => 'down', 'usage_percent' => null, 'source' => 'unavailable', 'checked_at' => $checkedAt, ]; } private function probeMemory(array &$warnings): array { $checkedAt = date('c'); $cgroupCandidates = [ ['/sys/fs/cgroup/memory.current', '/sys/fs/cgroup/memory.max', 'cgroup_v2'], ['/sys/fs/cgroup/memory/memory.usage_in_bytes', '/sys/fs/cgroup/memory/memory.limit_in_bytes', 'cgroup_v1'], ]; foreach ($cgroupCandidates as [$usagePath, $limitPath, $source]) { if (!is_readable($usagePath) || !is_readable($limitPath)) { continue; } $usedBytes = (int)trim((string)file_get_contents($usagePath)); $limitRaw = trim((string)file_get_contents($limitPath)); if ($limitRaw === 'max') { continue; } $totalBytes = (int)$limitRaw; if ($totalBytes <= 0) { continue; } $usagePercent = round(($usedBytes / $totalBytes) * 100, 2); return [ 'status' => self::statusFromUsagePercent($usagePercent, 90, 99), 'usage_percent' => $usagePercent, 'used_bytes' => $usedBytes, 'total_bytes' => $totalBytes, 'source' => $source, 'checked_at' => $checkedAt, ]; } if (is_readable('/proc/meminfo')) { $content = (string)file_get_contents('/proc/meminfo'); preg_match('/MemTotal:\s+(\d+)\s+kB/i', $content, $totalMatch); preg_match('/MemAvailable:\s+(\d+)\s+kB/i', $content, $availableMatch); if (!empty($totalMatch[1]) && !empty($availableMatch[1])) { $totalBytes = (int)$totalMatch[1] * 1024; $availableBytes = (int)$availableMatch[1] * 1024; $usedBytes = max(0, $totalBytes - $availableBytes); $usagePercent = round(($usedBytes / $totalBytes) * 100, 2); $this->pushWarning( $warnings, 'memory_proc_fallback', [], 'Memory usage fell back to /proc/meminfo because cgroup limits were unavailable.' ); return [ 'status' => self::statusFromUsagePercent($usagePercent, 90, 99), 'usage_percent' => $usagePercent, 'used_bytes' => $usedBytes, 'total_bytes' => $totalBytes, 'source' => 'proc_meminfo', 'checked_at' => $checkedAt, ]; } } $this->pushWarning( $warnings, 'memory_unavailable', [], 'Memory metrics are unavailable in this runtime.' ); return [ 'status' => 'down', 'usage_percent' => null, 'used_bytes' => null, 'total_bytes' => null, 'source' => 'unavailable', 'checked_at' => $checkedAt, ]; } private function probeDisk(): array { $checkedAt = date('c'); $path = is_dir(WD) ? WD : '/'; $totalBytes = @disk_total_space($path); $freeBytes = @disk_free_space($path); if ($totalBytes === false || $freeBytes === false || $totalBytes <= 0) { return [ 'status' => 'down', 'usage_percent' => null, 'used_bytes' => null, 'free_bytes' => null, 'total_bytes' => null, 'path' => $path, 'checked_at' => $checkedAt, ]; } $usedBytes = $totalBytes - $freeBytes; $usagePercent = round(($usedBytes / $totalBytes) * 100, 2); return [ 'status' => self::statusFromUsagePercent($usagePercent, 90, 99), 'usage_percent' => $usagePercent, 'used_bytes' => $usedBytes, 'free_bytes' => $freeBytes, 'total_bytes' => $totalBytes, 'path' => $path, 'checked_at' => $checkedAt, ]; } private function readProcStatTotals(string $path): ?array { $line = strtok((string)file_get_contents($path), PHP_EOL); if (!is_string($line) || !str_starts_with($line, 'cpu ')) { return null; } $parts = preg_split('/\s+/', trim($line)); if (!is_array($parts) || count($parts) < 5) { return null; } $values = array_map('intval', array_slice($parts, 1)); return [ 'idle' => ($values[3] ?? 0) + ($values[4] ?? 0), 'total' => array_sum($values), ]; } protected function minioBuckets(): array { return replication_manager::normalizeMinioBuckets($GLOBALS['MINIO']['buckets'] ?? ['attachments', 'backups', 'invoices', 'pdfs', 'uploads', 'truckwashdev']); } protected function collectModules(bool $force, array &$warnings): array { $descriptors = $this->moduleDescriptors(); $moduleNames = array_values(array_unique(array_map(static fn(array $descriptor): string => $descriptor['module'], $descriptors))); $configRows = $this->loadModuleConfigRows($moduleNames); $modulesWithoutProbes = []; $results = []; foreach ($descriptors as $descriptor) { $moduleConfig = $configRows[$descriptor['module']] ?? []; $enabled = $this->resolveModuleEnabled($descriptor, $moduleConfig); $configuration = $this->resolveModuleConfiguration($descriptor, $moduleConfig); $missingRequired = $configuration['missing']; $configured = $configuration['configured']; $result = [ 'key' => $descriptor['key'], 'enabled' => $enabled, 'configured' => $configured, 'probe_supported' => isset($descriptor['probe']), 'status' => 'configured', 'status_reason' => null, 'status_reason_key' => null, 'status_reason_params' => [], 'checked_at' => date('c'), ]; if (!$enabled) { $result['configured'] = false; $result['status'] = 'disabled'; $result = array_merge($result, $this->moduleReason('module_disabled', [], 'Module is disabled.')); $result = $this->attachModuleUsageMetrics($result); $results[] = $result; continue; } if (!$configured) { $result['status'] = 'not_configured'; $result['status_reason'] = (string)($configuration['reason'] ?? ('Missing required configuration: ' . implode(', ', $missingRequired))); $result['status_reason_key'] = $configuration['reason_key'] ?? 'missing_config'; $result['status_reason_params'] = isset($configuration['reason_params']) && is_array($configuration['reason_params']) ? $configuration['reason_params'] : ['variables' => implode(', ', $missingRequired), 'variables_list' => $missingRequired]; $result = $this->attachModuleUsageMetrics($result); $results[] = $result; continue; } if (!isset($descriptor['probe'])) { $modulesWithoutProbes[] = $descriptor['key']; $result['status'] = 'configured'; $result = array_merge( $result, $this->moduleReason( 'safe_probe_unavailable', [], 'Configuration is present, but no safe read-only probe is available.' ) ); $result = $this->attachModuleUsageMetrics($result); $results[] = $result; continue; } $probeResult = $this->resolveModuleProbeResult($descriptor, $moduleConfig, $force, $warnings); $result['status'] = (string)($probeResult['status'] ?? 'configured'); $result['status_reason'] = $probeResult['status_reason'] ?? null; $result['status_reason_key'] = $probeResult['status_reason_key'] ?? null; $result['status_reason_params'] = isset($probeResult['status_reason_params']) && is_array($probeResult['status_reason_params']) ? $probeResult['status_reason_params'] : []; $result['checked_at'] = (string)($probeResult['checked_at'] ?? $result['checked_at']); if (isset($probeResult['usage']) && is_array($probeResult['usage'])) { $result['usage'] = $probeResult['usage']; } $result = $this->attachModuleUsageMetrics($result, isset($probeResult['usage']) && is_array($probeResult['usage']) ? $probeResult['usage'] : null); $results[] = $result; } if (!empty($modulesWithoutProbes)) { $this->pushWarning( $warnings, 'modules_without_probes', [ 'modules' => implode(', ', $modulesWithoutProbes), 'module_keys' => $modulesWithoutProbes, ], 'Some modules expose configuration-only status because no safe read-only probe exists: ' . implode(', ', $modulesWithoutProbes) . '.' ); } return $results; } protected function attachModuleUsageMetrics(array $moduleResult, ?array $probeUsage = null): array { try { $service = new module_usage_service(); $metrics = $service->metricsForModule((string)($moduleResult['key'] ?? '')); if ($probeUsage !== null) { $probeMetric = $service->recordProviderSnapshotFromLegacyUsage( (string)($moduleResult['key'] ?? ''), $probeUsage, ['source' => 'system_status_probe'] ); if ($probeMetric !== null) { $metrics = $this->replaceModuleUsageMetric($metrics, $probeMetric); } } if ($metrics !== []) { $moduleResult['usage_metrics'] = $metrics; if (!isset($moduleResult['usage'])) { $primaryUsage = $service->primarySystemUsage($metrics); if ($primaryUsage !== null) { $moduleResult['usage'] = $primaryUsage; } } } } catch (Throwable $throwable) { $moduleResult['usage_metrics_error'] = $throwable->getMessage(); } return $moduleResult; } protected function replaceModuleUsageMetric(array $metrics, array $replacement): array { $replaced = false; foreach ($metrics as $index => $metric) { if ( ($metric['module_key'] ?? null) === ($replacement['module_key'] ?? null) && ($metric['metric_key'] ?? null) === ($replacement['metric_key'] ?? null) ) { $metrics[$index] = $replacement; $replaced = true; break; } } if (!$replaced) { $metrics[] = $replacement; } return $metrics; } protected function resolveModuleProbeResult(array $descriptor, array $moduleConfig, bool $force, array &$warnings): array { $cacheKey = self::MODULE_PROBE_CACHE_KEY_PREFIX . $descriptor['key']; $cachedProbe = null; if (defined('redis')) { try { $rawCached = redis->get($cacheKey); if (is_string($rawCached) && trim($rawCached) !== '') { $decoded = json_decode($rawCached, true); if (is_array($decoded)) { $cachedProbe = $decoded; } } } catch (Throwable $throwable) { $this->pushWarning( $warnings, 'redis_cache_lookup_failed', [ 'module' => $descriptor['key'], 'error' => $throwable->getMessage(), ], 'Redis cache lookup failed for module probe ' . $descriptor['key'] . ': ' . $throwable->getMessage() ); } } if (self::shouldReuseCachedModuleProbe($cachedProbe, $force)) { return $cachedProbe; } $probeCallable = $descriptor['probe']; $probeResult = $probeCallable($moduleConfig); if (!is_array($probeResult)) { $probeResult = [ 'status' => 'down', 'status_reason' => 'Probe returned an invalid payload.', 'status_reason_key' => 'invalid_probe_payload', 'status_reason_params' => [], 'checked_at' => date('c'), ]; } if (!isset($probeResult['checked_at'])) { $probeResult['checked_at'] = date('c'); } if (defined('redis')) { try { redis->setEx( $cacheKey, json_encode($probeResult, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), self::MODULE_PROBE_TTL_SECONDS ); } catch (Throwable $throwable) { $this->pushWarning( $warnings, 'redis_cache_write_failed', [ 'module' => $descriptor['key'], 'error' => $throwable->getMessage(), ], 'Redis cache write failed for module probe ' . $descriptor['key'] . ': ' . $throwable->getMessage() ); } } return $probeResult; } protected function resolveModuleConfiguration(array $descriptor, array $moduleConfig): array { if (($descriptor['key'] ?? '') === 'email') { return $this->resolveEmailModuleConfiguration($moduleConfig); } $missing = $this->collectMissingRequiredVariables((array)($descriptor['required'] ?? []), $moduleConfig); return [ 'configured' => count($missing) === 0, 'missing' => $missing, 'reason' => count($missing) === 0 ? null : 'Missing required configuration: ' . implode(', ', $missing), 'reason_key' => count($missing) === 0 ? null : 'missing_config', 'reason_params' => count($missing) === 0 ? [] : ['variables' => implode(', ', $missing), 'variables_list' => $missing], ]; } protected function loadModuleConfigRows(array $moduleNames): array { global $db; if (empty($moduleNames)) { return []; } $escapedModules = array_map(static fn(string $module): string => "'" . $db->escape_string($module) . "'", $moduleNames); $result = $db->query( 'SELECT module, variable, value, type FROM module_config WHERE module IN (' . implode(', ', $escapedModules) . ')' ); $rows = []; while ($row = $result->fetch_assoc()) { $module = (string)($row['module'] ?? ''); $variable = (string)($row['variable'] ?? ''); if ($module === '' || $variable === '') { continue; } $rows[$module][$variable] = [ 'raw' => $row['value'] ?? null, 'parsed' => $this->parseModuleConfigValue((string)($row['type'] ?? 'string'), $row['value'] ?? null), 'type' => (string)($row['type'] ?? 'string'), ]; } return $rows; } protected function resolveModuleEnabled(array $descriptor, array $moduleConfig): bool { if (($descriptor['always_enabled'] ?? false) === true) { return true; } $enabledVariable = $descriptor['enabled_variable'] ?? 'enabled'; return (bool)($moduleConfig[$enabledVariable]['parsed'] ?? false); } protected function resolveMissingRequiredVariables(array $descriptor, array $moduleConfig): array { return $this->collectMissingRequiredVariables((array)($descriptor['required'] ?? []), $moduleConfig); } protected function collectMissingRequiredVariables(array $requiredVariables, array $moduleConfig): array { $missing = []; foreach ($requiredVariables as $variable) { $value = $moduleConfig[$variable]['parsed'] ?? null; if (!$this->isConfiguredValuePresent($value)) { $missing[] = $variable; } } return $missing; } protected function resolveEmailModuleConfiguration(array $moduleConfig): array { $mailersendEnabled = (bool)($moduleConfig['mailersend_enabled']['parsed'] ?? false); if (!$mailersendEnabled) { return [ 'configured' => false, 'missing' => ['mailersend_enabled'], 'reason' => 'MailerSend must be enabled because default SMTP delivery is not implemented.', 'reason_key' => 'email_delivery_not_implemented', 'reason_params' => [], ]; } $required = ['mailersend_api_key', 'smtp_from', 'smtp_from_name', 'smtp_reply_to', 'smtp_reply_to_name']; $missing = $this->collectMissingRequiredVariables($required, $moduleConfig); return [ 'configured' => count($missing) === 0, 'missing' => $missing, 'reason' => count($missing) === 0 ? null : 'Missing required configuration: ' . implode(', ', $missing), 'reason_key' => count($missing) === 0 ? null : 'missing_config', 'reason_params' => count($missing) === 0 ? [] : ['variables' => implode(', ', $missing), 'variables_list' => $missing], ]; } protected function isConfiguredValuePresent(mixed $value): bool { if ($value === null) { return false; } if (is_bool($value)) { return true; } if (is_int($value) || is_float($value)) { return true; } if (is_string($value)) { return trim($value) !== ''; } if (is_array($value)) { return !empty($value); } return true; } protected function parseModuleConfigValue(string $type, mixed $value): mixed { return match (strtolower($type)) { 'bool' => in_array(strtolower(trim((string)$value)), ['1', 'true', 'yes', 'on'], true), 'int', 'integer' => is_numeric($value) ? (int)$value : null, 'float', 'double' => is_numeric($value) ? (float)$value : null, 'json' => is_string($value) ? json_decode($value, true) : null, default => $value, }; } protected function moduleDescriptors(): array { return [ ['key' => 'economic', 'module' => 'economic', 'always_enabled' => true, 'required' => ['invoiceLayoutNumber', 'invoiceDiscountLayoutNumber', 'paymentTermsNumber', 'adminFeeMonthly', 'adminFeeOrder', 'feeProductId'], 'probe' => fn(array $config): array => $this->probeEconomicModule($config)], ['key' => 'reCAPTCHA', 'module' => 'reCAPTCHA', 'enabled_variable' => 'enabled', 'required' => ['site_key_v2', 'secret_key_v2'], 'probe' => fn(array $config): array => $this->probeRecaptchaModule($config)], ['key' => 'email', 'module' => 'Email', 'enabled_variable' => 'enabled', 'required' => ['smtp_host', 'smtp_port', 'smtp_username', 'smtp_password', 'smtp_encryption', 'smtp_from', 'smtp_from_name', 'smtp_reply_to', 'smtp_reply_to_name'], 'probe' => fn(array $config): array => $this->probeEmailModule($config)], ['key' => 'backups', 'module' => 'Backups', 'enabled_variable' => 'enabled', 'required' => [], 'probe' => fn(array $config): array => $this->probeBackupsModule($config)], ['key' => 'motorapi', 'module' => 'motorapi', 'enabled_variable' => 'enabled', 'required' => ['secret_key'], 'probe' => fn(array $config): array => $this->probeMotorApiModule($config)], ['key' => 'stripe', 'module' => 'Stripe', 'enabled_variable' => 'enabled', 'required' => ['publishable_key', 'secret_key', 'economic_customer_number'], 'probe' => fn(array $config): array => $this->probeStripeModule($config)], ['key' => 'fxratesapi', 'module' => 'fxratesapi', 'enabled_variable' => 'enabled', 'required' => ['secret_key'], 'probe' => fn(array $config): array => $this->probeFxRatesApiModule($config)], ['key' => 'weatherapi', 'module' => 'weatherapi', 'enabled_variable' => 'enabled', 'required' => ['secret_key'], 'probe' => fn(array $config): array => $this->probeWeatherApiModule($config)], ['key' => 'workfeed', 'module' => 'workfeed', 'enabled_variable' => 'enabled', 'required' => ['api_url', 'api_key', 'CompanyID'], 'probe' => fn(array $config): array => $this->probeWorkfeedModule($config)], ['key' => 'gatewayapi', 'module' => 'GatewayAPI', 'enabled_variable' => 'enabled', 'required' => ['api_secret', 'api_token', 'sender'], 'probe' => fn(array $config): array => $this->probeGatewayApiModule($config)], ['key' => 'xlvask', 'module' => 'xlvask', 'enabled_variable' => 'enabled', 'required' => ['username', 'password'], 'probe' => fn(array $config): array => $this->probeXlVaskModule($config)], ['key' => 'entra', 'module' => 'Entra', 'enabled_variable' => 'enabled', 'required' => ['entra_client_id', 'entra_client_secret', 'entra_tenant_id'], 'probe' => fn(array $config): array => $this->probeEntraModule($config)], ['key' => 'limble', 'module' => 'limble', 'enabled_variable' => 'enabled', 'required' => ['client_id', 'client_secret'], 'probe' => fn(array $config): array => $this->probeLimbleModule($config)], ['key' => 'ocrspace', 'module' => 'ocrSpace', 'enabled_variable' => 'enabled', 'required' => ['api_key']], ['key' => 'openai', 'module' => 'openAI', 'enabled_variable' => 'enabled', 'required' => ['api_key'], 'probe' => fn(array $config): array => $this->probeOpenAiModule($config)], ['key' => 'licenseplaterecognizer', 'module' => 'licenseplaterecognizer', 'enabled_variable' => 'enabled', 'required' => ['api_key'], 'probe' => fn(array $config): array => $this->probeLicensePlateRecognizerModule($config)], ['key' => 'virkdata', 'module' => 'virkdata', 'enabled_variable' => 'enabled', 'required' => ['secret_key']], ['key' => 'shelly', 'module' => 'shelly', 'enabled_variable' => 'enabled', 'required' => ['server_url', 'secret_key'], 'probe' => fn(array $config): array => $this->probeShellyModule($config)], ['key' => 'coolify', 'module' => 'Coolify', 'enabled_variable' => 'enabled', 'required' => [], 'probe' => fn(array $config): array => $this->probeCoolifyModule($config)], ['key' => 'releasemanager', 'module' => 'ReleaseManager', 'enabled_variable' => 'enabled', 'required' => [], 'always_enabled' => true, 'probe' => fn(array $config): array => $this->probeReleaseManagerModule($config)], ['key' => 'selfserve', 'module' => 'selfserve', 'enabled_variable' => 'enabled', 'required' => ['machine_wash_minutes_included', 'minute_product'], 'probe' => fn(array $config): array => $this->probeSelfserveModule($config)], ['key' => 'bird', 'module' => 'bird', 'enabled_variable' => 'enabled', 'required' => ['server_url', 'api_key', 'channelId', 'workplaceId'], 'probe' => fn(array $config): array => $this->probeBirdModule($config)], ]; } protected function probeReleaseManagerModule(array $config): array { return (new release_manager())->healthProbe(); } protected function probeCoolifyModule(array $config): array { $startedAt = microtime(true); try { $summary = (new coolify_manager())->summary(); $instances = is_array($summary['instances'] ?? null) ? $summary['instances'] : []; $targets = is_array($summary['targets'] ?? null) ? $summary['targets'] : []; if ($instances === []) { return [ 'status' => 'not_configured', 'status_reason' => 'Coolify is enabled, but no Coolify API instance is configured.', 'status_reason_key' => 'coolify_instances_missing', 'status_reason_params' => [], 'checked_at' => date('c'), 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } $downInstances = array_values(array_filter($instances, static fn(array $instance): bool => ($instance['status'] ?? 'unknown') === 'down')); $blockedTargets = array_values(array_filter($targets, static function (array $target): bool { $state = (string)($target['availability_state'] ?? 'degraded'); return $state === 'destructive_action_required' || str_contains($state, 'blocked'); })); $failedTargets = array_values(array_filter($targets, static function (array $target): bool { return in_array((string)($target['deployment_status'] ?? ''), ['reconcile_failed', 'restart_failed', 'provision_blocked'], true); })); $status = 'ok'; $reason = 'Coolify deployment state is available.'; $reasonKey = 'coolify_available'; if ($downInstances !== []) { $status = 'down'; $reason = 'One or more Coolify API instances are unreachable.'; $reasonKey = 'coolify_instances_down'; } elseif ($blockedTargets !== [] || $failedTargets !== []) { $status = 'degraded'; $reason = 'One or more Coolify targets need operator attention before availability can be protected.'; $reasonKey = 'coolify_targets_need_attention'; } elseif ($targets === []) { $status = 'degraded'; $reason = 'Coolify is connected, but no replicated infrastructure targets are managed yet.'; $reasonKey = 'coolify_targets_missing'; } return [ 'status' => $status, 'status_reason' => $reason, 'status_reason_key' => $reasonKey, 'status_reason_params' => [ 'instances' => count($instances), 'targets' => count($targets), 'blocked_targets' => count($blockedTargets), 'failed_targets' => count($failedTargets), ], 'checked_at' => date('c'), 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } catch (Throwable $throwable) { return [ 'status' => 'down', 'status_reason' => 'Coolify module probe failed: ' . $throwable->getMessage(), 'status_reason_key' => 'coolify_probe_failed', 'status_reason_params' => ['error' => $throwable->getMessage()], 'checked_at' => date('c'), 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } } protected function probeEconomicModule(array $config): array { $appSecretToken = trim((string)($GLOBALS['ECONOMIC_API']['app_secret_token'] ?? '')); $agreementGrantToken = trim((string)($GLOBALS['ECONOMIC_API']['app_access_grant'] ?? '')); if ($appSecretToken === '' || $agreementGrantToken === '') { return [ 'status' => 'down', 'status_reason' => 'e-conomic credentials are missing from runtime environment configuration.', 'status_reason_key' => 'economic_credentials_missing', 'status_reason_params' => [], 'checked_at' => date('c'), ]; } return $this->performHttpProbe( $this->buildUrlWithQuery('https://restapi.e-conomic.com', '/layouts/'), [ 'X-AppSecretToken: ' . $appSecretToken, 'X-AgreementGrantToken: ' . $agreementGrantToken, 'Accept: application/json', ], 'e-conomic API' ); } protected function probeRecaptchaModule(array $config): array { $secretKey = trim((string)($config['secret_key_v2']['parsed'] ?? '')); return $this->performHttpProbe( 'https://www.google.com/recaptcha/api/siteverify', ['Content-Type: application/x-www-form-urlencoded'], 'reCAPTCHA', null, 'POST', http_build_query([ 'secret' => $secretKey, 'response' => 'system-status-probe', ]), fn(array $httpResponse, string $label): array => $this->evaluateRecaptchaProbeResponse($httpResponse, $label) ); } protected function probeEmailModule(array $config): array { $apiKey = trim((string)($config['mailersend_api_key']['parsed'] ?? '')); return $this->performHttpProbe( 'https://api.mailersend.com/v1/api-quota', [ 'Authorization: Bearer ' . $apiKey, 'Accept: application/json', ], 'MailerSend API', null, 'GET', null, fn(array $httpResponse, string $label): array => $this->evaluateProviderQuotaProbeResponse($httpResponse, $label, 'email') ); } protected function probeBackupsModule(array $config): array { $checkedAt = date('c'); $startedAt = microtime(true); try { $this->validateBackupsStore(); $health = $this->backupHealthSummary(); if (empty($health['encryption']['available'])) { $error = (string)($health['encryption']['error'] ?? 'Backup encryption key is missing.'); return [ 'status' => 'down', 'status_reason' => 'Backup encryption is not ready: ' . $error, 'status_reason_key' => 'backup_encryption_key_missing', 'status_reason_params' => ['error' => $error], 'checked_at' => $checkedAt, 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } if (empty($health['latest_verified_backup'])) { return [ 'status' => 'degraded', 'status_reason' => 'No verified backup is available for restore.', 'status_reason_key' => 'backup_no_verified_backup', 'status_reason_params' => [], 'checked_at' => $checkedAt, 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } if (empty($health['fresh'])) { return [ 'status' => 'degraded', 'status_reason' => 'Latest verified backup is stale.', 'status_reason_key' => 'backup_latest_verified_stale', 'status_reason_params' => ['age_seconds' => (string)($health['latest_verified_age_seconds'] ?? '')], 'checked_at' => $checkedAt, 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } return [ 'status' => 'ok', 'status_reason' => 'Backup store connectivity confirmed.', 'status_reason_key' => 'backup_connectivity_confirmed', 'status_reason_params' => [ 'latest_verified_age_seconds' => (string)($health['latest_verified_age_seconds'] ?? ''), 'encryption_key_id' => (string)($health['encryption']['key_id'] ?? ''), ], 'checked_at' => $checkedAt, 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } catch (Throwable $throwable) { return [ 'status' => 'down', 'status_reason' => 'Backup store probe failed: ' . $throwable->getMessage(), 'status_reason_key' => 'backup_probe_failed', 'status_reason_params' => ['error' => $throwable->getMessage()], 'checked_at' => $checkedAt, 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } } protected function probeMotorApiModule(array $config): array { $secretKey = trim((string)($config['secret_key']['parsed'] ?? '')); return $this->performHttpProbe( $this->buildUrlWithQuery('https://v1.motorapi.dk', '/usage'), ['X-AUTH-TOKEN: ' . $secretKey], 'MotorAPI', null, 'GET', null, fn(array $httpResponse, string $label): array => $this->evaluateProviderQuotaProbeResponse($httpResponse, $label, 'motorapi') ); } protected function probeOpenAiModule(array $config): array { $apiKey = (string)($config['api_key']['parsed'] ?? ''); return $this->performHttpProbe( 'https://api.openai.com/v1/models', [ 'Authorization: Bearer ' . $apiKey, 'Accept: application/json', ], 'OpenAI API' ); } protected function probeStripeModule(array $config): array { $secretKey = (string)($config['secret_key']['parsed'] ?? ''); return $this->performHttpProbe( 'https://api.stripe.com/v1/balance', ['Accept: application/json'], 'Stripe API', $secretKey . ':' ); } protected function probeFxRatesApiModule(array $config): array { $secretKey = trim((string)($config['secret_key']['parsed'] ?? '')); return $this->performHttpProbe( $this->buildUrlWithQuery('https://api.fxratesapi.com', '/latest', [ 'base' => 'EUR', 'currencies' => 'DKK', ]), ['X-AUTH-TOKEN: ' . $secretKey], 'FXRatesAPI' ); } protected function probeWeatherApiModule(array $config): array { $secretKey = trim((string)($config['secret_key']['parsed'] ?? '')); return $this->performHttpProbe( $this->buildUrlWithQuery('https://api.weatherapi.com/v1', '/current.json', [ 'key' => $secretKey, 'q' => 'Copenhagen', ]), ['Accept: application/json'], 'WeatherAPI' ); } protected function probeWorkfeedModule(array $config): array { $apiUrl = trim((string)($config['api_url']['parsed'] ?? '')); $companyId = trim((string)($config['CompanyID']['parsed'] ?? '')); $apiKey = trim((string)($config['api_key']['parsed'] ?? '')); return $this->performHttpProbe( $this->buildUrlWithQuery($apiUrl, '/companies/' . rawurlencode($companyId) . '/departments'), [ 'Accept: application/json', 'Authorization: ' . $apiKey, ], 'Workfeed API' ); } protected function probeGatewayApiModule(array $config): array { $apiToken = trim((string)($config['api_token']['parsed'] ?? '')); return $this->performHttpProbe( $this->buildUrlWithQuery('https://gatewayapi.eu', '/rest/me'), [ 'Authorization: Token ' . $apiToken, 'Accept: application/json', ], 'GatewayAPI' ); } protected function probeXlVaskModule(array $config): array { $username = trim((string)($config['username']['parsed'] ?? '')); $password = trim((string)($config['password']['parsed'] ?? '')); return $this->performHttpProbe( $this->buildUrlWithQuery('https://api.xlwash.com', '/customers'), ['Accept: application/json'], 'XLVask API', $username . ':' . $password ); } protected function probeEntraModule(array $config): array { $tenantId = trim((string)($config['entra_tenant_id']['parsed'] ?? '')); return $this->performHttpProbe( 'https://login.microsoftonline.com/' . rawurlencode($tenantId) . '/v2.0/.well-known/openid-configuration', ['Accept: application/json'], 'Microsoft Entra' ); } protected function probeLimbleModule(array $config): array { $clientId = trim((string)($config['client_id']['parsed'] ?? '')); $clientSecret = trim((string)($config['client_secret']['parsed'] ?? '')); return $this->performHttpProbe( $this->buildUrlWithQuery('https://api.limblecmms.com:443/v2', '/tasks', [ 'limit' => 1, 'page' => 1, ]), [ 'Accept: application/json', 'Content-Type: application/json', ], 'Limble API', $clientId . ':' . $clientSecret ); } protected function probeLicensePlateRecognizerModule(array $config): array { $apiKey = trim((string)($config['api_key']['parsed'] ?? '')); return $this->performHttpProbe( $this->buildUrlWithQuery(licenseplaterecognizer::configuredApiBaseUrl(), '/info/'), [ 'Authorization: Token ' . $apiKey, 'Accept: application/json', ], 'License Plate Recognizer', null, 'GET', null, fn(array $httpResponse, string $label): array => $this->evaluateLicensePlateRecognizerProbeResponse($httpResponse, $label) ); } protected function probeShellyModule(array $config): array { $deviceId = $this->findShellyProbeDeviceId(); if ($deviceId === null) { return [ 'status' => 'configured', 'status_reason' => 'Shelly cloud credentials are configured, but no known device id is available for a safe read-only probe.', 'status_reason_key' => 'shelly_no_device_id', 'status_reason_params' => [], 'checked_at' => date('c'), ]; } $serverUrl = trim((string)($config['server_url']['parsed'] ?? '')); $secretKey = trim((string)($config['secret_key']['parsed'] ?? '')); return $this->performHttpProbe( $this->buildUrlWithQuery($serverUrl, '/device/status', [ 'id' => $deviceId, 'auth_key' => $secretKey, ]), ['Accept: application/json'], 'Shelly server' ); } protected function probeSelfserveModule(array $config): array { $checkedAt = date('c'); $startedAt = microtime(true); try { $minutesIncluded = $this->normalizePositiveInt($config['machine_wash_minutes_included']['parsed'] ?? null); if ($minutesIncluded === null) { return [ 'status' => 'down', 'status_reason' => 'Self-serve machine minutes configuration is invalid.', 'status_reason_key' => 'selfserve_minutes_invalid', 'status_reason_params' => [], 'checked_at' => $checkedAt, 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } $minuteProductId = $this->normalizePositiveInt($config['minute_product']['parsed'] ?? null); if ($minuteProductId === null) { return [ 'status' => 'down', 'status_reason' => 'Self-serve minute product configuration is invalid.', 'status_reason_key' => 'selfserve_minute_product_invalid', 'status_reason_params' => [], 'checked_at' => $checkedAt, 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } $this->bootstrapSelfserveSchema(); if (!$this->selfserveMinuteProductExists($minuteProductId)) { return [ 'status' => 'down', 'status_reason' => 'Self-serve minute product #' . $minuteProductId . ' does not exist.', 'status_reason_key' => 'selfserve_minute_product_missing', 'status_reason_params' => ['productId' => $minuteProductId], 'checked_at' => $checkedAt, 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } return [ 'status' => 'ok', 'status_reason' => 'Self-serve schema and minute product configuration confirmed.', 'status_reason_key' => 'selfserve_configuration_confirmed', 'status_reason_params' => [], 'checked_at' => $checkedAt, 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } catch (Throwable $throwable) { return [ 'status' => 'down', 'status_reason' => 'Self-serve probe failed: ' . $throwable->getMessage(), 'status_reason_key' => 'selfserve_probe_failed', 'status_reason_params' => ['error' => $throwable->getMessage()], 'checked_at' => $checkedAt, 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), ]; } } protected function probeBirdModule(array $config): array { $serverUrl = trim((string)($config['server_url']['parsed'] ?? '')); $workspaceId = trim((string)($config['workspaceId']['parsed'] ?? $config['workplaceId']['parsed'] ?? '')); $channelId = trim((string)($config['channelId']['parsed'] ?? '')); $apiKey = trim((string)($config['api_key']['parsed'] ?? '')); return $this->performHttpProbe( $this->buildUrlWithQuery( $serverUrl, '/workspaces/' . rawurlencode($workspaceId) . '/channels/' . rawurlencode($channelId) . '/calls', ['limit' => 1] ), [ $this->buildBirdAuthorizationHeader($apiKey), 'Accept: application/json', ], 'Bird API' ); } protected function probeReachableUrlModule(array $config, string $urlVariable, string $label): array { $url = trim((string)($config[$urlVariable]['parsed'] ?? '')); return $this->performHttpProbe($url, ['Accept: application/json'], $label); } protected function buildUrlWithQuery(string $baseUrl, string $path, array $query = []): string { $baseUrl = rtrim(trim($baseUrl), '/'); if ($baseUrl === '') { return ''; } $url = $baseUrl; if ($path !== '') { $url .= '/' . ltrim($path, '/'); } $query = array_filter($query, static fn(mixed $value): bool => $value !== null && $value !== ''); if ($query !== []) { $url .= '?' . http_build_query($query); } return $url; } protected function performHttpProbe( string $url, array $headers, string $label, ?string $basicAuth = null, string $method = 'GET', ?string $body = null, ?callable $responseEvaluator = null ): array { $checkedAt = date('c'); if ($url === '') { return [ 'status' => 'down', 'status_reason' => $label . ' probe could not run because the endpoint is missing.', 'status_reason_key' => 'http_endpoint_missing', 'status_reason_params' => ['label' => $label], 'checked_at' => $checkedAt, ]; } $curl = curl_init($url); if ($curl === false) { return [ 'status' => 'down', 'status_reason' => $label . ' probe could not initialize cURL.', 'status_reason_key' => 'http_curl_init_failed', 'status_reason_params' => ['label' => $label], 'checked_at' => $checkedAt, ]; } curl_setopt_array($curl, [ CURLOPT_RETURNTRANSFER => true, CURLOPT_FOLLOWLOCATION => true, CURLOPT_CONNECTTIMEOUT => 3, CURLOPT_TIMEOUT => 5, CURLOPT_CUSTOMREQUEST => strtoupper($method), CURLOPT_HTTPHEADER => $headers, CURLOPT_SSL_VERIFYPEER => true, ]); if ($basicAuth !== null) { curl_setopt($curl, CURLOPT_USERPWD, $basicAuth); } if ($body !== null) { curl_setopt($curl, CURLOPT_POSTFIELDS, $body); } $startedAt = microtime(true); $body = curl_exec($curl); $httpStatus = (int)curl_getinfo($curl, CURLINFO_RESPONSE_CODE); $error = curl_error($curl); curl_close($curl); $httpResponse = [ 'checked_at' => $checkedAt, 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), 'http_status' => $httpStatus, 'body' => is_string($body) ? $body : '', 'error' => $error, ]; if ($body === false && $error !== '') { return [ 'status' => 'down', 'status_reason' => $label . ' probe failed: ' . $error, 'status_reason_key' => 'http_probe_failed', 'status_reason_params' => ['label' => $label, 'error' => $error], 'checked_at' => $checkedAt, 'latency_ms' => $httpResponse['latency_ms'], ]; } if ($responseEvaluator !== null) { $evaluated = $responseEvaluator($httpResponse, $label); if (is_array($evaluated)) { if (!isset($evaluated['checked_at'])) { $evaluated['checked_at'] = $httpResponse['checked_at']; } if (!isset($evaluated['latency_ms'])) { $evaluated['latency_ms'] = $httpResponse['latency_ms']; } if (($httpResponse['http_status'] ?? 0) > 0 && !isset($evaluated['http_status'])) { $evaluated['http_status'] = $httpResponse['http_status']; } return $evaluated; } } return $this->classifyHttpProbeResult($httpResponse, $label); } protected function classifyHttpProbeResult(array $httpResponse, string $label): array { $checkedAt = (string)($httpResponse['checked_at'] ?? date('c')); $latencyMs = $httpResponse['latency_ms'] ?? null; $httpStatus = (int)($httpResponse['http_status'] ?? 0); $error = trim((string)($httpResponse['error'] ?? '')); if ($error !== '') { return [ 'status' => 'down', 'status_reason' => $label . ' probe failed: ' . $error, 'status_reason_key' => 'http_probe_failed', 'status_reason_params' => ['label' => $label, 'error' => $error], 'checked_at' => $checkedAt, 'latency_ms' => $latencyMs, ]; } if ($httpStatus === 0) { return [ 'status' => 'down', 'status_reason' => $label . ' did not return an HTTP response.', 'status_reason_key' => 'http_no_response', 'status_reason_params' => ['label' => $label], 'checked_at' => $checkedAt, 'latency_ms' => $latencyMs, ]; } if ($httpStatus >= 200 && $httpStatus < 300) { return [ 'status' => 'ok', 'status_reason' => $label . ' connectivity confirmed.', 'status_reason_key' => 'http_ok', 'status_reason_params' => ['label' => $label], 'checked_at' => $checkedAt, 'latency_ms' => $latencyMs, 'http_status' => $httpStatus, ]; } if ($httpStatus === 429) { return [ 'status' => 'degraded', 'status_reason' => $label . ' probe was rate limited (HTTP 429).', 'status_reason_key' => 'http_rate_limited', 'status_reason_params' => ['label' => $label, 'httpStatus' => $httpStatus], 'checked_at' => $checkedAt, 'latency_ms' => $latencyMs, 'http_status' => $httpStatus, ]; } if ($httpStatus >= 500) { return [ 'status' => 'degraded', 'status_reason' => $label . ' returned HTTP ' . $httpStatus . '.', 'status_reason_key' => 'http_status', 'status_reason_params' => ['label' => $label, 'httpStatus' => $httpStatus], 'checked_at' => $checkedAt, 'latency_ms' => $latencyMs, 'http_status' => $httpStatus, ]; } if ($httpStatus >= 300) { return [ 'status' => 'down', 'status_reason' => $label . ' returned HTTP ' . $httpStatus . '.', 'status_reason_key' => 'http_status', 'status_reason_params' => ['label' => $label, 'httpStatus' => $httpStatus], 'checked_at' => $checkedAt, 'latency_ms' => $latencyMs, 'http_status' => $httpStatus, ]; } return [ 'status' => 'degraded', 'status_reason' => $label . ' returned an unexpected HTTP response.', 'status_reason_key' => 'http_unexpected_response', 'status_reason_params' => ['label' => $label], 'checked_at' => $checkedAt, 'latency_ms' => $latencyMs, 'http_status' => $httpStatus, ]; } protected function evaluateLicensePlateRecognizerProbeResponse(array $httpResponse, string $label): array { $classified = $this->classifyHttpProbeResult($httpResponse, $label); if (($classified['status'] ?? 'down') !== 'ok') { return $classified; } $decoded = json_decode((string)($httpResponse['body'] ?? ''), true); if (!is_array($decoded)) { return $this->providerQuotaUnavailableProbeResult( $httpResponse, $label, 'licenseplaterecognizer', 'unreadable_payload', [] ); } $usage = is_array($decoded['usage'] ?? null) ? $decoded['usage'] : []; $callsUsedRaw = $usage['calls'] ?? null; $quotaCallsRaw = $decoded['total_calls'] ?? null; if (!is_numeric($callsUsedRaw) || !is_numeric($quotaCallsRaw) || (int)$quotaCallsRaw <= 0) { $reason = !is_numeric($quotaCallsRaw) ? 'missing_limit' : 'missing_usage'; if (is_numeric($quotaCallsRaw) && (int)$quotaCallsRaw <= 0) { $reason = 'invalid_limit'; } return $this->providerQuotaUnavailableProbeResult( $httpResponse, $label, 'licenseplaterecognizer', $reason, $this->payloadKeyPaths($decoded) ); } $callsUsed = max(0, (int)$callsUsedRaw); $quotaCalls = max(1, (int)$quotaCallsRaw); $callsRemaining = max(0, $quotaCalls - $callsUsed); $usagePercent = round(($callsUsed / $quotaCalls) * 100, 2); $usagePayload = [ 'provider' => 'licenseplaterecognizer', 'calls_used' => $callsUsed, 'quota_calls' => $quotaCalls, 'calls_remaining' => $callsRemaining, 'usage_percent' => $usagePercent, 'version' => trim((string)($decoded['version'] ?? '')), ]; $reasonParams = [ 'label' => $label, 'used' => (string)$callsUsed, 'quota' => (string)$quotaCalls, 'remaining' => (string)$callsRemaining, 'percent' => number_format($usagePercent, 1, '.', ''), ]; if ($callsRemaining <= 0 || $usagePercent >= 100.0) { return [ 'status' => 'down', 'status_reason' => $label . ' quota is exhausted.', 'status_reason_key' => 'licenseplaterecognizer_quota_exhausted', 'status_reason_params' => $reasonParams, 'checked_at' => $httpResponse['checked_at'] ?? date('c'), 'latency_ms' => $httpResponse['latency_ms'] ?? null, 'http_status' => $httpResponse['http_status'] ?? null, 'usage' => $usagePayload, ]; } if ($usagePercent >= 90.0) { return [ 'status' => 'degraded', 'status_reason' => $label . ' quota usage is near the limit.', 'status_reason_key' => 'licenseplaterecognizer_quota_near_limit', 'status_reason_params' => $reasonParams, 'checked_at' => $httpResponse['checked_at'] ?? date('c'), 'latency_ms' => $httpResponse['latency_ms'] ?? null, 'http_status' => $httpResponse['http_status'] ?? null, 'usage' => $usagePayload, ]; } return [ 'status' => 'ok', 'status_reason' => $label . ' usage and quota are available.', 'status_reason_key' => 'licenseplaterecognizer_usage_available', 'status_reason_params' => $reasonParams, 'checked_at' => $httpResponse['checked_at'] ?? date('c'), 'latency_ms' => $httpResponse['latency_ms'] ?? null, 'http_status' => $httpResponse['http_status'] ?? null, 'usage' => $usagePayload, ]; } protected function evaluateProviderQuotaProbeResponse(array $httpResponse, string $label, string $moduleKey): array { $classified = $this->classifyHttpProbeResult($httpResponse, $label); if (($classified['status'] ?? 'down') !== 'ok') { return $classified; } $decoded = json_decode((string)($httpResponse['body'] ?? ''), true); if (!is_array($decoded)) { return $this->providerQuotaUnavailableProbeResult( $httpResponse, $label, $moduleKey, 'unreadable_payload', [] ); } $used = $this->findFirstNumericPayloadValue($decoded, ['used', 'usage', 'calls', 'messages_used', 'used_messages', 'sent', 'total_used']); $limit = $this->findFirstNumericPayloadValue($decoded, ['limit', 'quota', 'total', 'total_calls', 'messages_limit', 'max', 'allowed']); $remaining = $this->findFirstNumericPayloadValue($decoded, ['remaining', 'left', 'available', 'calls_remaining', 'messages_remaining']); if ($used === null && $limit !== null && $remaining !== null) { $used = max(0.0, $limit - $remaining); } if ($remaining === null && $used !== null && $limit !== null) { $remaining = max(0.0, $limit - $used); } if ($limit === null) { return $this->providerQuotaUnavailableProbeResult( $httpResponse, $label, $moduleKey, 'missing_limit', $this->payloadKeyPaths($decoded) ); } if ($limit <= 0) { return $this->providerQuotaUnavailableProbeResult( $httpResponse, $label, $moduleKey, 'invalid_limit', $this->payloadKeyPaths($decoded) ); } if ($used === null) { return $this->providerQuotaUnavailableProbeResult( $httpResponse, $label, $moduleKey, 'missing_usage', $this->payloadKeyPaths($decoded) ); } $usagePercent = round(($used / $limit) * 100, 2); $usagePayload = [ 'provider' => $moduleKey, 'calls_used' => $used, 'quota_calls' => $limit, 'calls_remaining' => $remaining, 'usage_percent' => $usagePercent, ]; if (($remaining !== null && $remaining <= 0) || $usagePercent >= 100.0) { return [ 'status' => 'down', 'status_reason' => $label . ' quota is exhausted.', 'status_reason_key' => 'provider_quota_exhausted', 'status_reason_params' => ['label' => $label, 'used' => (string)$used, 'quota' => (string)$limit, 'percent' => number_format($usagePercent, 1, '.', '')], 'checked_at' => $httpResponse['checked_at'] ?? date('c'), 'latency_ms' => $httpResponse['latency_ms'] ?? null, 'http_status' => $httpResponse['http_status'] ?? null, 'usage' => $usagePayload, ]; } if ($usagePercent >= 90.0) { return [ 'status' => 'degraded', 'status_reason' => $label . ' quota usage is near the limit.', 'status_reason_key' => 'provider_quota_near_limit', 'status_reason_params' => ['label' => $label, 'used' => (string)$used, 'quota' => (string)$limit, 'percent' => number_format($usagePercent, 1, '.', '')], 'checked_at' => $httpResponse['checked_at'] ?? date('c'), 'latency_ms' => $httpResponse['latency_ms'] ?? null, 'http_status' => $httpResponse['http_status'] ?? null, 'usage' => $usagePayload, ]; } return [ 'status' => 'ok', 'status_reason' => $label . ' quota usage is available.', 'status_reason_key' => 'provider_quota_available', 'status_reason_params' => ['label' => $label, 'used' => (string)$used, 'quota' => (string)$limit, 'percent' => number_format($usagePercent, 1, '.', '')], 'checked_at' => $httpResponse['checked_at'] ?? date('c'), 'latency_ms' => $httpResponse['latency_ms'] ?? null, 'http_status' => $httpResponse['http_status'] ?? null, 'usage' => $usagePayload, ]; } protected function providerQuotaUnavailableProbeResult(array $httpResponse, string $label, string $moduleKey, string $reason, array $detectedKeys = []): array { $usagePayload = [ 'provider' => $moduleKey, 'usage_available' => false, 'status' => 'unknown', 'calls_used' => null, 'quota_calls' => null, 'calls_remaining' => null, 'usage_percent' => null, 'unavailable_reason' => $reason, 'detected_keys' => $detectedKeys, ]; return [ 'status' => 'degraded', 'status_reason' => $label . ' responded, but quota usage could not be read.', 'status_reason_key' => 'provider_quota_unavailable', 'status_reason_params' => [ 'label' => $label, 'reason' => $reason, 'detected_keys' => implode(', ', array_slice($detectedKeys, 0, 12)), ], 'checked_at' => $httpResponse['checked_at'] ?? date('c'), 'latency_ms' => $httpResponse['latency_ms'] ?? null, 'http_status' => $httpResponse['http_status'] ?? null, 'usage' => $usagePayload, ]; } protected function findFirstNumericPayloadValue(array $payload, array $keys): ?float { foreach ($payload as $key => $value) { $normalizedKey = strtolower((string)$key); if (in_array($normalizedKey, $keys, true) && is_numeric($value)) { return (float)$value; } if (is_array($value)) { $nested = $this->findFirstNumericPayloadValue($value, $keys); if ($nested !== null) { return $nested; } } } return null; } protected function payloadKeyPaths(array $payload, string $prefix = ''): array { $paths = []; foreach ($payload as $key => $value) { $path = $prefix === '' ? (string)$key : $prefix . '.' . (string)$key; $paths[] = $path; if (is_array($value)) { array_push($paths, ...$this->payloadKeyPaths($value, $path)); } } return array_values(array_unique($paths)); } protected function evaluateRecaptchaProbeResponse(array $httpResponse, string $label): array { $classified = $this->classifyHttpProbeResult($httpResponse, $label); if (($classified['status'] ?? 'down') !== 'ok') { return $classified; } $decoded = json_decode((string)($httpResponse['body'] ?? ''), true); if (!is_array($decoded)) { return [ 'status' => 'degraded', 'status_reason' => $label . ' returned an unreadable response payload.', 'status_reason_key' => 'recaptcha_unreadable_payload', 'status_reason_params' => ['label' => $label], 'checked_at' => $httpResponse['checked_at'] ?? date('c'), 'latency_ms' => $httpResponse['latency_ms'] ?? null, 'http_status' => $httpResponse['http_status'] ?? null, ]; } $errorCodes = array_values(array_filter( array_map(static fn(mixed $value): string => trim((string)$value), (array)($decoded['error-codes'] ?? [])), static fn(string $value): bool => $value !== '' )); if (($decoded['success'] ?? false) === true || in_array('invalid-input-response', $errorCodes, true)) { return [ 'status' => 'ok', 'status_reason' => $label . ' connectivity confirmed.', 'status_reason_key' => 'http_ok', 'status_reason_params' => ['label' => $label], 'checked_at' => $httpResponse['checked_at'] ?? date('c'), 'latency_ms' => $httpResponse['latency_ms'] ?? null, 'http_status' => $httpResponse['http_status'] ?? null, ]; } if (in_array('invalid-input-secret', $errorCodes, true) || in_array('missing-input-secret', $errorCodes, true)) { return [ 'status' => 'down', 'status_reason' => $label . ' credentials were rejected by Google.', 'status_reason_key' => 'recaptcha_credentials_rejected', 'status_reason_params' => ['label' => $label], 'checked_at' => $httpResponse['checked_at'] ?? date('c'), 'latency_ms' => $httpResponse['latency_ms'] ?? null, 'http_status' => $httpResponse['http_status'] ?? null, ]; } return [ 'status' => 'degraded', 'status_reason' => $label . ' returned unexpected validation errors: ' . implode(', ', $errorCodes), 'status_reason_key' => 'recaptcha_validation_errors', 'status_reason_params' => ['label' => $label, 'errors' => implode(', ', $errorCodes)], 'checked_at' => $httpResponse['checked_at'] ?? date('c'), 'latency_ms' => $httpResponse['latency_ms'] ?? null, 'http_status' => $httpResponse['http_status'] ?? null, ]; } protected function validateBackupsStore(): void { new backup_store(); } protected function backupHealthSummary(): array { return (new backup_store())->healthSummary(); } protected function bootstrapSelfserveSchema(): void { selfserve_schema_bootstrap::ensureTables(); } protected function selfserveMinuteProductExists(int $productId): bool { global $db; $result = $db->query('SELECT id FROM products WHERE id = ' . (int)$productId . ' LIMIT 1'); if ($result === false) { throw new \RuntimeException('Failed to query the products table.'); } $row = $result->fetch_assoc(); return isset($row['id']) && (int)$row['id'] === $productId; } protected function normalizePositiveInt(mixed $value): ?int { if (is_int($value)) { return $value > 0 ? $value : null; } if (is_string($value) && preg_match('/^\d+$/', trim($value)) === 1) { $normalized = (int)trim($value); return $normalized > 0 ? $normalized : null; } if (is_float($value) && $value > 0 && floor($value) === $value) { return (int)$value; } return null; } protected function buildBirdAuthorizationHeader(string $apiKey): string { $apiKey = trim($apiKey); if ($apiKey === '') { return 'Authorization: AccessKey'; } if (preg_match('/^(Bearer|AccessKey)\s+/i', $apiKey) === 1) { return 'Authorization: ' . $apiKey; } return 'Authorization: AccessKey ' . $apiKey; } protected function findShellyProbeDeviceId(): ?string { global $db; $queries = [ "SELECT device_id FROM edge_gateway_relay_bindings WHERE deleted_at IS NULL AND device_id IS NOT NULL AND device_id != '' ORDER BY id DESC LIMIT 1", "SELECT device_id FROM edge_gateway_device_inventory WHERE deleted_at IS NULL AND device_id IS NOT NULL AND device_id != '' ORDER BY id DESC LIMIT 1", ]; foreach ($queries as $sql) { try { $result = $db->query($sql); if ($result === false) { continue; } $row = $result->fetch_assoc(); $deviceId = trim((string)($row['device_id'] ?? '')); if ($deviceId !== '') { return $deviceId; } } catch (Throwable) { continue; } } return null; } }