diff --git a/services/edge-broker/server.mjs b/services/edge-broker/server.mjs index 4475efd2..81576304 100644 --- a/services/edge-broker/server.mjs +++ b/services/edge-broker/server.mjs @@ -535,17 +535,29 @@ export function createBrokerServer(options = {}) { if (message.type === "TELEMETRY") { const payload = message.payload || {}; - const ingested = await ingestTelemetry(String(ws.gatewayId), payload); + let ingested = null; + let ingestError = null; + try { + ingested = await ingestTelemetry(String(ws.gatewayId), payload); + } catch (error) { + ingestError = error instanceof Error ? error.message : String(error); + } + const fallbackStatistics = { + system_metrics: payload?.metadata?.system_metrics || {}, + container_health: payload?.metadata?.container_health || {}, + }; broadcastGatewayEvent(String(ws.gatewayId), { type: "gateway.telemetry", gatewayId: String(ws.gatewayId), telemetry: payload, gateway: ingested?.gateway || ingested || null, + error: ingestError, }); broadcastGatewayEvent(String(ws.gatewayId), { type: "stats.updated", gatewayId: String(ws.gatewayId), - statistics: ingested?.statistics || ingested || null, + statistics: ingested?.statistics || ingested || fallbackStatistics, + error: ingestError, }); return; } diff --git a/services/edge-broker/test/broker.test.mjs b/services/edge-broker/test/broker.test.mjs index 3146b976..d005df47 100644 --- a/services/edge-broker/test/broker.test.mjs +++ b/services/edge-broker/test/broker.test.mjs @@ -380,3 +380,59 @@ test("broker survives telemetry ingestion failures for stale gateways", async () agent.terminate(); await broker.close(); }); + +test("broker still fans out telemetry when manager ingestion fails", async () => { + const broker = createBrokerServer({ + authMode: "stub", + validateAgent: async () => ({ id: "701", gateway_id: "701", label: "CPH Edge 01" }), + validateBrowserStream: async () => ({ + id: "stream-telemetry-fallback", + gateway_id: "701", + scopes: ["overview", "statistics"], + }), + ingestTelemetry: async () => { + throw new Error("manager unavailable"); + }, + }); + const address = await broker.listen(0); + const port = address.port; + + const browser = new WebSocket(`ws://127.0.0.1:${port}/ws/browser-gateway-stream?token=stream-token`); + const browserMessages = collectMessages(browser); + await new Promise((resolve) => browser.once("open", resolve)); + + const agent = new WebSocket(`ws://127.0.0.1:${port}/ws/agent?gatewayId=701&token=agent-token`); + await new Promise((resolve) => agent.once("open", resolve)); + + agent.send( + JSON.stringify({ + type: "TELEMETRY", + payload: { + status: "ONLINE", + metadata: { + system_metrics: { + cpu_usage_pct: 31, + }, + container_health: { + state: "ONLINE", + summary: "6/6 containers healthy", + }, + }, + }, + }) + ); + + await waitFor( + () => browserMessages.some((message) => message.type === "gateway.telemetry" && message.error === "manager unavailable"), + { description: "telemetry fanout after ingest failure" } + ); + assert.ok( + browserMessages.some( + (message) => message.type === "stats.updated" && message.statistics?.system_metrics?.cpu_usage_pct === 31 + ) + ); + + browser.terminate(); + agent.terminate(); + await broker.close(); +}); diff --git a/services/nginx/app/resources/edge-gateway-agent/agent.php b/services/nginx/app/resources/edge-gateway-agent/agent.php index 96d5ebd9..9d1aba5c 100644 --- a/services/nginx/app/resources/edge-gateway-agent/agent.php +++ b/services/nginx/app/resources/edge-gateway-agent/agent.php @@ -2157,7 +2157,7 @@ final class TruckwashEdgeAgent $endpoint = (string)$item['endpoint']; $payload = is_array($item['payload'] ?? null) ? (array)$item['payload'] : []; $brokerDispatch = $this->dispatchBrokerControlPlaneEvent($endpoint, $payload); - if ($brokerDispatch !== true) { + if ($brokerDispatch !== true || $this->shouldPersistControlPlaneEventOverHttp($endpoint)) { $this->http->post($endpoint, $payload, $timeoutSeconds); } $this->stateStore->removeOutboxItem((int)$item['id']); @@ -2263,7 +2263,7 @@ final class TruckwashEdgeAgent private function sendControlPlaneEvent(string $endpoint, array $payload, string $type): bool { $brokerDispatch = $this->dispatchBrokerControlPlaneEvent($endpoint, $payload); - if ($brokerDispatch === true) { + if ($brokerDispatch === true && !$this->shouldPersistControlPlaneEventOverHttp($endpoint)) { return true; } @@ -2272,6 +2272,14 @@ final class TruckwashEdgeAgent $this->recordSuccessfulSync(); return true; } catch (Throwable $throwable) { + if ($brokerDispatch === true && !$this->shouldPersistControlPlaneEventOverHttp($endpoint)) { + $this->recordTransportFailure( + 'Broker dispatched ' . $type . ' but HTTP persistence failed on ' . $endpoint, + $throwable + ); + return true; + } + $this->stateStore->enqueue($type, $endpoint, $payload); $this->recordTransportFailure( 'Queued ' . $type . ' to local outbox after transport failure on ' . $endpoint, @@ -2281,6 +2289,11 @@ final class TruckwashEdgeAgent } } + private function shouldPersistControlPlaneEventOverHttp(string $endpoint): bool + { + return preg_match('#/edge-agent/gateways/\d+/heartbeat$#', $endpoint) === 1; + } + private function requestControlPlaneEvent(string $endpoint, array $payload, string $type, int $timeoutSeconds = 20): ?array { $brokerDispatch = $this->dispatchBrokerControlPlaneEvent($endpoint, $payload); diff --git a/services/nginx/app/tests/Unit/Selfserve/EdgeGatewayUpdateLifecycleTest.php b/services/nginx/app/tests/Unit/Selfserve/EdgeGatewayUpdateLifecycleTest.php index fcd094b5..333b7a5c 100644 --- a/services/nginx/app/tests/Unit/Selfserve/EdgeGatewayUpdateLifecycleTest.php +++ b/services/nginx/app/tests/Unit/Selfserve/EdgeGatewayUpdateLifecycleTest.php @@ -123,6 +123,10 @@ it('exposes update payload, credential rotation, cancel endpoints, and operation expect($agentSource)->toContain('private BrokerWebSocketClient $brokerClient;'); expect($agentSource)->toContain('private AgentShellBridge $shellBridge;'); expect($agentSource)->toContain('private function dispatchBrokerControlPlaneEvent(string $endpoint, array $payload): ?bool'); + expect($agentSource)->toContain('private function shouldPersistControlPlaneEventOverHttp(string $endpoint): bool'); + expect($agentSource)->toContain('if ($brokerDispatch === true && !$this->shouldPersistControlPlaneEventOverHttp($endpoint))'); + expect($agentSource)->toContain('if ($brokerDispatch !== true || $this->shouldPersistControlPlaneEventOverHttp($endpoint))'); + expect($agentSource)->toContain('Broker dispatched \' . $type . \' but HTTP persistence failed on \' . $endpoint'); expect($agentSource)->toContain("'type' => 'TELEMETRY'"); expect($agentSource)->toContain("'type' => 'TASK_EVENT'"); expect($agentSource)->toContain("'type' => 'TASK_RESULT'");