Fix edge gateway relay command draining

This commit is contained in:
Jeppe Bundgaard
2026-07-02 12:41:34 +02:00
parent 62c1393f62
commit 243d68ab59
9 changed files with 1170 additions and 38 deletions
+453
View File
@@ -0,0 +1,453 @@
import { spawn } from "node:child_process";
import crypto from "node:crypto";
import fs from "node:fs";
import http from "node:http";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const DEFAULT_AGENT_PATH = path.join(
repoRoot,
"services/nginx/app/resources/edge-gateway-agent/agent.php"
);
const DEFAULT_PHP_IMAGE = "php:8.2-cli-bookworm";
const DEFAULT_TIMEOUT_MS = 12000;
function parseArgs(argv = process.argv.slice(2)) {
const options = {
agentPath: DEFAULT_AGENT_PATH,
phpImage: DEFAULT_PHP_IMAGE,
timeoutMs: DEFAULT_TIMEOUT_MS,
keepTemp: false,
help: false,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
const next = argv[index + 1];
switch (arg) {
case "--agent-path":
options.agentPath = path.resolve(String(next || "").trim());
index += 1;
break;
case "--php-image":
options.phpImage = String(next || "").trim() || DEFAULT_PHP_IMAGE;
index += 1;
break;
case "--timeout-ms":
options.timeoutMs = Number.parseInt(String(next || ""), 10) || DEFAULT_TIMEOUT_MS;
index += 1;
break;
case "--keep-temp":
options.keepTemp = true;
break;
case "--help":
case "-h":
options.help = true;
break;
default:
throw new Error(`Unknown argument: ${arg}`);
}
}
return options;
}
function printUsage() {
process.stdout.write(`Usage:
node scripts/edge-agent-command-drain-proof.mjs [options]
Verifies that a broker-connected PHP compose edge agent still drains API-queued
SET_RELAY_STATE jobs to the LAN worker /relay/switch endpoint.
Options:
--agent-path <path> PHP agent artifact to execute.
Default: ${DEFAULT_AGENT_PATH}
--php-image <image> Docker PHP image with curl, sqlite3, and pdo_sqlite.
Default: ${DEFAULT_PHP_IMAGE}
--timeout-ms <ms> Proof timeout. Default: ${DEFAULT_TIMEOUT_MS}
--keep-temp Keep the temporary config/runtime directory.
--help Show this help text.
`);
}
function readJson(request) {
return new Promise((resolve) => {
let raw = "";
request.setEncoding("utf8");
request.on("data", (chunk) => {
raw += chunk;
});
request.on("end", () => {
if (raw.trim() === "") {
resolve({});
return;
}
try {
resolve(JSON.parse(raw));
} catch {
resolve({ __invalid: raw });
}
});
});
}
function sendJson(response, status, payload) {
const body = JSON.stringify(payload);
response.writeHead(status, {
"content-type": "application/json; charset=utf-8",
"content-length": Buffer.byteLength(body),
});
response.end(body);
}
function listen(server) {
return new Promise((resolve) => {
server.listen(0, "127.0.0.1", () => resolve(server.address().port));
});
}
function closeServer(server) {
return new Promise((resolve) => {
server.close(() => resolve());
});
}
function websocketAcceptKey(key) {
return crypto
.createHash("sha1")
.update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`)
.digest("base64");
}
function createBrokerServer(state) {
const sockets = new Set();
const server = net.createServer((socket) => {
sockets.add(socket);
socket.on("close", () => sockets.delete(socket));
let buffer = "";
socket.on("data", (chunk) => {
buffer += chunk.toString("binary");
if (state.brokerHandshakeSeen || !buffer.includes("\r\n\r\n")) {
return;
}
const requestText = Buffer.from(buffer, "binary").toString("utf8");
const key = requestText.match(/Sec-WebSocket-Key:\s*(.+)\r\n/i)?.[1]?.trim();
const requestLine = requestText.split("\r\n")[0] || "";
if (!requestLine.includes("/ws/agent?")) {
state.failure = new Error(`unexpected broker path: ${requestLine}`);
}
if (!key) {
state.failure = new Error("broker handshake missing Sec-WebSocket-Key");
return;
}
socket.write([
"HTTP/1.1 101 Switching Protocols",
"Upgrade: websocket",
"Connection: Upgrade",
`Sec-WebSocket-Accept: ${websocketAcceptKey(key)}`,
"",
"",
].join("\r\n"));
state.brokerHandshakeSeen = true;
buffer = "";
});
});
return { server, sockets };
}
function createWorkerServer(state) {
return http.createServer(async (request, response) => {
const url = new URL(request.url, "http://127.0.0.1");
const body = await readJson(request);
state.requests.push({ service: "worker", method: request.method, path: url.pathname, body });
if (request.method === "GET" && url.pathname === "/health") {
sendJson(response, 200, { status: "healthy", timestamp: new Date().toISOString() });
return;
}
if (request.method === "POST" && url.pathname === "/relay/switch") {
state.relaySwitchSeen = true;
if (body.local_ip !== "10.123.0.31" || body.channel !== 0 || body.on !== true) {
state.failure = new Error(`unexpected relay switch payload: ${JSON.stringify(body)}`);
}
sendJson(response, 200, {
online: true,
on: true,
output: true,
raw: { source: "fake-worker" },
});
return;
}
sendJson(response, 404, { message: "not found" });
});
}
function createApiServer(state, brokerPort, workerPort) {
return http.createServer(async (request, response) => {
const url = new URL(request.url, "http://127.0.0.1");
const body = await readJson(request);
state.requests.push({ service: "api", method: request.method, path: url.pathname, body });
if (request.method === "POST" && url.pathname === "/edge-agent/gateways/42/heartbeat") {
sendJson(response, 200, { data: { ok: true, broker_url: `ws://127.0.0.1:${brokerPort}` } });
return;
}
if (request.method === "POST" && url.pathname === "/edge-agent/gateways/42/selfserve/machine-signal-bindings") {
sendJson(response, 200, { data: { monitors: [] } });
return;
}
if (request.method === "POST" && url.pathname === "/edge-agent/gateways/42/commands/poll") {
state.commandPollSeen = true;
if (body.wait_seconds !== 0) {
state.failure = new Error(
`broker-connected command poll should be non-blocking, got wait_seconds=${body.wait_seconds}`
);
}
if (!state.commandDelivered) {
state.commandDelivered = true;
sendJson(response, 200, {
data: {
id: 77,
command_type: "SET_RELAY_STATE",
payload: {
localIp: "10.123.0.31",
channel: 0,
on: true,
relayId: "relay-proof",
},
},
});
return;
}
sendJson(response, 200, { data: null });
return;
}
if (request.method === "POST" && url.pathname === "/edge-agent/gateways/42/commands/77/result") {
state.resultSeen = true;
if (body.ok !== true || body.result?.on !== true || body.result?.raw?.source !== "fake-worker") {
state.failure = new Error(`unexpected command result: ${JSON.stringify(body)}`);
}
sendJson(response, 200, { data: { acknowledged: true } });
return;
}
sendJson(response, 404, { message: "not found", path: url.pathname, workerPort });
});
}
function writeConfig(tempDir, apiPort, brokerPort, workerPort) {
const containerProofDir = "/proof";
const runtimeDir = `${containerProofDir}/runtime`;
const config = {
apiUrl: `http://127.0.0.1:${apiPort}`,
brokerUrl: `ws://127.0.0.1:${brokerPort}`,
gatewayId: 42,
agentToken: "agent-token",
installDir: containerProofDir,
runtimeDir,
stateDatabasePath: `${runtimeDir}/gateway-state.sqlite`,
workerBaseUrl: `http://127.0.0.1:${workerPort}`,
heartbeatIntervalSeconds: 60,
operationPollTimeoutSeconds: 20,
};
const configPath = path.join(tempDir, "config.json");
fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
return { configPath, containerConfigPath: `${containerProofDir}/config.json` };
}
function spawnAgent({ agentPath, phpImage, tempDir, containerConfigPath }) {
return spawn("docker", [
"run",
"--rm",
"--network",
"host",
"-v",
`${agentPath}:/agent.php:ro`,
"-v",
`${tempDir}:/proof`,
phpImage,
"php",
"/agent.php",
"--config",
containerConfigPath,
], { stdio: ["ignore", "pipe", "pipe"] });
}
async function stopChild(child) {
if (child.exitCode !== null || child.signalCode !== null) {
return;
}
child.kill("SIGTERM");
const hardKill = setTimeout(() => {
if (child.exitCode === null && child.signalCode === null) {
child.kill("SIGKILL");
}
}, 1500);
await Promise.race([
new Promise((resolve) => child.once("exit", resolve)),
new Promise((resolve) => setTimeout(resolve, 2200)),
]);
clearTimeout(hardKill);
}
function evidenceFromState(state, childExited) {
return {
brokerHandshakeSeen: state.brokerHandshakeSeen,
commandPollSeen: state.commandPollSeen,
relaySwitchSeen: state.relaySwitchSeen,
resultSeen: state.resultSeen,
agentStayedRunningUntilProofComplete: !childExited,
};
}
export async function runProof(options) {
if (process.platform !== "linux") {
throw new Error("This proof uses Docker --network host and currently expects Linux.");
}
if (!fs.existsSync(options.agentPath)) {
throw new Error(`Agent artifact not found: ${options.agentPath}`);
}
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "edge-agent-command-drain-proof-"));
fs.mkdirSync(path.join(tempDir, "runtime"), { recursive: true });
const state = {
brokerHandshakeSeen: false,
commandPollSeen: false,
relaySwitchSeen: false,
resultSeen: false,
commandDelivered: false,
failure: null,
requests: [],
};
const broker = createBrokerServer(state);
const workerServer = createWorkerServer(state);
let apiServer = null;
let child = null;
let stdout = "";
let stderr = "";
let childExited = false;
try {
const brokerPort = await listen(broker.server);
const workerPort = await listen(workerServer);
apiServer = createApiServer(state, brokerPort, workerPort);
const apiPort = await listen(apiServer);
const { containerConfigPath } = writeConfig(tempDir, apiPort, brokerPort, workerPort);
child = spawnAgent({ ...options, tempDir, containerConfigPath });
child.stdout.on("data", (chunk) => {
stdout += chunk.toString();
});
child.stderr.on("data", (chunk) => {
stderr += chunk.toString();
});
child.once("exit", () => {
childExited = true;
});
const deadline = Date.now() + options.timeoutMs;
while (Date.now() < deadline && !state.failure && !childExited) {
if (state.brokerHandshakeSeen && state.commandPollSeen && state.relaySwitchSeen && state.resultSeen) {
break;
}
await new Promise((resolve) => setTimeout(resolve, 100));
}
const evidence = evidenceFromState(state, childExited);
if (
state.failure ||
!state.brokerHandshakeSeen ||
!state.commandPollSeen ||
!state.relaySwitchSeen ||
!state.resultSeen
) {
const error = state.failure || new Error("missing proof evidence");
error.evidence = evidence;
error.requests = state.requests;
error.stdout = stdout.slice(-3000);
error.stderr = stderr.slice(-3000);
throw error;
}
return {
evidence,
agentPath: options.agentPath,
phpImage: options.phpImage,
tempDir,
requestCount: state.requests.length,
};
} finally {
if (child) {
await stopChild(child);
}
for (const socket of broker.sockets) {
socket.destroy();
}
await Promise.allSettled([
closeServer(broker.server),
closeServer(workerServer),
apiServer ? closeServer(apiServer) : Promise.resolve(),
]);
if (!options.keepTemp) {
fs.rmSync(tempDir, { recursive: true, force: true });
}
}
}
async function main() {
const options = parseArgs();
if (options.help) {
printUsage();
return;
}
const result = await runProof(options);
process.stdout.write("PASS broker-connected API command poll triggered local relay switch and posted result\n");
process.stdout.write(`${JSON.stringify(result.evidence)}\n`);
process.stdout.write(`Agent: ${result.agentPath}\n`);
process.stdout.write(`PHP image: ${result.phpImage}\n`);
if (options.keepTemp) {
process.stdout.write(`Temp dir: ${result.tempDir}\n`);
}
}
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch((error) => {
process.stderr.write(`FAIL ${error.message}\n`);
if (error.evidence) {
process.stderr.write(`Evidence: ${JSON.stringify(error.evidence)}\n`);
}
if (error.requests) {
process.stderr.write(`Requests: ${JSON.stringify(error.requests, null, 2)}\n`);
}
if (error.stdout) {
process.stderr.write(`stdout: ${error.stdout}\n`);
}
if (error.stderr) {
process.stderr.write(`stderr: ${error.stderr}\n`);
}
process.exit(1);
});
}
+619
View File
@@ -0,0 +1,619 @@
import { spawn } from "node:child_process";
import crypto from "node:crypto";
import fs from "node:fs";
import http from "node:http";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const DEFAULT_AGENT_PATH = path.join(
repoRoot,
"services/nginx/app/resources/edge-gateway-agent/agent.php"
);
const DEFAULT_WORKER_PATH = path.join(
repoRoot,
"services/nginx/app/resources/edge-gateway-agent/lan-worker.php"
);
const DEFAULT_PHP_IMAGE = "php:8.2-cli-bookworm";
const DEFAULT_TIMEOUT_MS = 15000;
const AGENT_TOKEN = "agent-token";
function parseArgs(argv = process.argv.slice(2)) {
const options = {
agentPath: DEFAULT_AGENT_PATH,
workerPath: DEFAULT_WORKER_PATH,
phpImage: DEFAULT_PHP_IMAGE,
timeoutMs: DEFAULT_TIMEOUT_MS,
keepTemp: false,
help: false,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
const next = argv[index + 1];
switch (arg) {
case "--agent-path":
options.agentPath = path.resolve(String(next || "").trim());
index += 1;
break;
case "--worker-path":
options.workerPath = path.resolve(String(next || "").trim());
index += 1;
break;
case "--php-image":
options.phpImage = String(next || "").trim() || DEFAULT_PHP_IMAGE;
index += 1;
break;
case "--timeout-ms":
options.timeoutMs = Number.parseInt(String(next || ""), 10) || DEFAULT_TIMEOUT_MS;
index += 1;
break;
case "--keep-temp":
options.keepTemp = true;
break;
case "--help":
case "-h":
options.help = true;
break;
default:
throw new Error(`Unknown argument: ${arg}`);
}
}
return options;
}
function printUsage() {
process.stdout.write(`Usage:
node scripts/edge-agent-to-shelly-proof.mjs [options]
Runs the PHP edge agent and real LAN worker against fake broker, API, and
Shelly RPC endpoints. Verifies that a broker-connected SET_RELAY_STATE command
drains from the API, reaches the worker, triggers a Shelly-style Switch.Set
call, reads Switch.GetStatus, and posts the command result.
Options:
--agent-path <path> PHP agent artifact to execute.
Default: ${DEFAULT_AGENT_PATH}
--worker-path <path> PHP LAN worker artifact to execute.
Default: ${DEFAULT_WORKER_PATH}
--php-image <image> Docker PHP image with curl, sqlite3, and pdo_sqlite.
Default: ${DEFAULT_PHP_IMAGE}
--timeout-ms <ms> Proof timeout. Default: ${DEFAULT_TIMEOUT_MS}
--keep-temp Keep the temporary config/runtime directory.
--help Show this help text.
`);
}
function readJson(request) {
return new Promise((resolve) => {
let raw = "";
request.setEncoding("utf8");
request.on("data", (chunk) => {
raw += chunk;
});
request.on("end", () => {
if (raw.trim() === "") {
resolve({});
return;
}
try {
resolve(JSON.parse(raw));
} catch {
resolve({ __invalid: raw });
}
});
});
}
function sendJson(response, status, payload) {
const body = JSON.stringify(payload);
response.writeHead(status, {
"content-type": "application/json; charset=utf-8",
"content-length": Buffer.byteLength(body),
});
response.end(body);
}
function requestJson({ method = "GET", port, path: requestPath, body = null, headers = {} }) {
return new Promise((resolve, reject) => {
const payload = body === null ? null : JSON.stringify(body);
const request = http.request({
hostname: "127.0.0.1",
port,
path: requestPath,
method,
headers: {
accept: "application/json",
...(payload === null ? {} : {
"content-type": "application/json",
"content-length": Buffer.byteLength(payload),
}),
...headers,
},
timeout: 1000,
}, (response) => {
let raw = "";
response.setEncoding("utf8");
response.on("data", (chunk) => {
raw += chunk;
});
response.on("end", () => {
let decoded = {};
try {
decoded = raw.trim() === "" ? {} : JSON.parse(raw);
} catch {
decoded = { __invalid: raw };
}
resolve({ status: response.statusCode || 0, body: decoded });
});
});
request.on("error", reject);
request.on("timeout", () => {
request.destroy(new Error("request timed out"));
});
if (payload !== null) {
request.write(payload);
}
request.end();
});
}
function listen(server) {
return new Promise((resolve) => {
server.listen(0, "127.0.0.1", () => resolve(server.address().port));
});
}
function closeServer(server) {
return new Promise((resolve) => {
server.close(() => resolve());
});
}
async function reservePort() {
const server = net.createServer();
const port = await new Promise((resolve) => {
server.listen(0, "127.0.0.1", () => resolve(server.address().port));
});
await closeServer(server);
return port;
}
function websocketAcceptKey(key) {
return crypto
.createHash("sha1")
.update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`)
.digest("base64");
}
function createBrokerServer(state) {
const sockets = new Set();
const server = net.createServer((socket) => {
sockets.add(socket);
socket.on("close", () => sockets.delete(socket));
let buffer = "";
socket.on("data", (chunk) => {
buffer += chunk.toString("binary");
if (state.brokerHandshakeSeen || !buffer.includes("\r\n\r\n")) {
return;
}
const requestText = Buffer.from(buffer, "binary").toString("utf8");
const key = requestText.match(/Sec-WebSocket-Key:\s*(.+)\r\n/i)?.[1]?.trim();
const requestLine = requestText.split("\r\n")[0] || "";
if (!requestLine.includes("/ws/agent?")) {
state.failure = new Error(`unexpected broker path: ${requestLine}`);
}
if (!key) {
state.failure = new Error("broker handshake missing Sec-WebSocket-Key");
return;
}
socket.write([
"HTTP/1.1 101 Switching Protocols",
"Upgrade: websocket",
"Connection: Upgrade",
`Sec-WebSocket-Accept: ${websocketAcceptKey(key)}`,
"",
"",
].join("\r\n"));
state.brokerHandshakeSeen = true;
buffer = "";
});
});
return { server, sockets };
}
function createShellyServer(state) {
return http.createServer((request, response) => {
const url = new URL(request.url, "http://127.0.0.1");
state.requests.push({
service: "shelly",
method: request.method,
path: url.pathname,
query: Object.fromEntries(url.searchParams.entries()),
});
if (request.method === "GET" && url.pathname === "/rpc/Switch.Set") {
state.shellySwitchSetSeen = true;
if (url.searchParams.get("id") !== "0" || url.searchParams.get("on") !== "true") {
state.failure = new Error(`unexpected Shelly Switch.Set query: ${url.search}`);
}
sendJson(response, 200, { was_on: false, output: true });
return;
}
if (request.method === "GET" && url.pathname === "/rpc/Switch.GetStatus") {
state.shellyStatusSeen = true;
if (url.searchParams.get("id") !== "0") {
state.failure = new Error(`unexpected Shelly Switch.GetStatus query: ${url.search}`);
}
sendJson(response, 200, { id: 0, output: true, source: "fake-shelly-rpc" });
return;
}
if (url.pathname.startsWith("/relay/")) {
state.failure = new Error(`legacy Shelly endpoint should not be used for generation 2 proof: ${url.pathname}`);
}
sendJson(response, 404, { message: "not found" });
});
}
function createApiServer(state, brokerPort, shellyAddress) {
return http.createServer(async (request, response) => {
const url = new URL(request.url, "http://127.0.0.1");
const body = await readJson(request);
state.requests.push({ service: "api", method: request.method, path: url.pathname, body });
if (request.method === "POST" && url.pathname === "/edge-agent/gateways/42/heartbeat") {
sendJson(response, 200, { data: { ok: true, broker_url: `ws://127.0.0.1:${brokerPort}` } });
return;
}
if (request.method === "POST" && url.pathname === "/edge-agent/gateways/42/selfserve/machine-signal-bindings") {
sendJson(response, 200, { data: { monitors: [] } });
return;
}
if (request.method === "POST" && url.pathname === "/edge-agent/gateways/42/commands/poll") {
state.commandPollSeen = true;
if (body.wait_seconds !== 0) {
state.failure = new Error(
`broker-connected command poll should be non-blocking, got wait_seconds=${body.wait_seconds}`
);
}
if (!state.commandDelivered) {
state.commandDelivered = true;
sendJson(response, 200, {
data: {
id: 77,
command_type: "SET_RELAY_STATE",
payload: {
localIp: shellyAddress,
local_ip: shellyAddress,
channel: 0,
on: true,
relayId: "relay-proof",
relay_id: "relay-proof",
deviceGeneration: 2,
device_generation: 2,
},
},
});
return;
}
sendJson(response, 200, { data: null });
return;
}
if (request.method === "POST" && url.pathname === "/edge-agent/gateways/42/commands/77/result") {
state.resultSeen = true;
if (
body.ok !== true ||
body.result?.on !== true ||
body.result?.output !== true ||
body.result?.raw?.source !== "fake-shelly-rpc"
) {
state.failure = new Error(`unexpected command result: ${JSON.stringify(body)}`);
}
sendJson(response, 200, { data: { acknowledged: true } });
return;
}
sendJson(response, 404, { message: "not found", path: url.pathname });
});
}
function writeConfig(tempDir, apiPort, brokerPort, workerPort) {
const containerProofDir = "/proof";
const runtimeDir = `${containerProofDir}/runtime`;
const config = {
apiUrl: `http://127.0.0.1:${apiPort}`,
brokerUrl: `ws://127.0.0.1:${brokerPort}`,
gatewayId: 42,
agentToken: AGENT_TOKEN,
installDir: containerProofDir,
runtimeDir,
stateDatabasePath: `${runtimeDir}/gateway-state.sqlite`,
workerBaseUrl: `http://127.0.0.1:${workerPort}`,
heartbeatIntervalSeconds: 60,
operationPollTimeoutSeconds: 20,
};
const configPath = path.join(tempDir, "config.json");
fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
return { containerConfigPath: `${containerProofDir}/config.json` };
}
function spawnWorker({ workerPath, phpImage, workerPort }) {
return spawn("docker", [
"run",
"--rm",
"--network",
"host",
"-e",
`TRUCKWASH_WORKER_TOKEN=${AGENT_TOKEN}`,
"-v",
`${workerPath}:/lan-worker.php:ro`,
phpImage,
"php",
"-S",
`127.0.0.1:${workerPort}`,
"/lan-worker.php",
], { stdio: ["ignore", "pipe", "pipe"] });
}
function spawnAgent({ agentPath, phpImage, tempDir, containerConfigPath }) {
return spawn("docker", [
"run",
"--rm",
"--network",
"host",
"-v",
`${agentPath}:/agent.php:ro`,
"-v",
`${tempDir}:/proof`,
phpImage,
"php",
"/agent.php",
"--config",
containerConfigPath,
], { stdio: ["ignore", "pipe", "pipe"] });
}
async function waitForWorker(workerPort, child, timeoutMs) {
const deadline = Date.now() + timeoutMs;
let lastError = null;
while (Date.now() < deadline) {
if (child.exitCode !== null || child.signalCode !== null) {
throw new Error(`LAN worker exited before becoming healthy: ${child.exitCode ?? child.signalCode}`);
}
try {
const response = await requestJson({ port: workerPort, path: "/health" });
if (response.status === 200 && response.body?.service === "lan-worker") {
return;
}
} catch (error) {
lastError = error;
}
await new Promise((resolve) => setTimeout(resolve, 100));
}
throw lastError || new Error("LAN worker did not become healthy");
}
async function stopChild(child) {
if (child.exitCode !== null || child.signalCode !== null) {
return;
}
child.kill("SIGTERM");
const hardKill = setTimeout(() => {
if (child.exitCode === null && child.signalCode === null) {
child.kill("SIGKILL");
}
}, 1500);
await Promise.race([
new Promise((resolve) => child.once("exit", resolve)),
new Promise((resolve) => setTimeout(resolve, 2200)),
]);
clearTimeout(hardKill);
}
function evidenceFromState(state, agentExited, workerExited) {
return {
brokerHandshakeSeen: state.brokerHandshakeSeen,
commandPollSeen: state.commandPollSeen,
shellySwitchSetSeen: state.shellySwitchSetSeen,
shellyStatusSeen: state.shellyStatusSeen,
resultSeen: state.resultSeen,
agentStayedRunningUntilProofComplete: !agentExited,
workerStayedRunningUntilProofComplete: !workerExited,
};
}
export async function runProof(options) {
if (process.platform !== "linux") {
throw new Error("This proof uses Docker --network host and currently expects Linux.");
}
if (!fs.existsSync(options.agentPath)) {
throw new Error(`Agent artifact not found: ${options.agentPath}`);
}
if (!fs.existsSync(options.workerPath)) {
throw new Error(`LAN worker artifact not found: ${options.workerPath}`);
}
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "edge-agent-to-shelly-proof-"));
fs.mkdirSync(path.join(tempDir, "runtime"), { recursive: true });
const state = {
brokerHandshakeSeen: false,
commandPollSeen: false,
shellySwitchSetSeen: false,
shellyStatusSeen: false,
resultSeen: false,
commandDelivered: false,
failure: null,
requests: [],
};
const broker = createBrokerServer(state);
const shellyServer = createShellyServer(state);
let apiServer = null;
let agent = null;
let worker = null;
let agentStdout = "";
let agentStderr = "";
let workerStdout = "";
let workerStderr = "";
let agentExited = false;
let workerExited = false;
try {
const brokerPort = await listen(broker.server);
const shellyPort = await listen(shellyServer);
const workerPort = await reservePort();
const shellyAddress = `127.0.0.1:${shellyPort}`;
apiServer = createApiServer(state, brokerPort, shellyAddress);
const apiPort = await listen(apiServer);
const { containerConfigPath } = writeConfig(tempDir, apiPort, brokerPort, workerPort);
worker = spawnWorker({ ...options, workerPort });
worker.stdout.on("data", (chunk) => {
workerStdout += chunk.toString();
});
worker.stderr.on("data", (chunk) => {
workerStderr += chunk.toString();
});
worker.once("exit", () => {
workerExited = true;
});
await waitForWorker(workerPort, worker, 5000);
agent = spawnAgent({ ...options, tempDir, containerConfigPath });
agent.stdout.on("data", (chunk) => {
agentStdout += chunk.toString();
});
agent.stderr.on("data", (chunk) => {
agentStderr += chunk.toString();
});
agent.once("exit", () => {
agentExited = true;
});
const deadline = Date.now() + options.timeoutMs;
while (Date.now() < deadline && !state.failure && !agentExited && !workerExited) {
if (
state.brokerHandshakeSeen &&
state.commandPollSeen &&
state.shellySwitchSetSeen &&
state.shellyStatusSeen &&
state.resultSeen
) {
break;
}
await new Promise((resolve) => setTimeout(resolve, 100));
}
const evidence = evidenceFromState(state, agentExited, workerExited);
if (
state.failure ||
!state.brokerHandshakeSeen ||
!state.commandPollSeen ||
!state.shellySwitchSetSeen ||
!state.shellyStatusSeen ||
!state.resultSeen
) {
const error = state.failure || new Error("missing proof evidence");
error.evidence = evidence;
error.requests = state.requests;
error.agentStdout = agentStdout.slice(-3000);
error.agentStderr = agentStderr.slice(-3000);
error.workerStdout = workerStdout.slice(-3000);
error.workerStderr = workerStderr.slice(-3000);
throw error;
}
return {
evidence,
agentPath: options.agentPath,
workerPath: options.workerPath,
phpImage: options.phpImage,
tempDir,
requestCount: state.requests.length,
};
} finally {
if (agent) {
await stopChild(agent);
}
if (worker) {
await stopChild(worker);
}
for (const socket of broker.sockets) {
socket.destroy();
}
await Promise.allSettled([
closeServer(broker.server),
closeServer(shellyServer),
apiServer ? closeServer(apiServer) : Promise.resolve(),
]);
if (!options.keepTemp) {
fs.rmSync(tempDir, { recursive: true, force: true });
}
}
}
async function main() {
const options = parseArgs();
if (options.help) {
printUsage();
return;
}
const result = await runProof(options);
process.stdout.write("PASS broker-connected API command triggered real LAN worker Shelly RPC signal and posted result\n");
process.stdout.write(`${JSON.stringify(result.evidence)}\n`);
process.stdout.write(`Agent: ${result.agentPath}\n`);
process.stdout.write(`LAN worker: ${result.workerPath}\n`);
process.stdout.write(`PHP image: ${result.phpImage}\n`);
if (options.keepTemp) {
process.stdout.write(`Temp dir: ${result.tempDir}\n`);
}
}
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch((error) => {
process.stderr.write(`FAIL ${error.message}\n`);
if (error.evidence) {
process.stderr.write(`Evidence: ${JSON.stringify(error.evidence)}\n`);
}
if (error.requests) {
process.stderr.write(`Requests: ${JSON.stringify(error.requests, null, 2)}\n`);
}
if (error.agentStdout) {
process.stderr.write(`agent stdout: ${error.agentStdout}\n`);
}
if (error.agentStderr) {
process.stderr.write(`agent stderr: ${error.agentStderr}\n`);
}
if (error.workerStdout) {
process.stderr.write(`worker stdout: ${error.workerStdout}\n`);
}
if (error.workerStderr) {
process.stderr.write(`worker stderr: ${error.workerStderr}\n`);
}
process.exit(1);
});
}
@@ -29,12 +29,12 @@ class edge_gateway_agent_artifact_locator
$candidateDirectories[] = self::normalizePath($configuredDirectory);
}
$candidateDirectories[] = self::routerArtifactDirectory($basePath);
foreach (self::edgeAgentBuildDirectories($basePath) as $directory) {
$candidateDirectories[] = $directory;
}
$candidateDirectories[] = self::routerArtifactDirectory($basePath);
$mountedArtifactDirectory = $mountedArtifactDirectory ?? self::mountedArtifactDirectory();
if ($mountedArtifactDirectory !== null) {
$candidateDirectories[] = self::normalizePath($mountedArtifactDirectory);
@@ -2434,17 +2434,17 @@ collect_install_diagnostics() {
capture_command_diagnostic "systemctl status" systemctl status --no-pager truckwash-edge-gateway-stack.service
fi
if command -v journalctl >/dev/null 2>&1; then
capture_command_diagnostic "journalctl" journalctl -u truckwash-edge-gateway-stack.service -n 60 --no-pager
capture_command_diagnostic "journalctl" journalctl -a -u truckwash-edge-gateway-stack.service -n 80 --no-pager
fi
if command -v docker >/dev/null 2>&1; then
capture_command_diagnostic "docker ps" docker ps --format '{{.Names}} {{.Status}}'
capture_command_diagnostic "docker ps -a" docker ps -a --format '{{.Names}} {{.Status}} {{.Ports}}'
if [ -f "$INSTALL_DIR/docker-compose.gateway.yml" ]; then
if docker compose version >/dev/null 2>&1; then
capture_command_diagnostic "docker compose ps" docker compose -f "$INSTALL_DIR/docker-compose.gateway.yml" ps
capture_command_diagnostic "docker compose logs" docker compose -f "$INSTALL_DIR/docker-compose.gateway.yml" logs --tail=80
capture_command_diagnostic "docker compose ps" docker compose -f "$INSTALL_DIR/docker-compose.gateway.yml" ps -a --no-trunc
capture_command_diagnostic "docker compose logs" docker compose -f "$INSTALL_DIR/docker-compose.gateway.yml" logs --no-color --no-log-prefix --tail=120
elif command -v docker-compose >/dev/null 2>&1; then
capture_command_diagnostic "docker-compose ps" docker-compose -f "$INSTALL_DIR/docker-compose.gateway.yml" ps
capture_command_diagnostic "docker-compose logs" docker-compose -f "$INSTALL_DIR/docker-compose.gateway.yml" logs --tail=80
capture_command_diagnostic "docker-compose ps" docker-compose -f "$INSTALL_DIR/docker-compose.gateway.yml" ps -a
capture_command_diagnostic "docker-compose logs" docker-compose -f "$INSTALL_DIR/docker-compose.gateway.yml" logs --no-color --tail=120
fi
fi
fi
@@ -2582,6 +2582,34 @@ verify_manifest_artifact() {
}
' "$manifest_path" "$artifact_name" "$artifact_path"
}
remove_docker_containers_matching_filter() {
local filter="$1"
local ids
ids="$(docker ps -aq --filter "$filter" 2>/dev/null || true)"
if [ -n "$ids" ]; then
printf '%s\n' "$ids" | xargs -r docker rm -f >/dev/null 2>&1 || true
fi
}
remove_edge_gateway_containers() {
local name
local project_name
for name in \
truckwash-edge-agent \
truckwash-lan-worker \
truckwash-auto-updater \
truckwash-redis \
truckwash-mariadb \
truckwash-minio; do
remove_docker_containers_matching_filter "name=${name}"
done
project_name="$(read_config_value "$CONFIG_PATH" composeProjectName 2>/dev/null || true)"
if [ -z "$project_name" ]; then
project_name="truckwash-edge-gateway"
fi
remove_docker_containers_matching_filter "label=com.docker.compose.project=${project_name}"
remove_docker_containers_matching_filter "label=com.docker.compose.project.working_dir=${INSTALL_DIR}"
}
cleanup_existing_installation() {
if [ "$INSTALL_DIR" != "/opt/truckwash-edge-agent" ]; then
echo "Refusing to remove unexpected install directory: $INSTALL_DIR" >&2
@@ -2602,13 +2630,7 @@ cleanup_existing_installation() {
fi
if command -v docker >/dev/null 2>&1; then
docker rm -f \
truckwash-edge-agent \
truckwash-lan-worker \
truckwash-auto-updater \
truckwash-redis \
truckwash-mariadb \
truckwash-minio >/dev/null 2>&1 || true
remove_edge_gateway_containers
fi
systemctl disable truckwash-edge-gateway-stack.service >/dev/null 2>&1 || true
@@ -1012,6 +1012,7 @@ final class TruckwashEdgeAgent
private const OUTBOX_REPLAY_FAILURE_COOLDOWN_SECONDS = 15;
private const MACHINE_SIGNAL_TIMEOUT_SECONDS = 3;
private const BROKER_MESSAGE_PUMP_LIMIT = 12;
private const BROKER_CONNECTED_COMMAND_POLL_INTERVAL_SECONDS = 1;
private const LOOP_STALE_AFTER_SECONDS = 30;
private const CONTROL_PLANE_SYNC_STALE_AFTER_SECONDS = 90;
private const MAX_RELAY_TOGGLE_AFTER_SECONDS = 5;
@@ -1032,6 +1033,7 @@ final class TruckwashEdgeAgent
private string $stagedUpdatePath;
private int $lastHeartbeatAt = 0;
private int $lastMachineSignalPollAt = 0;
private int $lastBrokerConnectedCommandPollAt = 0;
private int $lastMachineSignalMonitorRefreshAt = 0;
private int $lastOutboxFailureAt = 0;
private ?array $lastControlPlaneResponse = null;
@@ -1084,12 +1086,13 @@ final class TruckwashEdgeAgent
$this->pumpBrokerTransport();
continue;
}
$brokerConnected = $this->isBrokerConnected();
$processedManagementOperation = false;
if (!$this->isBrokerConnected()) {
if (!$brokerConnected) {
$processedManagementOperation = $this->processManagementOperation();
}
if (!$processedManagementOperation && !$this->isBrokerConnected()) {
$this->processCommandQueue();
if (!$processedManagementOperation && $this->shouldPollApiCommandQueue($brokerConnected)) {
$this->processCommandQueue($brokerConnected ? 0 : null);
}
$this->pumpBrokerTransport();
$this->flushOutbox();
@@ -1480,14 +1483,29 @@ final class TruckwashEdgeAgent
return null;
}
private function processCommandQueue(): void
private function shouldPollApiCommandQueue(bool $brokerConnected): bool
{
if (!$brokerConnected) {
return true;
}
$now = time();
if (($now - $this->lastBrokerConnectedCommandPollAt) < self::BROKER_CONNECTED_COMMAND_POLL_INTERVAL_SECONDS) {
return false;
}
$this->lastBrokerConnectedCommandPollAt = $now;
return true;
}
private function processCommandQueue(?int $waitSecondsOverride = null): void
{
$gatewayId = (int)$this->config->get('gatewayId');
if ($gatewayId <= 0) {
return;
}
$waitSeconds = (int)$this->config->get('operationPollTimeoutSeconds', 20);
$waitSeconds = $waitSecondsOverride ?? (int)$this->config->get('operationPollTimeoutSeconds', 20);
try {
$response = $this->http->post('/edge-agent/gateways/' . $gatewayId . '/commands/poll', [
'agent_token' => (string)$this->config->get('agentToken'),
@@ -34,14 +34,14 @@ print_compose_diagnostics() {
local compose_project_name
compose_project_name="$(config_value composeProjectName 'truckwash-edge-gateway')"
log "docker ps --format '{{.Names}} {{.Status}}'"
docker ps --format '{{.Names}} {{.Status}}' || true
log "docker ps -a --format '{{.Names}} {{.Status}} {{.Ports}}'"
docker ps -a --format '{{.Names}} {{.Status}} {{.Ports}}' || true
log "compose ps"
COMPOSE_PROJECT_NAME="$compose_project_name" compose_cmd -f "$COMPOSE_FILE" ps || true
log "compose ps -a --no-trunc"
COMPOSE_PROJECT_NAME="$compose_project_name" compose_cmd -f "$COMPOSE_FILE" ps -a --no-trunc || true
log "compose logs --tail=80"
COMPOSE_PROJECT_NAME="$compose_project_name" compose_cmd -f "$COMPOSE_FILE" logs --tail=80 || true
log "compose logs --no-color --no-log-prefix --tail=120"
COMPOSE_PROJECT_NAME="$compose_project_name" compose_cmd -f "$COMPOSE_FILE" logs --no-color --no-log-prefix --tail=120 || true
}
config_value() {
@@ -32,6 +32,23 @@ it('bounds stale outbox replay so it cannot monopolize the gateway loop', functi
->and($agentSource)->toContain(': self::OUTBOX_REPLAY_TIMEOUT_SECONDS;');
});
it('keeps draining API command jobs while broker transport is connected', function (): void {
$agentSource = (string)file_get_contents(app_path('resources/edge-gateway-agent/agent.php'));
$runOffset = strpos($agentSource, 'public function run(): void');
expect($runOffset)->not->toBeFalse();
$loopSource = substr($agentSource, (int)$runOffset, 1900);
expect($agentSource)->toContain('private const BROKER_CONNECTED_COMMAND_POLL_INTERVAL_SECONDS = 1;')
->and($agentSource)->toContain('private int $lastBrokerConnectedCommandPollAt = 0;')
->and($agentSource)->toContain('private function shouldPollApiCommandQueue(bool $brokerConnected): bool')
->and($agentSource)->toContain('private function processCommandQueue(?int $waitSecondsOverride = null): void')
->and($loopSource)->toContain('$brokerConnected = $this->isBrokerConnected();')
->and($loopSource)->toContain('if (!$brokerConnected) {')
->and($loopSource)->toContain('$processedManagementOperation = $this->processManagementOperation();')
->and($loopSource)->toContain('$this->processCommandQueue($brokerConnected ? 0 : null);');
});
it('uses a short control-plane timeout for self-serve machine signals before queueing', function (): void {
$agentSource = (string)file_get_contents(app_path('resources/edge-gateway-agent/agent.php'));
@@ -67,14 +67,11 @@ it('resolves edge-agent artifacts from a supported runtime layout', function ():
$path = edge_gateway_agent_artifact_locator::resolve('agent.php', app_path());
$normalizedPath = str_replace('\\', '/', $path);
expect(
str_ends_with($normalizedPath, '/edge-agent/build/install/agent.php')
|| str_ends_with($normalizedPath, '/resources/edge-gateway-agent/agent.php')
)->toBeTrue();
expect($normalizedPath)->toEndWith('/resources/edge-gateway-agent/agent.php');
expect(is_file($path))->toBeTrue();
});
it('prioritizes generated edge-agent build output before router, mounted, and baked-in artifact directories', function (): void {
it('prioritizes router-owned artifacts before implicit generated build output', function (): void {
with_edge_gateway_artifact_env(['EDGE_AGENT_ARTIFACT_DIR' => null], function (): void {
$candidatePaths = array_map(
static fn(string $path): string => str_replace('\\', '/', $path),
@@ -87,11 +84,11 @@ it('prioritizes generated edge-agent build output before router, mounted, and ba
);
expect(array_slice($candidatePaths, 0, 5))->toBe([
'/var/www/html/resources/edge-gateway-agent/agent.php',
'/edge-agent/build/install/agent.php',
'/services/edge-agent/build/install/agent.php',
'/var/edge-agent/build/install/agent.php',
'/var/www/edge-agent/build/install/agent.php',
'/var/www/html/resources/edge-gateway-agent/agent.php',
]);
expect($candidatePaths)->toContain('/services/edge-agent/php-agent/agent.php');
expect($candidatePaths)->toContain('/opt/truckwash-edge-agent-artifacts/agent.php');
@@ -114,11 +111,11 @@ it('prioritizes EDGE_AGENT_ARTIFACT_DIR before generated edge-agent build output
expect(array_slice($candidatePaths, 0, 4))->toBe([
'/tmp/custom-edge-artifacts/agent.php',
'/var/www/html/resources/edge-gateway-agent/agent.php',
'/edge-agent/build/install/agent.php',
'/services/edge-agent/build/install/agent.php',
'/var/edge-agent/build/install/agent.php',
]);
expect($candidatePaths)->toContain('/var/www/html/resources/edge-gateway-agent/agent.php');
expect($candidatePaths)->toContain('/var/edge-agent/build/install/agent.php');
});
});
@@ -51,8 +51,11 @@ it('builds the installer around the compose stack artifacts and management polli
expect($managerSource)->toContain('systemctl stop truckwash-edge-agent.service');
expect($managerSource)->toContain('TRUCKWASH_INSTALL_DIR="$INSTALL_DIR" "$INSTALL_DIR/gateway-launcher.sh" down >/dev/null 2>&1 || true');
expect($managerSource)->toContain('docker rm -f');
expect($managerSource)->toContain('remove_edge_gateway_containers()');
expect($managerSource)->toContain('remove_docker_containers_matching_filter "name=${name}"');
expect($managerSource)->toContain('remove_docker_containers_matching_filter "label=com.docker.compose.project=${project_name}"');
expect($managerSource)->toContain('remove_docker_containers_matching_filter "label=com.docker.compose.project.working_dir=${INSTALL_DIR}"');
expect($managerSource)->toContain('truckwash-minio');
expect($managerSource)->toContain('truckwash-minio >/dev/null 2>&1 || true');
expect($managerSource)->toContain('rm -f "$STACK_SERVICE_PATH" "$LEGACY_SERVICE_PATH"');
expect($managerSource)->toContain('rm -rf "$INSTALL_DIR"');
expect($managerSource)->toContain('systemctl reset-failed truckwash-edge-agent.service >/dev/null 2>&1 || true');
@@ -76,7 +79,9 @@ it('builds the installer around the compose stack artifacts and management polli
expect($managerSource)->toContain('run_step "Waiting for gateway claim" wait_for_gateway_claim "$CONFIG_PATH" "$HEARTBEAT_MARKER_PATH" "$INSTALL_STARTED_AT" 180');
expect($managerSource)->not->toContain('run_step "Waiting for post-reinstall heartbeat"');
expect($managerSource)->not->toContain('Gateway reconnected using preserved credentials.');
expect($managerSource)->toContain('journalctl -u truckwash-edge-gateway-stack.service -n 60 --no-pager || true');
expect($managerSource)->toContain('journalctl -a -u truckwash-edge-gateway-stack.service -n 80 --no-pager');
expect($managerSource)->toContain("docker ps -a --format '{{.Names}} {{.Status}} {{.Ports}}'");
expect($managerSource)->toContain('logs --no-color --no-log-prefix --tail=120');
expect($managerSource)->not->toContain('agent.mjs');
expect($managerSource)->toContain("'brokerUrl' => \$this->buildBrokerPublicUrl()");
expect($installServiceSource)->toContain('normalizeLineEndings($this->manager()->buildInstallScript($plainToken))');
@@ -102,8 +107,9 @@ it('builds the installer around the compose stack artifacts and management polli
expect($launcherSource)->toContain('window="$(staged_update_value update_window \'\')"');
expect($launcherSource)->toContain('Staged update already applied; nothing to reconcile');
expect($launcherSource)->toContain('AUTO_UPDATER_BASE_IMAGE="$auto_updater_base_image"');
expect($launcherSource)->toContain('COMPOSE_PROJECT_NAME="$compose_project_name" compose_cmd -f "$COMPOSE_FILE" ps || true');
expect($launcherSource)->toContain('COMPOSE_PROJECT_NAME="$compose_project_name" compose_cmd -f "$COMPOSE_FILE" logs --tail=80 || true');
expect($launcherSource)->toContain("docker ps -a --format '{{.Names}} {{.Status}} {{.Ports}}'");
expect($launcherSource)->toContain('COMPOSE_PROJECT_NAME="$compose_project_name" compose_cmd -f "$COMPOSE_FILE" ps -a --no-trunc || true');
expect($launcherSource)->toContain('COMPOSE_PROJECT_NAME="$compose_project_name" compose_cmd -f "$COMPOSE_FILE" logs --no-color --no-log-prefix --tail=120 || true');
expect($launcherSource)->toContain('log "Compose rollout failed during build/startup"');
expect($launcherSource)->toContain('write_rollback_status "FAILED" "compose_up_failed" "$installed_version"');
expect($launcherSource)->toContain('write_rollback_status "FAILED" "rollback_apply_failed" "$installed_version"');