copenhagentruckwash/api
1
0
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

Add delivery metadata support, preferred channels, and enhanced agent validation

Browse Source 
This commit introduces delivery metadata tracking for gateway commands, updates, and shells. Adds preferred delivery channel handling, refined validation for edge agents, improved relay management logic, and broker presence reporting. Includes schema changes, enhanced shell handling, and test coverage.
This commit is contained in:
Jeppe Bundgaard
2026-04-16 13:44:28 +02:00
parent d4d162d38a
commit d30a006457
16 changed files with 2188 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/edge-agent/dist/agent.mjs
Vendored
+397 -37
View File
@@ -16,6 +16,7 @@ const DEFAULT_EDGE_AGENT_SERVICE_NAME = "truckwash-edge-agent.service";
const DEFAULT_UPDATE_VERIFICATION_TIMEOUT_SECONDS = 45;
const DEFAULT_UPDATE_VERIFY_INTERVAL_MS = 500;
const DEFAULT_UPDATE_RESTART_GRACE_MS = 150;
const DEFAULT_BROKER_RECONNECT_DELAY_MS = 1500;
const UPDATE_VERIFY_COMMAND = "post-update-verify";
const execFile = promisify(execFileCallback);
@@ -130,16 +131,69 @@ function createUpdateErrorMessage(error, fallback = "Edge agent update failed")
return error instanceof Error ? error.message : String(error || fallback);
}
function buildTransportHeartbeatState() {
function buildTransportHeartbeatState(brokerState = {}) {
const brokerConnected = Boolean(brokerState.connected);
return {
status: "ONLINE",
metadata: {
command_transport: "API_POLLING",
command_transport: brokerConnected ? "BROKER_FAST_PATH" : "API_POLLING",
shell_transport: "API_POLLING",
broker_connected: brokerConnected,
broker_url: brokerState.url || null,
broker_last_error: brokerState.lastError || null,
broker_last_connected_at: brokerState.lastConnectedAt || null,
broker_last_disconnected_at: brokerState.lastDisconnectedAt || null,
broker_disconnect_reason: brokerState.disconnectReason || null,
},
};
}
function normalizeBrokerBaseUrl(value) {
const trimmed = String(value || "").trim().replace(/\/+$/, "");
if (trimmed === "") {
return null;
}
if (trimmed.startsWith("ws://") || trimmed.startsWith("wss://")) {
return trimmed;
}
if (trimmed.startsWith("https://")) {
return `wss://${trimmed.slice("https://".length)}`;
}
if (trimmed.startsWith("http://")) {
return `ws://${trimmed.slice("http://".length)}`;
}
return `ws://${trimmed}`;
}
function buildBrokerSocketUrl(brokerUrl, gatewayId, token) {
const baseUrl = normalizeBrokerBaseUrl(brokerUrl);
if (!baseUrl || !gatewayId || !token) {
return null;
}
const search = new URLSearchParams({
gatewayId: String(gatewayId),
token: String(token),
});
return `${baseUrl}/ws/agent?${search.toString()}`;
}
async function readSocketMessageText(data) {
if (typeof data === "string") {
return data;
}
if (data instanceof ArrayBuffer) {
return Buffer.from(data).toString("utf8");
}
if (ArrayBuffer.isView(data)) {
return Buffer.from(data.buffer, data.byteOffset, data.byteLength).toString("utf8");
}
if (typeof Blob !== "undefined" && data instanceof Blob) {
return data.text();
}
return String(data || "");
}
export async function loadConfig(configPath) {
const raw = await fs.readFile(configPath, "utf8");
return JSON.parse(raw);
@@ -160,7 +214,8 @@ export function buildStatusReport(configPath, config) {
state: claimed ? "CLAIMED" : "PENDING_CLAIM",
gatewayId: config.gatewayId ?? null,
apiUrl: config.apiUrl ?? null,
transport: "API_POLLING",
brokerUrl: config.brokerUrl ?? null,
transport: config.brokerUrl ? "HYBRID" : "API_POLLING",
hostname: config.hostname || os.hostname(),
releaseChannel: config.releaseChannel || "stable",
installedVersion: config.installedVersion || DEFAULT_VERSION,
@@ -364,6 +419,9 @@ export async function claimIfNeeded(config, configPath, fetchImpl = fetch) {
agentToken: claimed.agent_token,
releaseChannel: claimed.release_channel || config.releaseChannel || "stable",
};
if (claimed.broker_url || config.brokerUrl) {
nextConfig.brokerUrl = claimed.broker_url || config.brokerUrl;
}
await saveConfig(configPath, nextConfig);
return nextConfig;
}
@@ -691,6 +749,69 @@ function buildUpdateRestartPlan(payload, configPath, config = {}) {
};
}
function buildUninstallPlan(payload = {}, configPath, config = {}) {
const paths = resolveUpdatePaths(configPath, config);
return {
configPath: paths.configPath,
serviceName: String(payload.serviceName || config.serviceName || paths.serviceName || DEFAULT_EDGE_AGENT_SERVICE_NAME),
restartMode: String(config.restartMode || (process.platform === "win32" ? "spawn" : "systemd")),
cleanupDelayMs: DEFAULT_UPDATE_RESTART_GRACE_MS,
};
}
function escapeShellArgument(value) {
return `'${String(value || "").replace(/'/g, `'\\''`)}'`;
}
async function persistUninstallState(configPath, config, { liveConfig = null } = {}) {
const currentConfig = cloneJson(config || (await loadConfig(configPath)));
const nextConfig = {
...currentConfig,
gatewayId: null,
agentToken: null,
installToken: null,
brokerUrl: null,
pendingUpdate: null,
lastUninstall: {
state: "SCHEDULED",
completedAt: formatUpdateTimestamp(),
},
};
await saveConfig(configPath, nextConfig);
if (liveConfig) {
replaceObjectContents(liveConfig, cloneJson(nextConfig));
}
return nextConfig;
}
async function scheduleAgentUninstall(uninstallPlan, {
spawnImpl = spawnCallback,
waitImpl = wait,
exitProcessImpl = (code) => process.exit(code),
} = {}) {
if (uninstallPlan.restartMode === "systemd" && process.platform !== "win32") {
const serviceName = escapeShellArgument(uninstallPlan.serviceName || DEFAULT_EDGE_AGENT_SERVICE_NAME);
const uninstallProcess = spawnImpl(
"/bin/sh",
[
"-lc",
`sleep 1; systemctl disable --now ${serviceName} >/dev/null 2>&1 || true; systemctl reset-failed ${serviceName} >/dev/null 2>&1 || true`,
],
{
detached: true,
stdio: "ignore",
windowsHide: true,
}
);
uninstallProcess.unref?.();
}
await waitImpl(uninstallPlan.cleanupDelayMs ?? DEFAULT_UPDATE_RESTART_GRACE_MS);
exitProcessImpl(0);
}
async function startDetachedUpdateVerifier(configPath, config, {
spawnImpl = spawnCallback,
} = {}) {
@@ -895,6 +1016,29 @@ export async function runUpdate(payload, fetchImpl = fetch, deps = {}) {
}
}
export async function runUninstall(payload = {}, deps = {}) {
const configPath = deps.configPath;
if (!configPath) {
throw new Error("Missing config path for edge agent uninstall");
}
const liveConfig = deps.liveConfig && typeof deps.liveConfig === "object" ? deps.liveConfig : null;
const currentConfig = cloneJson(deps.config || liveConfig || (await loadConfig(configPath)));
return {
__agentCommandEnvelope: true,
payload: {
uninstall_scheduled: true,
scheduled_at: formatUpdateTimestamp(),
service_name: String(payload.serviceName || currentConfig.serviceName || DEFAULT_EDGE_AGENT_SERVICE_NAME),
},
followUp: {
type: "UNINSTALL_AGENT",
uninstallPlan: buildUninstallPlan(payload, configPath, currentConfig),
},
};
}
function defaultShellCommand() {
if (process.platform === "win32") {
return { command: process.env.ComSpec || "cmd.exe", args: [] };
@@ -1038,6 +1182,8 @@ export async function handleAgentCommand(command, deps = {}) {
return await setRelayState(command.payload || {}, fetchImpl);
case "RUN_UPDATE":
return await runUpdate(command.payload || {}, fetchImpl, deps);
case "UNINSTALL_AGENT":
return await runUninstall(command.payload || {}, deps);
case "RESTART_AGENT":
return { restarted: true };
case "REBOOT_HOST":
@@ -1189,49 +1335,95 @@ export async function submitShellSessionEvents(config, sessionId, events = [], f
);
}
export async function processPolledCommand(config, command, fetchImpl = fetch) {
const jobId = command?.id;
async function executeAgentCommandEnvelope(config, command, fetchImpl = fetch, deps = {}) {
const commandType = command?.commandType || command?.command_type;
const payload = command?.payload || {};
if (!jobId || !commandType) {
if (!commandType) {
return null;
}
let followUp = null;
try {
const result = await handleAgentCommand(
{ commandType, payload },
{
fetchImpl,
config,
configPath: payload.configPath || config.configPath || null,
liveConfig: config,
}
);
const responsePayload =
result && result.__agentCommandEnvelope === true
? (followUp = result.followUp || null, result.payload || {})
: result;
await submitCommandJobResult(config, jobId, {
ok: true,
payload: responsePayload,
}, fetchImpl);
const result = await handleAgentCommand(
{ commandType, payload },
{
...deps,
fetchImpl,
config,
configPath: payload.configPath || config.configPath || null,
liveConfig: config,
}
);
if (commandType === "RUN_UPDATE" && followUp?.type === "RUN_UPDATE") {
try {
await startDetachedUpdateVerifier(config.configPath || payload.configPath || null, config);
await restartAgentAfterUpdate(followUp.restartPlan, {});
} catch (followUpError) {
await rollbackPendingUpdate(config.configPath || payload.configPath || null, {
config,
liveConfig: config,
reason: createUpdateErrorMessage(followUpError, "Edge agent restart failed after update"),
}).catch(() => {});
}
return {
commandType,
payload,
followUp: result && result.__agentCommandEnvelope === true ? (followUp = result.followUp || null, followUp) : null,
responsePayload: result && result.__agentCommandEnvelope === true ? result.payload || {} : result,
};
}
async function runAgentCommandFollowUp(
config,
execution,
followUpErrorMessage = "Edge agent restart failed after update",
deps = {}
) {
if (!execution || !execution.followUp?.type) {
return;
}
if (execution.commandType === "RUN_UPDATE" && execution.followUp.type === "RUN_UPDATE") {
try {
await startDetachedUpdateVerifier(config.configPath || execution.payload?.configPath || null, config, {
spawnImpl: deps.spawnImpl,
});
await restartAgentAfterUpdate(execution.followUp.restartPlan, {
spawnImpl: deps.spawnImpl,
waitImpl: deps.waitImpl,
exitProcessImpl: deps.exitProcessImpl,
});
} catch (followUpError) {
await rollbackPendingUpdate(config.configPath || execution.payload?.configPath || null, {
config,
liveConfig: config,
reason: createUpdateErrorMessage(followUpError, followUpErrorMessage),
}).catch(() => {});
}
return;
}
if (execution.commandType === "UNINSTALL_AGENT" && execution.followUp.type === "UNINSTALL_AGENT") {
await persistUninstallState(config.configPath || execution.followUp.uninstallPlan?.configPath, config, {
liveConfig: config,
});
await scheduleAgentUninstall(execution.followUp.uninstallPlan, {
spawnImpl: deps.spawnImpl,
waitImpl: deps.waitImpl,
exitProcessImpl: deps.exitProcessImpl,
});
}
}
export async function processPolledCommand(config, command, fetchImpl = fetch, deps = {}) {
const jobId = command?.id;
if (!jobId) {
return null;
}
try {
const execution = await executeAgentCommandEnvelope(config, command, fetchImpl, deps);
if (!execution) {
return null;
}
return { ok: true, payload: responsePayload };
await submitCommandJobResult(config, jobId, {
ok: true,
payload: execution.responsePayload,
}, fetchImpl);
await runAgentCommandFollowUp(config, execution, "Edge agent restart failed after update", deps);
return { ok: true, payload: execution.responsePayload };
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
await submitCommandJobResult(config, jobId, {
@@ -1395,6 +1587,162 @@ export async function processPolledShellAction(config, action, shell, fetchImpl
}
}
function createBrokerBridge({
config,
shell,
fetchImpl = fetch,
reconnectDelayMs = DEFAULT_BROKER_RECONNECT_DELAY_MS,
} = {}) {
const state = {
url: config?.brokerUrl || null,
connected: false,
lastError: null,
disconnectReason: null,
lastConnectedAt: null,
lastDisconnectedAt: null,
};
let stopped = false;
let reconnectTimer = null;
let socket = null;
const clearReconnectTimer = () => {
if (reconnectTimer) {
clearTimeout(reconnectTimer);
reconnectTimer = null;
}
};
const send = (message) => {
if (!socket || socket.readyState !== WebSocket.OPEN) {
return false;
}
socket.send(JSON.stringify(message));
return true;
};
const scheduleReconnect = () => {
if (stopped || reconnectTimer || !state.url) {
return;
}
reconnectTimer = setTimeout(() => {
reconnectTimer = null;
connect();
}, Math.max(250, reconnectDelayMs));
};
const connect = () => {
if (stopped) {
return;
}
const socketUrl = buildBrokerSocketUrl(state.url, config?.gatewayId, config?.agentToken);
if (!socketUrl) {
return;
}
try {
socket = new WebSocket(socketUrl);
} catch (error) {
state.connected = false;
state.lastError = error instanceof Error ? error.message : String(error);
state.lastDisconnectedAt = formatUpdateTimestamp();
scheduleReconnect();
return;
}
socket.onopen = () => {
state.connected = true;
state.lastError = null;
state.disconnectReason = null;
state.lastConnectedAt = formatUpdateTimestamp();
};
socket.onmessage = async (event) => {
try {
const raw = await readSocketMessageText(event.data);
const message = JSON.parse(raw);
if (message.type === "COMMAND") {
try {
const execution = await executeAgentCommandEnvelope(config, message, fetchImpl);
if (!execution) {
return;
}
send({
type: "COMMAND_RESULT",
commandId: message.commandId,
ok: true,
payload: execution.responsePayload,
});
await runAgentCommandFollowUp(config, execution);
} catch (error) {
send({
type: "COMMAND_RESULT",
commandId: message.commandId,
ok: false,
error: error instanceof Error ? error.message : String(error),
});
}
return;
}
if (message.type === "OPEN_ROOT_SHELL") {
await shell.open(message.payload || {});
return;
}
if (message.type === "SHELL_INPUT") {
shell.input(message.payload || {});
return;
}
if (message.type === "RESIZE_ROOT_SHELL") {
shell.resize(message.payload || {});
return;
}
if (message.type === "CLOSE_ROOT_SHELL") {
shell.close(message.payload || {});
}
} catch {
// Ignore malformed or unsupported broker messages so polling remains authoritative.
}
};
socket.onerror = () => {
state.lastError = "Broker connection failed";
};
socket.onclose = (event) => {
socket = null;
state.connected = false;
state.disconnectReason = event.reason || "broker_disconnected";
state.lastDisconnectedAt = formatUpdateTimestamp();
if (!stopped) {
scheduleReconnect();
}
};
};
return {
state,
start() {
connect();
},
stop() {
stopped = true;
clearReconnectTimer();
if (socket && socket.readyState <= WebSocket.OPEN) {
socket.close();
}
socket = null;
state.connected = false;
},
send,
};
}
export async function startAgent({
configPath,
fetchImpl = fetch,
@@ -1418,17 +1766,27 @@ export async function startAgent({
const shellActionPollTimeoutSeconds = Number(config.shellActionPollTimeoutSeconds || 20);
const commandPollRetryDelayMs = Number(config.commandPollRetryDelayMs || 1000);
const shellActionPollRetryDelayMs = Number(config.shellActionPollRetryDelayMs || 1000);
const brokerReconnectDelayMs = Number(config.brokerReconnectDelayMs || DEFAULT_BROKER_RECONNECT_DELAY_MS);
let stopped = false;
let cpuSnapshot = null;
let lastHeartbeatLatencyMs = null;
const shellEventPublisher = createShellEventPublisher(config, fetchImpl);
let brokerBridge = null;
const shell = createShellBridgeImpl((message) => {
shellEventPublisher.publish(message);
brokerBridge?.send(message);
});
brokerBridge = createBrokerBridge({
config,
shell,
fetchImpl,
reconnectDelayMs: brokerReconnectDelayMs,
});
brokerBridge.start();
const sendTransportHeartbeat = async (extra = {}) => {
const transportState = buildTransportHeartbeatState();
const transportState = buildTransportHeartbeatState(brokerBridge?.state || { url: config.brokerUrl || null });
const collectedMetrics = await collectMetricsImpl({
previousCpuSnapshot: cpuSnapshot,
latencyMs: lastHeartbeatLatencyMs,
@@ -1518,12 +1876,14 @@ export async function startAgent({
const stop = async () => {
stopped = true;
clearInterval(timer);
brokerBridge?.stop();
shell.dispose();
await shellEventPublisher.drain().catch(() => {});
await Promise.allSettled([commandPollPromise, shellActionPollPromise]);
};
return {
brokerBridge,
commandPollPromise,
shellActionPollPromise,
timer,