Add edge agent update handling and verification mechanisms

Refactored the edge agent to include robust update handling, verification, and rollback procedures. Enhanced test coverage for critical update flows and introduced support for background edge gateway refresh without disrupting current user actions.
This commit is contained in:
Jeppe Bundgaard
2026-04-15 12:07:32 +02:00
parent 56685d7bf3
commit b6454e5d36
+579 -19
View File
@@ -1,5 +1,5 @@
import { createHash } from "node:crypto";
import { execFile as execFileCallback } from "node:child_process";
import { execFile as execFileCallback, spawn as spawnCallback } from "node:child_process";
import { promises as fs } from "node:fs";
import os from "node:os";
import path from "node:path";
@@ -12,8 +12,124 @@ const DEFAULT_SHELL_COLS = 120;
const DEFAULT_SHELL_ROWS = 32;
const DEFAULT_CPU_SAMPLE_DELAY_MS = 150;
const DEFAULT_SHELL_EVENT_FLUSH_DELAY_MS = 40;
const DEFAULT_EDGE_AGENT_SERVICE_NAME = "truckwash-edge-agent.service";
const DEFAULT_UPDATE_VERIFICATION_TIMEOUT_SECONDS = 45;
const DEFAULT_UPDATE_VERIFY_INTERVAL_MS = 500;
const DEFAULT_UPDATE_RESTART_GRACE_MS = 150;
const UPDATE_VERIFY_COMMAND = "post-update-verify";
const execFile = promisify(execFileCallback);
function cloneJson(value) {
return value === undefined ? undefined : JSON.parse(JSON.stringify(value));
}
function replaceObjectContents(target, source) {
if (!target || typeof target !== "object" || Array.isArray(target) || !source || typeof source !== "object") {
return source;
}
for (const key of Object.keys(target)) {
delete target[key];
}
for (const [key, value] of Object.entries(source)) {
target[key] = value;
}
return target;
}
function formatUpdateTimestamp(date = new Date()) {
return date.toISOString();
}
function resolveUpdateVerificationTimeoutSeconds(config = {}, pendingUpdate = {}) {
const configuredTimeout = Number(
pendingUpdate.verificationTimeoutSeconds ??
config.updateVerificationTimeoutSeconds ??
DEFAULT_UPDATE_VERIFICATION_TIMEOUT_SECONDS
);
return Number.isFinite(configuredTimeout) && configuredTimeout > 0
? Math.max(5, Math.round(configuredTimeout))
: DEFAULT_UPDATE_VERIFICATION_TIMEOUT_SECONDS;
}
function resolveUpdatePaths(configPath, config = {}) {
const installDir = path.resolve(String(config.installDir || path.dirname(configPath)));
return {
installDir,
configPath: path.resolve(configPath),
agentPath: path.resolve(String(config.agentPath || path.join(installDir, "agent.mjs"))),
packagePath: path.resolve(String(config.packagePath || path.join(installDir, "package.json"))),
updatesDir: path.resolve(String(config.updatesDir || path.join(installDir, ".updates"))),
serviceName: String(config.serviceName || DEFAULT_EDGE_AGENT_SERVICE_NAME),
};
}
function sanitizeUpdateSegment(value, fallback = "update") {
const normalized = String(value || "")
.trim()
.replace(/[^a-zA-Z0-9._-]+/g, "-")
.replace(/^-+|-+$/g, "");
return normalized === "" ? fallback : normalized;
}
function buildUpdateMetadata(config = {}) {
const lastUpdate = config.lastUpdate;
if (!lastUpdate || typeof lastUpdate !== "object" || Array.isArray(lastUpdate)) {
return null;
}
return {
state: String(lastUpdate.state || "UNKNOWN"),
target_version: lastUpdate.targetVersion ?? null,
previous_version: lastUpdate.previousVersion ?? null,
restored_version: lastUpdate.restoredVersion ?? null,
error: lastUpdate.error ?? null,
completed_at: lastUpdate.completedAt ?? null,
};
}
async function pathExists(pathToCheck) {
try {
await fs.access(pathToCheck);
return true;
} catch {
return false;
}
}
async function ensureDirectory(pathToEnsure) {
await fs.mkdir(pathToEnsure, { recursive: true });
}
async function writeBuffer(pathToWrite, buffer) {
await fs.writeFile(pathToWrite, buffer);
}
async function backupFileIfPresent(sourcePath, destinationPath) {
if (!(await pathExists(sourcePath))) {
return false;
}
await ensureDirectory(path.dirname(destinationPath));
await fs.copyFile(sourcePath, destinationPath);
return true;
}
async function restoreFileIfPresent(sourcePath, destinationPath) {
if (!(await pathExists(sourcePath))) {
return false;
}
await ensureDirectory(path.dirname(destinationPath));
await fs.copyFile(sourcePath, destinationPath);
return true;
}
function createUpdateErrorMessage(error, fallback = "Edge agent update failed") {
return error instanceof Error ? error.message : String(error || fallback);
}
function buildTransportHeartbeatState() {
return {
status: "ONLINE",
@@ -354,7 +470,344 @@ export async function setRelayState(payload, fetchImpl = fetch) {
}
}
export async function runUpdate(payload, fetchImpl = fetch) {
async function fetchArtifactBuffer(url, expectedSha256, label, fetchImpl = fetch) {
if (!url) {
return null;
}
const response = await fetchImpl(url);
if (!response.ok) {
throw new Error(`${label} download failed: HTTP ${response.status}`);
}
const buffer = Buffer.from(await response.arrayBuffer());
const sha256 = createHash("sha256").update(buffer).digest("hex");
if (expectedSha256 && String(expectedSha256).toLowerCase() !== sha256.toLowerCase()) {
throw new Error(`${label} checksum mismatch`);
}
return {
buffer,
sha256,
bytes: buffer.length,
};
}
async function installAgentDependencies(installDir, execFileImpl = execFile) {
await execFileImpl("npm", ["install", "--omit=dev"], {
cwd: installDir,
encoding: "utf8",
windowsHide: true,
});
}
async function runAgentStatusPreflight(agentPath, configPath, execFileImpl = execFile) {
await execFileImpl(process.execPath, [agentPath, "status", "--config", configPath], {
encoding: "utf8",
windowsHide: true,
});
}
function buildPendingUpdateState(payload, config, paths, backupDir) {
return {
targetVersion: String(payload.targetVersion || payload.target_version || config.targetVersion || config.installedVersion || DEFAULT_VERSION),
previousVersion: String(config.installedVersion || DEFAULT_VERSION),
releaseChannel: String(payload.releaseChannel || payload.release_channel || config.releaseChannel || "stable"),
requestedAt: formatUpdateTimestamp(),
backupDir,
installDir: paths.installDir,
serviceName: paths.serviceName,
restartMode: String(payload.restartMode || config.restartMode || (process.platform === "win32" ? "spawn" : "systemd")),
verificationTimeoutSeconds: resolveUpdateVerificationTimeoutSeconds(config),
};
}
function buildPreparedUpdatePayload({
pendingUpdate,
agentArtifact,
packageArtifact,
paths,
}) {
return {
updated: true,
verification_pending: true,
target_version: pendingUpdate.targetVersion,
previous_version: pendingUpdate.previousVersion,
release_channel: pendingUpdate.releaseChannel,
restart_mode: pendingUpdate.restartMode,
verification_timeout_seconds: pendingUpdate.verificationTimeoutSeconds,
artifact_sha256: agentArtifact?.sha256 ?? null,
artifact_bytes: agentArtifact?.bytes ?? null,
package_sha256: packageArtifact?.sha256 ?? null,
package_bytes: packageArtifact?.bytes ?? null,
install_dir: paths.installDir,
};
}
async function restorePreparedUpdate(paths, pendingUpdate, originalConfig, {
execFileImpl = execFile,
liveConfig = null,
rollbackMessage = "Edge agent update failed",
} = {}) {
const backupDir = pendingUpdate?.backupDir;
if (!backupDir) {
if (originalConfig && pendingUpdate !== null) {
await saveConfig(paths.configPath, originalConfig);
if (liveConfig) {
replaceObjectContents(liveConfig, cloneJson(originalConfig));
}
}
return {
rolledBack: false,
restoredVersion: originalConfig?.installedVersion || null,
};
}
await restoreFileIfPresent(path.join(backupDir, "agent.mjs"), paths.agentPath);
await restoreFileIfPresent(path.join(backupDir, "package.json"), paths.packagePath);
try {
await installAgentDependencies(paths.installDir, execFileImpl);
} catch {
// Prefer preserving the restored files and rollback metadata over surfacing a secondary npm failure here.
}
const nextConfig = {
...(cloneJson(originalConfig) || {}),
targetVersion: pendingUpdate.previousVersion || originalConfig?.targetVersion || originalConfig?.installedVersion || DEFAULT_VERSION,
installedVersion: pendingUpdate.previousVersion || originalConfig?.installedVersion || DEFAULT_VERSION,
pendingUpdate: null,
lastUpdate: {
state: "ROLLED_BACK",
targetVersion: pendingUpdate.targetVersion ?? null,
previousVersion: pendingUpdate.previousVersion ?? null,
restoredVersion: pendingUpdate.previousVersion ?? null,
error: rollbackMessage,
completedAt: formatUpdateTimestamp(),
},
};
await saveConfig(paths.configPath, nextConfig);
if (liveConfig) {
replaceObjectContents(liveConfig, cloneJson(nextConfig));
}
return {
rolledBack: true,
restoredVersion: nextConfig.installedVersion || null,
};
}
export async function rollbackPendingUpdate(configPath, {
config = null,
execFileImpl = execFile,
liveConfig = null,
reason = "Edge agent update verification failed",
} = {}) {
const currentConfig = cloneJson(config || (await loadConfig(configPath)));
const pendingUpdate = currentConfig.pendingUpdate;
if (!pendingUpdate || typeof pendingUpdate !== "object") {
return {
rolledBack: false,
restoredVersion: currentConfig.installedVersion || null,
};
}
const paths = resolveUpdatePaths(configPath, currentConfig);
const originalConfig = {
...currentConfig,
pendingUpdate: null,
lastUpdate: currentConfig.lastUpdate ?? null,
};
originalConfig.installedVersion = pendingUpdate.previousVersion || originalConfig.installedVersion || DEFAULT_VERSION;
originalConfig.targetVersion = pendingUpdate.previousVersion || originalConfig.targetVersion || originalConfig.installedVersion;
return restorePreparedUpdate(paths, pendingUpdate, originalConfig, {
execFileImpl,
liveConfig,
rollbackMessage: reason,
});
}
export async function finalizePendingUpdateOnStartup(config, configPath, {
liveConfig = null,
} = {}) {
const pendingUpdate = config?.pendingUpdate;
if (!pendingUpdate || typeof pendingUpdate !== "object") {
return config;
}
const nextConfig = {
...cloneJson(config),
installedVersion:
pendingUpdate.targetVersion ||
config.targetVersion ||
config.installedVersion ||
DEFAULT_VERSION,
targetVersion:
pendingUpdate.targetVersion ||
config.targetVersion ||
config.installedVersion ||
DEFAULT_VERSION,
releaseChannel: pendingUpdate.releaseChannel || config.releaseChannel || "stable",
pendingUpdate: null,
lastUpdate: {
state: "COMPLETED",
targetVersion: pendingUpdate.targetVersion ?? null,
previousVersion: pendingUpdate.previousVersion ?? null,
restoredVersion: null,
error: null,
completedAt: formatUpdateTimestamp(),
},
};
await saveConfig(configPath, nextConfig);
if (liveConfig) {
replaceObjectContents(liveConfig, cloneJson(nextConfig));
return liveConfig;
}
return nextConfig;
}
function buildUpdateRestartPlan(payload, configPath, config = {}) {
const paths = resolveUpdatePaths(configPath, config);
return {
configPath: paths.configPath,
agentPath: paths.agentPath,
serviceName: String(
payload.serviceName ||
payload.service_name ||
config.serviceName ||
DEFAULT_EDGE_AGENT_SERVICE_NAME
),
restartMode: String(
payload.restartMode ||
payload.restart_mode ||
config.restartMode ||
(process.platform === "win32" ? "spawn" : "systemd")
),
restartGraceMs: DEFAULT_UPDATE_RESTART_GRACE_MS,
};
}
async function startDetachedUpdateVerifier(configPath, config, {
spawnImpl = spawnCallback,
} = {}) {
const paths = resolveUpdatePaths(configPath, config);
const verifier = spawnImpl(
process.execPath,
[paths.agentPath, UPDATE_VERIFY_COMMAND, "--config", paths.configPath],
{
detached: true,
stdio: "ignore",
windowsHide: true,
}
);
verifier.unref?.();
return true;
}
async function restartAgentAfterUpdate(restartPlan, {
spawnImpl = spawnCallback,
waitImpl = wait,
exitProcessImpl = (code) => process.exit(code),
} = {}) {
if (restartPlan.restartMode === "systemd" && process.platform !== "win32") {
const restartProcess = spawnImpl("systemctl", ["restart", restartPlan.serviceName], {
detached: true,
stdio: "ignore",
windowsHide: true,
});
restartProcess.unref?.();
} else {
const nextProcess = spawnImpl(process.execPath, [restartPlan.agentPath, "--config", restartPlan.configPath], {
detached: true,
stdio: "ignore",
windowsHide: true,
});
nextProcess.unref?.();
}
await waitImpl(restartPlan.restartGraceMs ?? DEFAULT_UPDATE_RESTART_GRACE_MS);
exitProcessImpl(0);
}
async function resumeAgentAfterRollback(configPath, config, {
spawnImpl = spawnCallback,
} = {}) {
const restartPlan = buildUpdateRestartPlan({}, configPath, config);
if (restartPlan.restartMode === "systemd" && process.platform !== "win32") {
const restartProcess = spawnImpl("systemctl", ["restart", restartPlan.serviceName], {
detached: true,
stdio: "ignore",
windowsHide: true,
});
restartProcess.unref?.();
return;
}
const nextProcess = spawnImpl(process.execPath, [restartPlan.agentPath, "--config", restartPlan.configPath], {
detached: true,
stdio: "ignore",
windowsHide: true,
});
nextProcess.unref?.();
}
export async function verifyPendingUpdate(configPath, {
waitImpl = wait,
execFileImpl = execFile,
spawnImpl = spawnCallback,
verifyIntervalMs = DEFAULT_UPDATE_VERIFY_INTERVAL_MS,
timeoutMs = null,
} = {}) {
const config = await loadConfig(configPath);
const pendingUpdate = config.pendingUpdate;
if (!pendingUpdate || typeof pendingUpdate !== "object") {
return {
verified: false,
skipped: true,
reason: "No pending update verification state",
};
}
const effectiveTimeoutMs =
Number.isFinite(timeoutMs) && timeoutMs !== null
? Math.max(1, Number(timeoutMs))
: resolveUpdateVerificationTimeoutSeconds(config, pendingUpdate) * 1000;
const deadline = Date.now() + effectiveTimeoutMs;
while (Date.now() < deadline) {
const latestConfig = await loadConfig(configPath).catch(() => null);
if (latestConfig && !latestConfig.pendingUpdate) {
return {
verified: true,
state: latestConfig.lastUpdate?.state || null,
};
}
await waitImpl(Math.max(25, verifyIntervalMs));
}
const rollbackReason = `Edge agent update to ${pendingUpdate.targetVersion || "unknown"} did not pass startup verification`;
const rollbackResult = await rollbackPendingUpdate(configPath, {
config,
execFileImpl,
reason: rollbackReason,
});
await resumeAgentAfterRollback(configPath, await loadConfig(configPath), {
spawnImpl,
});
return {
verified: false,
rolledBack: rollbackResult.rolledBack,
restoredVersion: rollbackResult.restoredVersion || null,
reason: rollbackReason,
};
}
export async function runUpdate(payload, fetchImpl = fetch, deps = {}) {
if (!payload.artifactUrl) {
return {
updated: false,
@@ -363,21 +816,83 @@ export async function runUpdate(payload, fetchImpl = fetch) {
};
}
const response = await fetchImpl(payload.artifactUrl);
if (!response.ok) {
throw new Error(`Artifact download failed: HTTP ${response.status}`);
}
const buffer = Buffer.from(await response.arrayBuffer());
const sha256 = createHash("sha256").update(buffer).digest("hex");
if (payload.sha256 && String(payload.sha256).toLowerCase() !== sha256.toLowerCase()) {
throw new Error("Artifact checksum mismatch");
const configPath = deps.configPath;
if (!configPath) {
throw new Error("Missing config path for edge agent update");
}
return {
updated: true,
sha256,
bytes: buffer.length,
};
const liveConfig = deps.liveConfig && typeof deps.liveConfig === "object" ? deps.liveConfig : null;
const currentConfig = cloneJson(deps.config || liveConfig || (await loadConfig(configPath)));
const originalConfig = cloneJson(currentConfig);
const paths = resolveUpdatePaths(configPath, currentConfig);
const safeTargetVersion = sanitizeUpdateSegment(
payload.targetVersion || payload.target_version || currentConfig.targetVersion || currentConfig.installedVersion
);
const backupDir = path.join(paths.updatesDir, `${Date.now()}-${safeTargetVersion}`);
await ensureDirectory(paths.installDir);
await ensureDirectory(paths.updatesDir);
const agentArtifact = await fetchArtifactBuffer(
payload.artifactUrl,
payload.sha256 || payload.artifactSha256 || payload.artifact_sha256,
"Agent artifact",
fetchImpl
);
const packageArtifact = await fetchArtifactBuffer(
payload.packageUrl || payload.package_url,
payload.packageSha256 || payload.package_sha256,
"Package manifest",
fetchImpl
);
let pendingUpdate = null;
try {
await ensureDirectory(backupDir);
await backupFileIfPresent(paths.agentPath, path.join(backupDir, "agent.mjs"));
await backupFileIfPresent(paths.packagePath, path.join(backupDir, "package.json"));
if (packageArtifact) {
await writeBuffer(paths.packagePath, packageArtifact.buffer);
}
await writeBuffer(paths.agentPath, agentArtifact.buffer);
await installAgentDependencies(paths.installDir, deps.execFileImpl || execFile);
await runAgentStatusPreflight(paths.agentPath, paths.configPath, deps.execFileImpl || execFile);
pendingUpdate = buildPendingUpdateState(payload, currentConfig, paths, backupDir);
const nextConfig = {
...currentConfig,
targetVersion: pendingUpdate.targetVersion,
pendingUpdate,
lastUpdate: null,
};
await saveConfig(paths.configPath, nextConfig);
if (liveConfig) {
replaceObjectContents(liveConfig, cloneJson(nextConfig));
}
return {
__agentCommandEnvelope: true,
payload: buildPreparedUpdatePayload({
pendingUpdate,
agentArtifact,
packageArtifact,
paths,
}),
followUp: {
type: "RUN_UPDATE",
restartPlan: buildUpdateRestartPlan(pendingUpdate, paths.configPath, nextConfig),
},
};
} catch (error) {
await restorePreparedUpdate(paths, pendingUpdate || buildPendingUpdateState(payload, currentConfig, paths, backupDir), originalConfig, {
execFileImpl: deps.execFileImpl || execFile,
liveConfig,
rollbackMessage: createUpdateErrorMessage(error),
});
throw error;
}
}
function defaultShellCommand() {
@@ -522,7 +1037,7 @@ export async function handleAgentCommand(command, deps = {}) {
case "SET_RELAY_STATE":
return await setRelayState(command.payload || {}, fetchImpl);
case "RUN_UPDATE":
return await runUpdate(command.payload || {}, fetchImpl);
return await runUpdate(command.payload || {}, fetchImpl, deps);
case "RESTART_AGENT":
return { restarted: true };
case "REBOOT_HOST":
@@ -533,6 +1048,7 @@ export async function handleAgentCommand(command, deps = {}) {
}
export function buildHeartbeatPayload(config, extra = {}) {
const lastUpdateMetadata = buildUpdateMetadata(config);
const payload = {
hostname: os.hostname(),
installed_version: config.installedVersion || DEFAULT_VERSION,
@@ -544,6 +1060,10 @@ export function buildHeartbeatPayload(config, extra = {}) {
},
};
if (lastUpdateMetadata) {
payload.metadata.last_update = lastUpdateMetadata;
}
if (Object.prototype.hasOwnProperty.call(extra, "discovery_status")) {
payload.discovery_status = extra.discovery_status;
}
@@ -678,13 +1198,40 @@ export async function processPolledCommand(config, command, fetchImpl = fetch) {
return null;
}
let followUp = null;
try {
const result = await handleAgentCommand({ commandType, payload }, { fetchImpl });
const result = await handleAgentCommand(
{ commandType, payload },
{
fetchImpl,
config,
configPath: payload.configPath || config.configPath || null,
liveConfig: config,
}
);
const responsePayload =
result && result.__agentCommandEnvelope === true
? (followUp = result.followUp || null, result.payload || {})
: result;
await submitCommandJobResult(config, jobId, {
ok: true,
payload: result,
payload: responsePayload,
}, fetchImpl);
return { ok: true, payload: result };
if (commandType === "RUN_UPDATE" && followUp?.type === "RUN_UPDATE") {
try {
await startDetachedUpdateVerifier(config.configPath || payload.configPath || null, config);
await restartAgentAfterUpdate(followUp.restartPlan, {});
} catch (followUpError) {
await rollbackPendingUpdate(config.configPath || payload.configPath || null, {
config,
liveConfig: config,
reason: createUpdateErrorMessage(followUpError, "Edge agent restart failed after update"),
}).catch(() => {});
}
}
return { ok: true, payload: responsePayload };
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
await submitCommandJobResult(config, jobId, {
@@ -859,7 +1406,13 @@ export async function startAgent({
}
let config = await loadConfig(configPath);
config.configPath = configPath;
config = await claimIfNeeded(config, configPath, fetchImpl);
config.configPath = configPath;
config = await finalizePendingUpdateOnStartup(config, configPath, {
liveConfig: config,
});
config.configPath = configPath;
const intervalMs = Number(config.heartbeatIntervalSeconds || 15) * 1000;
const commandPollTimeoutSeconds = Number(config.commandPollTimeoutSeconds || 20);
const shellActionPollTimeoutSeconds = Number(config.shellActionPollTimeoutSeconds || 20);
@@ -989,6 +1542,10 @@ export function parseCliArgs(argv = process.argv.slice(2)) {
command = "status";
continue;
}
if (arg === UPDATE_VERIFY_COMMAND && command === "start") {
command = UPDATE_VERIFY_COMMAND;
continue;
}
if (arg === "--config") {
configPath = argv[index + 1] || null;
index += 1;
@@ -1006,6 +1563,9 @@ export async function runCli(argv = process.argv.slice(2)) {
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
return report;
}
if (command === UPDATE_VERIFY_COMMAND) {
return verifyPendingUpdate(configPath);
}
return startAgent({ configPath });
}