Add resilient dual-radio network failover
This commit is contained in:
@@ -19,6 +19,7 @@ import {
|
||||
} from "./security.js";
|
||||
import { collectSystemStatus } from "./status.js";
|
||||
import { readUpdateStatus, triggerSystemUpdate } from "./update.js";
|
||||
import { networkActionSchema, readNetworkStatus, requestNetworkAction } from "./network.js";
|
||||
import { collectAdbStatus } from "./adb.js";
|
||||
import "./types.js";
|
||||
|
||||
@@ -313,6 +314,41 @@ export async function buildApp(config: AppConfig): Promise<FastifyInstance> {
|
||||
return reply.code(202).send(run);
|
||||
});
|
||||
|
||||
app.get("/api/network", async (request, reply) => {
|
||||
if (!requireUser(request, reply)) return;
|
||||
return readNetworkStatus(config);
|
||||
});
|
||||
|
||||
app.post("/api/network/actions", async (request, reply) => {
|
||||
if (!requireUser(request, reply)) return;
|
||||
const parsed = z.object({ action: networkActionSchema }).safeParse(request.body);
|
||||
if (!parsed.success) {
|
||||
return reply.code(400).send({ error: { code: "INVALID_NETWORK_ACTION", message: "That network action is not available" } });
|
||||
}
|
||||
try {
|
||||
const command = await requestNetworkAction(config, parsed.data.action, request.authUser!.id);
|
||||
audit(database, {
|
||||
actorUserId: request.authUser!.id,
|
||||
action: `network.${parsed.data.action}`,
|
||||
result: "success",
|
||||
ipAddress: clientAddress(request),
|
||||
details: { commandId: command.id }
|
||||
});
|
||||
return reply.code(202).send({ accepted: true, commandId: command.id });
|
||||
} catch {
|
||||
audit(database, {
|
||||
actorUserId: request.authUser!.id,
|
||||
action: `network.${parsed.data.action}`,
|
||||
result: "failure",
|
||||
ipAddress: clientAddress(request),
|
||||
details: { reason: "controller_unavailable" }
|
||||
});
|
||||
return reply.code(503).send({
|
||||
error: { code: "NETWORK_CONTROL_UNAVAILABLE", message: "Automatic network management is not configured" }
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/api/system/update", async (request, reply) => {
|
||||
if (!requireUser(request, reply)) return;
|
||||
return readUpdateStatus(config);
|
||||
|
||||
@@ -17,6 +17,9 @@ const schema = z.object({
|
||||
ALLOWED_ORIGINS: z.string().default("http://mg4pi.local:8787"),
|
||||
UPDATE_STATUS_PATH: z.string().default("/var/lib/pi-car-companion/update-status.json"),
|
||||
VERSION_FILE: z.string().default(resolve("REVISION")),
|
||||
NETWORK_MANAGER_ENABLED: booleanFromString,
|
||||
NETWORK_STATUS_PATH: z.string().default("/var/lib/pi-car-companion/network-status.json"),
|
||||
NETWORK_COMMAND_PATH: z.string().default("/var/lib/pi-car-companion/network-command.json"),
|
||||
ADB_ENABLED: booleanFromString,
|
||||
ADB_PATH: z.string().min(1).default("adb"),
|
||||
ADB_SERIAL: z.string().max(128).regex(/^[a-zA-Z0-9._:-]*$/).default(""),
|
||||
@@ -33,6 +36,11 @@ export type AppConfig = {
|
||||
allowedOrigins: ReadonlySet<string>;
|
||||
updateStatusPath: string;
|
||||
versionFile: string;
|
||||
network: {
|
||||
enabled: boolean;
|
||||
statusPath: string;
|
||||
commandPath: string;
|
||||
};
|
||||
adb: AdbConfig;
|
||||
};
|
||||
|
||||
@@ -60,6 +68,11 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): AppCon
|
||||
allowedOrigins,
|
||||
updateStatusPath: parsed.UPDATE_STATUS_PATH,
|
||||
versionFile: parsed.VERSION_FILE,
|
||||
network: {
|
||||
enabled: parsed.NETWORK_MANAGER_ENABLED,
|
||||
statusPath: parsed.NETWORK_STATUS_PATH,
|
||||
commandPath: parsed.NETWORK_COMMAND_PATH
|
||||
},
|
||||
adb: {
|
||||
enabled: parsed.ADB_ENABLED,
|
||||
executable: parsed.ADB_PATH,
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { promisify } from "node:util";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
consumeNetworkCommand,
|
||||
networkStatusSchema,
|
||||
writeNetworkStatus,
|
||||
type NetworkAction,
|
||||
type NetworkMode,
|
||||
type NetworkStatus
|
||||
} from "./network.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const sleep = (milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
|
||||
const controllerConfigSchema = z.object({
|
||||
NETWORK_MANAGER_ENABLED: z.enum(["true", "false"]).default("false").transform((value) => value === "true"),
|
||||
NETWORK_STATUS_PATH: z.string().default("/var/lib/pi-car-companion/network-status.json"),
|
||||
NETWORK_COMMAND_PATH: z.string().default("/var/lib/pi-car-companion/network-command.json"),
|
||||
NETWORK_HOTSPOT_INTERFACE: z.string().regex(/^[a-zA-Z0-9_.-]+$/).default("wlan0"),
|
||||
NETWORK_UPSTREAM_INTERFACE: z.string().regex(/^[a-zA-Z0-9_.-]+$/).default("wlan1"),
|
||||
NETWORK_HOTSPOT_CONNECTION: z.string().min(1).max(128).default("pi-car-hotspot"),
|
||||
NETWORK_PROBE_TIMEOUT_SECONDS: z.coerce.number().int().min(15).max(180).default(60),
|
||||
NETWORK_STABILITY_SECONDS: z.coerce.number().int().min(3).max(60).default(10),
|
||||
NETWORK_POLL_SECONDS: z.coerce.number().int().min(2).max(30).default(5)
|
||||
});
|
||||
|
||||
type ControllerConfig = z.infer<typeof controllerConfigSchema>;
|
||||
type Adapter = NetworkStatus["hotspot"];
|
||||
|
||||
async function nmcli(args: string[], timeoutMs = 20_000): Promise<string> {
|
||||
const { stdout } = await execFileAsync("/usr/bin/nmcli", args, { timeout: timeoutMs, maxBuffer: 32 * 1024 });
|
||||
return stdout.trim();
|
||||
}
|
||||
|
||||
async function field(device: string, name: string): Promise<string | null> {
|
||||
try {
|
||||
return (await nmcli(["--get-values", name, "device", "show", device])).split("\n")[0]?.trim() || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function inspectAdapter(device: string): Promise<Adapter> {
|
||||
const type = await field(device, "GENERAL.TYPE");
|
||||
if (type !== "wifi") return { interface: device, available: false, connected: false, connection: null, address: null };
|
||||
const state = await field(device, "GENERAL.STATE");
|
||||
return {
|
||||
interface: device,
|
||||
available: true,
|
||||
connected: state?.startsWith("100") ?? false,
|
||||
connection: await field(device, "GENERAL.CONNECTION"),
|
||||
address: await field(device, "IP4.ADDRESS")
|
||||
};
|
||||
}
|
||||
|
||||
async function connectivity(): Promise<NetworkStatus["connectivity"]> {
|
||||
try {
|
||||
const value = await nmcli(["networking", "connectivity", "check"], 10_000);
|
||||
return ["full", "limited", "portal", "none", "unknown"].includes(value)
|
||||
? (value as NetworkStatus["connectivity"])
|
||||
: "unknown";
|
||||
} catch {
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
class NetworkController {
|
||||
private mode: NetworkMode = "starting";
|
||||
private message = "Inspecting network adapters.";
|
||||
private lastTransitionAt = new Date().toISOString();
|
||||
private lastFailure: string | null = null;
|
||||
private probeDeadline: string | null = null;
|
||||
private fallbackAttempted = false;
|
||||
private nextAwusAttemptAt = 0;
|
||||
private stopped = false;
|
||||
|
||||
constructor(private readonly config: ControllerConfig) {}
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
}
|
||||
|
||||
private transition(mode: NetworkMode, message: string, failure: string | null = null): void {
|
||||
if (this.mode !== mode) this.lastTransitionAt = new Date().toISOString();
|
||||
this.mode = mode;
|
||||
this.message = message;
|
||||
this.lastFailure = failure;
|
||||
}
|
||||
|
||||
private async publish(): Promise<{ hotspot: Adapter; upstream: Adapter }> {
|
||||
const [hotspot, upstream, internet] = await Promise.all([
|
||||
inspectAdapter(this.config.NETWORK_HOTSPOT_INTERFACE),
|
||||
inspectAdapter(this.config.NETWORK_UPSTREAM_INTERFACE),
|
||||
connectivity()
|
||||
]);
|
||||
await writeNetworkStatus(this.config.NETWORK_STATUS_PATH, {
|
||||
enabled: true,
|
||||
mode: this.mode,
|
||||
message: this.message,
|
||||
updatedAt: new Date().toISOString(),
|
||||
lastTransitionAt: this.lastTransitionAt,
|
||||
probeDeadline: this.probeDeadline,
|
||||
lastFailure: this.lastFailure,
|
||||
connectivity: internet,
|
||||
hotspot,
|
||||
upstream
|
||||
});
|
||||
return { hotspot, upstream };
|
||||
}
|
||||
|
||||
private async connectUpstreamAdapter(): Promise<boolean> {
|
||||
try {
|
||||
await nmcli(["--wait", "20", "device", "connect", this.config.NETWORK_UPSTREAM_INTERFACE], 25_000);
|
||||
await sleep(this.config.NETWORK_STABILITY_SECONDS * 1000);
|
||||
return (await inspectAdapter(this.config.NETWORK_UPSTREAM_INTERFACE)).connected;
|
||||
} catch (error) {
|
||||
this.lastFailure = error instanceof Error ? error.message.slice(0, 500) : "The upstream adapter could not connect.";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async restoreHotspot(message = "Offline hotspot restored; the dashboard remains available without internet."): Promise<boolean> {
|
||||
this.probeDeadline = null;
|
||||
try {
|
||||
await nmcli(
|
||||
["--wait", "20", "connection", "up", this.config.NETWORK_HOTSPOT_CONNECTION, "ifname", this.config.NETWORK_HOTSPOT_INTERFACE],
|
||||
25_000
|
||||
);
|
||||
const upstream = await inspectAdapter(this.config.NETWORK_UPSTREAM_INTERFACE);
|
||||
this.transition(upstream.connected ? "dual_radio" : "offline_hotspot", upstream.connected ? "Car hotspot is active with AWUS upstream internet." : message);
|
||||
return true;
|
||||
} catch (error) {
|
||||
const failure = error instanceof Error ? error.message.slice(0, 500) : "The hotspot could not be restored.";
|
||||
this.transition("error", "The recovery hotspot could not be activated. Local intervention is required.", failure);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async probeBuiltIn(): Promise<void> {
|
||||
const timeout = this.config.NETWORK_PROBE_TIMEOUT_SECONDS;
|
||||
this.probeDeadline = new Date(Date.now() + timeout * 1000).toISOString();
|
||||
this.transition("probing_built_in", `Trying saved upstream networks on the built-in adapter for up to ${timeout} seconds.`);
|
||||
await this.publish();
|
||||
try {
|
||||
await nmcli(["--wait", "10", "connection", "down", this.config.NETWORK_HOTSPOT_CONNECTION], 15_000).catch(() => undefined);
|
||||
await nmcli(["--wait", String(timeout), "device", "connect", this.config.NETWORK_HOTSPOT_INTERFACE], (timeout + 5) * 1000);
|
||||
if ((await inspectAdapter(this.config.NETWORK_HOTSPOT_INTERFACE)).connected) {
|
||||
this.probeDeadline = null;
|
||||
this.transition("built_in_upstream", "The built-in adapter is connected upstream; the car hotspot is temporarily unavailable.");
|
||||
this.fallbackAttempted = false;
|
||||
return;
|
||||
}
|
||||
throw new Error("The built-in adapter did not activate a saved upstream profile.");
|
||||
} catch (error) {
|
||||
this.lastFailure = error instanceof Error ? error.message.slice(0, 500) : "No saved upstream network was available.";
|
||||
this.fallbackAttempted = true;
|
||||
await this.restoreHotspot();
|
||||
}
|
||||
}
|
||||
|
||||
private async handleAction(action: NetworkAction): Promise<void> {
|
||||
if (action === "restore-hotspot") {
|
||||
this.fallbackAttempted = true;
|
||||
await this.restoreHotspot("The offline hotspot was restored manually.");
|
||||
return;
|
||||
}
|
||||
this.fallbackAttempted = false;
|
||||
const upstream = await inspectAdapter(this.config.NETWORK_UPSTREAM_INTERFACE);
|
||||
if (upstream.available && (upstream.connected || (await this.connectUpstreamAdapter()))) {
|
||||
await this.restoreHotspot();
|
||||
return;
|
||||
}
|
||||
await this.probeBuiltIn();
|
||||
}
|
||||
|
||||
async run(): Promise<void> {
|
||||
try {
|
||||
const prior = networkStatusSchema.parse(JSON.parse(await readFile(this.config.NETWORK_STATUS_PATH, "utf8")));
|
||||
this.mode = prior.mode;
|
||||
this.message = prior.message;
|
||||
this.lastTransitionAt = prior.lastTransitionAt;
|
||||
this.lastFailure = prior.lastFailure;
|
||||
this.fallbackAttempted = prior.mode === "offline_hotspot";
|
||||
} catch {
|
||||
// A first start intentionally discovers the live NetworkManager state.
|
||||
}
|
||||
let previous = await this.publish();
|
||||
|
||||
while (!this.stopped) {
|
||||
const command = await consumeNetworkCommand(this.config.NETWORK_COMMAND_PATH);
|
||||
if (command) await this.handleAction(command.action);
|
||||
|
||||
let { hotspot, upstream } = await this.publish();
|
||||
const hotspotActive = hotspot.connected && hotspot.connection === this.config.NETWORK_HOTSPOT_CONNECTION;
|
||||
const builtInUpstream = hotspot.connected && !hotspotActive;
|
||||
|
||||
if (upstream.connected) {
|
||||
if (!hotspotActive) await this.restoreHotspot();
|
||||
else this.transition("dual_radio", "Car hotspot is active with AWUS upstream internet.");
|
||||
this.fallbackAttempted = false;
|
||||
} else if (builtInUpstream) {
|
||||
this.transition("built_in_upstream", "The built-in adapter is connected upstream; the car hotspot is temporarily unavailable.");
|
||||
if (upstream.available && Date.now() >= this.nextAwusAttemptAt) {
|
||||
this.nextAwusAttemptAt = Date.now() + 60_000;
|
||||
if (await this.connectUpstreamAdapter()) await this.restoreHotspot();
|
||||
}
|
||||
} else {
|
||||
const awusJustFailed = previous.upstream.connected && !upstream.connected;
|
||||
if (upstream.available && Date.now() >= this.nextAwusAttemptAt) {
|
||||
this.nextAwusAttemptAt = Date.now() + 60_000;
|
||||
if (await this.connectUpstreamAdapter()) {
|
||||
await this.restoreHotspot();
|
||||
({ hotspot, upstream } = await this.publish());
|
||||
}
|
||||
}
|
||||
if (!upstream.connected && !this.fallbackAttempted && (awusJustFailed || hotspotActive || !hotspot.connected)) {
|
||||
await this.probeBuiltIn();
|
||||
} else if (!upstream.connected && hotspotActive) {
|
||||
this.transition("offline_hotspot", "Offline hotspot is active; no upstream network is currently connected.", this.lastFailure);
|
||||
}
|
||||
}
|
||||
|
||||
previous = await this.publish();
|
||||
await sleep(this.config.NETWORK_POLL_SECONDS * 1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const config = controllerConfigSchema.parse(process.env);
|
||||
if (!config.NETWORK_MANAGER_ENABLED) {
|
||||
process.exit(0);
|
||||
}
|
||||
const controller = new NetworkController(config);
|
||||
process.on("SIGINT", () => controller.stop());
|
||||
process.on("SIGTERM", () => controller.stop());
|
||||
await controller.run();
|
||||
@@ -0,0 +1,116 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { chmod, readFile, rename, writeFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { z } from "zod";
|
||||
import type { AppConfig } from "./config.js";
|
||||
|
||||
export const networkModeSchema = z.enum([
|
||||
"disabled",
|
||||
"starting",
|
||||
"dual_radio",
|
||||
"built_in_upstream",
|
||||
"probing_built_in",
|
||||
"offline_hotspot",
|
||||
"degraded",
|
||||
"error"
|
||||
]);
|
||||
export type NetworkMode = z.infer<typeof networkModeSchema>;
|
||||
|
||||
const adapterSchema = z.object({
|
||||
interface: z.string().max(64),
|
||||
available: z.boolean(),
|
||||
connected: z.boolean(),
|
||||
connection: z.string().max(128).nullable(),
|
||||
address: z.string().max(128).nullable()
|
||||
});
|
||||
|
||||
export const networkStatusSchema = z.object({
|
||||
enabled: z.boolean(),
|
||||
mode: networkModeSchema,
|
||||
message: z.string().max(500),
|
||||
updatedAt: z.string(),
|
||||
lastTransitionAt: z.string(),
|
||||
probeDeadline: z.string().nullable(),
|
||||
lastFailure: z.string().max(500).nullable(),
|
||||
connectivity: z.enum(["full", "limited", "portal", "none", "unknown"]),
|
||||
hotspot: adapterSchema,
|
||||
upstream: adapterSchema
|
||||
});
|
||||
|
||||
export type NetworkStatus = z.infer<typeof networkStatusSchema> & { supported: boolean };
|
||||
export const networkActionSchema = z.enum(["retry-upstream", "restore-hotspot"]);
|
||||
export type NetworkAction = z.infer<typeof networkActionSchema>;
|
||||
|
||||
const commandSchema = z.object({
|
||||
id: z.string().uuid(),
|
||||
action: networkActionSchema,
|
||||
requestedAt: z.string(),
|
||||
actorUserId: z.number().int().positive()
|
||||
});
|
||||
export type NetworkCommand = z.infer<typeof commandSchema>;
|
||||
|
||||
async function atomicJsonWrite(path: string, value: unknown): Promise<void> {
|
||||
const temporary = join(dirname(path), `.${randomUUID()}.tmp`);
|
||||
await writeFile(temporary, `${JSON.stringify(value)}\n`, { mode: 0o640 });
|
||||
await chmod(temporary, 0o640);
|
||||
await rename(temporary, path);
|
||||
}
|
||||
|
||||
export async function readNetworkStatus(config: AppConfig): Promise<NetworkStatus> {
|
||||
if (!config.network.enabled) {
|
||||
return unsupportedStatus("Automatic network management is disabled until its hotspot and upstream profiles are configured.");
|
||||
}
|
||||
try {
|
||||
return { ...networkStatusSchema.parse(JSON.parse(await readFile(config.network.statusPath, "utf8"))), supported: true };
|
||||
} catch {
|
||||
return {
|
||||
...unsupportedStatus("The network controller is starting or its status is unavailable."),
|
||||
enabled: true,
|
||||
mode: "starting"
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function requestNetworkAction(
|
||||
config: AppConfig,
|
||||
action: NetworkAction,
|
||||
actorUserId: number
|
||||
): Promise<NetworkCommand> {
|
||||
if (!config.network.enabled) throw new Error("NETWORK_MANAGER_DISABLED");
|
||||
const command = commandSchema.parse({ id: randomUUID(), action, requestedAt: new Date().toISOString(), actorUserId });
|
||||
await atomicJsonWrite(config.network.commandPath, command);
|
||||
return command;
|
||||
}
|
||||
|
||||
export async function consumeNetworkCommand(path: string): Promise<NetworkCommand | null> {
|
||||
try {
|
||||
const parsed = commandSchema.parse(JSON.parse(await readFile(path, "utf8")));
|
||||
await writeFile(path, "", { mode: 0o640 });
|
||||
return parsed;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function unsupportedStatus(message: string): NetworkStatus {
|
||||
const epoch = new Date(0).toISOString();
|
||||
const adapter = { interface: "unconfigured", available: false, connected: false, connection: null, address: null };
|
||||
return {
|
||||
supported: false,
|
||||
enabled: false,
|
||||
mode: "disabled",
|
||||
message,
|
||||
updatedAt: epoch,
|
||||
lastTransitionAt: epoch,
|
||||
probeDeadline: null,
|
||||
lastFailure: null,
|
||||
connectivity: "unknown",
|
||||
hotspot: adapter,
|
||||
upstream: adapter
|
||||
};
|
||||
}
|
||||
|
||||
export async function writeNetworkStatus(path: string, status: z.infer<typeof networkStatusSchema>): Promise<void> {
|
||||
await atomicJsonWrite(path, networkStatusSchema.parse(status));
|
||||
await chmod(path, 0o644);
|
||||
}
|
||||
Reference in New Issue
Block a user