Add resilient dual-radio network failover

2026-07-31 20:24:34 +02:00
parent 0a0dae144c
commit 6940cfb3a8
19 changed files with 857 additions and 11 deletions
@@ -0,0 +1,238 @@
+import { execFile } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { promisify } from "node:util";
+import { z } from "zod";
+import {
+  consumeNetworkCommand,
+  networkStatusSchema,
+  writeNetworkStatus,
+  type NetworkAction,
+  type NetworkMode,
+  type NetworkStatus
+} from "./network.js";
+
+const execFileAsync = promisify(execFile);
+const sleep = (milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds));
+
+const controllerConfigSchema = z.object({
+  NETWORK_MANAGER_ENABLED: z.enum(["true", "false"]).default("false").transform((value) => value === "true"),
+  NETWORK_STATUS_PATH: z.string().default("/var/lib/pi-car-companion/network-status.json"),
+  NETWORK_COMMAND_PATH: z.string().default("/var/lib/pi-car-companion/network-command.json"),
+  NETWORK_HOTSPOT_INTERFACE: z.string().regex(/^[a-zA-Z0-9_.-]+$/).default("wlan0"),
+  NETWORK_UPSTREAM_INTERFACE: z.string().regex(/^[a-zA-Z0-9_.-]+$/).default("wlan1"),
+  NETWORK_HOTSPOT_CONNECTION: z.string().min(1).max(128).default("pi-car-hotspot"),
+  NETWORK_PROBE_TIMEOUT_SECONDS: z.coerce.number().int().min(15).max(180).default(60),
+  NETWORK_STABILITY_SECONDS: z.coerce.number().int().min(3).max(60).default(10),
+  NETWORK_POLL_SECONDS: z.coerce.number().int().min(2).max(30).default(5)
+});
+
+type ControllerConfig = z.infer<typeof controllerConfigSchema>;
+type Adapter = NetworkStatus["hotspot"];
+
+async function nmcli(args: string[], timeoutMs = 20_000): Promise<string> {
+  const { stdout } = await execFileAsync("/usr/bin/nmcli", args, { timeout: timeoutMs, maxBuffer: 32 * 1024 });
+  return stdout.trim();
+}
+
+async function field(device: string, name: string): Promise<string | null> {
+  try {
+    return (await nmcli(["--get-values", name, "device", "show", device])).split("\n")[0]?.trim() || null;
+  } catch {
+    return null;
+  }
+}
+
+async function inspectAdapter(device: string): Promise<Adapter> {
+  const type = await field(device, "GENERAL.TYPE");
+  if (type !== "wifi") return { interface: device, available: false, connected: false, connection: null, address: null };
+  const state = await field(device, "GENERAL.STATE");
+  return {
+    interface: device,
+    available: true,
+    connected: state?.startsWith("100") ?? false,
+    connection: await field(device, "GENERAL.CONNECTION"),
+    address: await field(device, "IP4.ADDRESS")
+  };
+}
+
+async function connectivity(): Promise<NetworkStatus["connectivity"]> {
+  try {
+    const value = await nmcli(["networking", "connectivity", "check"], 10_000);
+    return ["full", "limited", "portal", "none", "unknown"].includes(value)
+      ? (value as NetworkStatus["connectivity"])
+      : "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
+class NetworkController {
+  private mode: NetworkMode = "starting";
+  private message = "Inspecting network adapters.";
+  private lastTransitionAt = new Date().toISOString();
+  private lastFailure: string | null = null;
+  private probeDeadline: string | null = null;
+  private fallbackAttempted = false;
+  private nextAwusAttemptAt = 0;
+  private stopped = false;
+
+  constructor(private readonly config: ControllerConfig) {}
+
+  stop(): void {
+    this.stopped = true;
+  }
+
+  private transition(mode: NetworkMode, message: string, failure: string | null = null): void {
+    if (this.mode !== mode) this.lastTransitionAt = new Date().toISOString();
+    this.mode = mode;
+    this.message = message;
+    this.lastFailure = failure;
+  }
+
+  private async publish(): Promise<{ hotspot: Adapter; upstream: Adapter }> {
+    const [hotspot, upstream, internet] = await Promise.all([
+      inspectAdapter(this.config.NETWORK_HOTSPOT_INTERFACE),
+      inspectAdapter(this.config.NETWORK_UPSTREAM_INTERFACE),
+      connectivity()
+    ]);
+    await writeNetworkStatus(this.config.NETWORK_STATUS_PATH, {
+      enabled: true,
+      mode: this.mode,
+      message: this.message,
+      updatedAt: new Date().toISOString(),
+      lastTransitionAt: this.lastTransitionAt,
+      probeDeadline: this.probeDeadline,
+      lastFailure: this.lastFailure,
+      connectivity: internet,
+      hotspot,
+      upstream
+    });
+    return { hotspot, upstream };
+  }
+
+  private async connectUpstreamAdapter(): Promise<boolean> {
+    try {
+      await nmcli(["--wait", "20", "device", "connect", this.config.NETWORK_UPSTREAM_INTERFACE], 25_000);
+      await sleep(this.config.NETWORK_STABILITY_SECONDS * 1000);
+      return (await inspectAdapter(this.config.NETWORK_UPSTREAM_INTERFACE)).connected;
+    } catch (error) {
+      this.lastFailure = error instanceof Error ? error.message.slice(0, 500) : "The upstream adapter could not connect.";
+      return false;
+    }
+  }
+
+  private async restoreHotspot(message = "Offline hotspot restored; the dashboard remains available without internet."): Promise<boolean> {
+    this.probeDeadline = null;
+    try {
+      await nmcli(
+        ["--wait", "20", "connection", "up", this.config.NETWORK_HOTSPOT_CONNECTION, "ifname", this.config.NETWORK_HOTSPOT_INTERFACE],
+        25_000
+      );
+      const upstream = await inspectAdapter(this.config.NETWORK_UPSTREAM_INTERFACE);
+      this.transition(upstream.connected ? "dual_radio" : "offline_hotspot", upstream.connected ? "Car hotspot is active with AWUS upstream internet." : message);
+      return true;
+    } catch (error) {
+      const failure = error instanceof Error ? error.message.slice(0, 500) : "The hotspot could not be restored.";
+      this.transition("error", "The recovery hotspot could not be activated. Local intervention is required.", failure);
+      return false;
+    }
+  }
+
+  private async probeBuiltIn(): Promise<void> {
+    const timeout = this.config.NETWORK_PROBE_TIMEOUT_SECONDS;
+    this.probeDeadline = new Date(Date.now() + timeout * 1000).toISOString();
+    this.transition("probing_built_in", `Trying saved upstream networks on the built-in adapter for up to ${timeout} seconds.`);
+    await this.publish();
+    try {
+      await nmcli(["--wait", "10", "connection", "down", this.config.NETWORK_HOTSPOT_CONNECTION], 15_000).catch(() => undefined);
+      await nmcli(["--wait", String(timeout), "device", "connect", this.config.NETWORK_HOTSPOT_INTERFACE], (timeout + 5) * 1000);
+      if ((await inspectAdapter(this.config.NETWORK_HOTSPOT_INTERFACE)).connected) {
+        this.probeDeadline = null;
+        this.transition("built_in_upstream", "The built-in adapter is connected upstream; the car hotspot is temporarily unavailable.");
+        this.fallbackAttempted = false;
+        return;
+      }
+      throw new Error("The built-in adapter did not activate a saved upstream profile.");
+    } catch (error) {
+      this.lastFailure = error instanceof Error ? error.message.slice(0, 500) : "No saved upstream network was available.";
+      this.fallbackAttempted = true;
+      await this.restoreHotspot();
+    }
+  }
+
+  private async handleAction(action: NetworkAction): Promise<void> {
+    if (action === "restore-hotspot") {
+      this.fallbackAttempted = true;
+      await this.restoreHotspot("The offline hotspot was restored manually.");
+      return;
+    }
+    this.fallbackAttempted = false;
+    const upstream = await inspectAdapter(this.config.NETWORK_UPSTREAM_INTERFACE);
+    if (upstream.available && (upstream.connected || (await this.connectUpstreamAdapter()))) {
+      await this.restoreHotspot();
+      return;
+    }
+    await this.probeBuiltIn();
+  }
+
+  async run(): Promise<void> {
+    try {
+      const prior = networkStatusSchema.parse(JSON.parse(await readFile(this.config.NETWORK_STATUS_PATH, "utf8")));
+      this.mode = prior.mode;
+      this.message = prior.message;
+      this.lastTransitionAt = prior.lastTransitionAt;
+      this.lastFailure = prior.lastFailure;
+      this.fallbackAttempted = prior.mode === "offline_hotspot";
+    } catch {
+      // A first start intentionally discovers the live NetworkManager state.
+    }
+    let previous = await this.publish();
+
+    while (!this.stopped) {
+      const command = await consumeNetworkCommand(this.config.NETWORK_COMMAND_PATH);
+      if (command) await this.handleAction(command.action);
+
+      let { hotspot, upstream } = await this.publish();
+      const hotspotActive = hotspot.connected && hotspot.connection === this.config.NETWORK_HOTSPOT_CONNECTION;
+      const builtInUpstream = hotspot.connected && !hotspotActive;
+
+      if (upstream.connected) {
+        if (!hotspotActive) await this.restoreHotspot();
+        else this.transition("dual_radio", "Car hotspot is active with AWUS upstream internet.");
+        this.fallbackAttempted = false;
+      } else if (builtInUpstream) {
+        this.transition("built_in_upstream", "The built-in adapter is connected upstream; the car hotspot is temporarily unavailable.");
+        if (upstream.available && Date.now() >= this.nextAwusAttemptAt) {
+          this.nextAwusAttemptAt = Date.now() + 60_000;
+          if (await this.connectUpstreamAdapter()) await this.restoreHotspot();
+        }
+      } else {
+        const awusJustFailed = previous.upstream.connected && !upstream.connected;
+        if (upstream.available && Date.now() >= this.nextAwusAttemptAt) {
+          this.nextAwusAttemptAt = Date.now() + 60_000;
+          if (await this.connectUpstreamAdapter()) {
+            await this.restoreHotspot();
+            ({ hotspot, upstream } = await this.publish());
+          }
+        }
+        if (!upstream.connected && !this.fallbackAttempted && (awusJustFailed || hotspotActive || !hotspot.connected)) {
+          await this.probeBuiltIn();
+        } else if (!upstream.connected && hotspotActive) {
+          this.transition("offline_hotspot", "Offline hotspot is active; no upstream network is currently connected.", this.lastFailure);
+        }
+      }
+
+      previous = await this.publish();
+      await sleep(this.config.NETWORK_POLL_SECONDS * 1000);
+    }
+  }
+}
+
+const config = controllerConfigSchema.parse(process.env);
+if (!config.NETWORK_MANAGER_ENABLED) {
+  process.exit(0);
+}
+const controller = new NetworkController(config);
+process.on("SIGINT", () => controller.stop());
+process.on("SIGTERM", () => controller.stop());
+await controller.run();