Add resilient dual-radio network failover

server/src/app.ts

@@ -19,6 +19,7 @@ import {
 } from "./security.js";
 import { collectSystemStatus } from "./status.js";
 import { readUpdateStatus, triggerSystemUpdate } from "./update.js";
+import { networkActionSchema, readNetworkStatus, requestNetworkAction } from "./network.js";
 import { collectAdbStatus } from "./adb.js";
 import "./types.js";
 
@@ -313,6 +314,41 @@ export async function buildApp(config: AppConfig): Promise<FastifyInstance> {
     return reply.code(202).send(run);
   });
 
+  app.get("/api/network", async (request, reply) => {
+    if (!requireUser(request, reply)) return;
+    return readNetworkStatus(config);
+  });
+
+  app.post("/api/network/actions", async (request, reply) => {
+    if (!requireUser(request, reply)) return;
+    const parsed = z.object({ action: networkActionSchema }).safeParse(request.body);
+    if (!parsed.success) {
+      return reply.code(400).send({ error: { code: "INVALID_NETWORK_ACTION", message: "That network action is not available" } });
+    }
+    try {
+      const command = await requestNetworkAction(config, parsed.data.action, request.authUser!.id);
+      audit(database, {
+        actorUserId: request.authUser!.id,
+        action: `network.${parsed.data.action}`,
+        result: "success",
+        ipAddress: clientAddress(request),
+        details: { commandId: command.id }
+      });
+      return reply.code(202).send({ accepted: true, commandId: command.id });
+    } catch {
+      audit(database, {
+        actorUserId: request.authUser!.id,
+        action: `network.${parsed.data.action}`,
+        result: "failure",
+        ipAddress: clientAddress(request),
+        details: { reason: "controller_unavailable" }
+      });
+      return reply.code(503).send({
+        error: { code: "NETWORK_CONTROL_UNAVAILABLE", message: "Automatic network management is not configured" }
+      });
+    }
+  });
+
   app.get("/api/system/update", async (request, reply) => {
     if (!requireUser(request, reply)) return;
     return readUpdateStatus(config);

server/src/config.ts

@@ -17,6 +17,9 @@ const schema = z.object({
   ALLOWED_ORIGINS: z.string().default("http://mg4pi.local:8787"),
   UPDATE_STATUS_PATH: z.string().default("/var/lib/pi-car-companion/update-status.json"),
   VERSION_FILE: z.string().default(resolve("REVISION")),
+  NETWORK_MANAGER_ENABLED: booleanFromString,
+  NETWORK_STATUS_PATH: z.string().default("/var/lib/pi-car-companion/network-status.json"),
+  NETWORK_COMMAND_PATH: z.string().default("/var/lib/pi-car-companion/network-command.json"),
   ADB_ENABLED: booleanFromString,
   ADB_PATH: z.string().min(1).default("adb"),
   ADB_SERIAL: z.string().max(128).regex(/^[a-zA-Z0-9._:-]*$/).default(""),
@@ -33,6 +36,11 @@ export type AppConfig = {
   allowedOrigins: ReadonlySet<string>;
   updateStatusPath: string;
   versionFile: string;
+  network: {
+    enabled: boolean;
+    statusPath: string;
+    commandPath: string;
+  };
   adb: AdbConfig;
 };
 
@@ -60,6 +68,11 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): AppCon
     allowedOrigins,
     updateStatusPath: parsed.UPDATE_STATUS_PATH,
     versionFile: parsed.VERSION_FILE,
+    network: {
+      enabled: parsed.NETWORK_MANAGER_ENABLED,
+      statusPath: parsed.NETWORK_STATUS_PATH,
+      commandPath: parsed.NETWORK_COMMAND_PATH
+    },
     adb: {
       enabled: parsed.ADB_ENABLED,
       executable: parsed.ADB_PATH,

server/src/network-controller.ts

@@ -0,0 +1,238 @@
+import { execFile } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { promisify } from "node:util";
+import { z } from "zod";
+import {
+  consumeNetworkCommand,
+  networkStatusSchema,
+  writeNetworkStatus,
+  type NetworkAction,
+  type NetworkMode,
+  type NetworkStatus
+} from "./network.js";
+
+const execFileAsync = promisify(execFile);
+const sleep = (milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds));
+
+const controllerConfigSchema = z.object({
+  NETWORK_MANAGER_ENABLED: z.enum(["true", "false"]).default("false").transform((value) => value === "true"),
+  NETWORK_STATUS_PATH: z.string().default("/var/lib/pi-car-companion/network-status.json"),
+  NETWORK_COMMAND_PATH: z.string().default("/var/lib/pi-car-companion/network-command.json"),
+  NETWORK_HOTSPOT_INTERFACE: z.string().regex(/^[a-zA-Z0-9_.-]+$/).default("wlan0"),
+  NETWORK_UPSTREAM_INTERFACE: z.string().regex(/^[a-zA-Z0-9_.-]+$/).default("wlan1"),
+  NETWORK_HOTSPOT_CONNECTION: z.string().min(1).max(128).default("pi-car-hotspot"),
+  NETWORK_PROBE_TIMEOUT_SECONDS: z.coerce.number().int().min(15).max(180).default(60),
+  NETWORK_STABILITY_SECONDS: z.coerce.number().int().min(3).max(60).default(10),
+  NETWORK_POLL_SECONDS: z.coerce.number().int().min(2).max(30).default(5)
+});
+
+type ControllerConfig = z.infer<typeof controllerConfigSchema>;
+type Adapter = NetworkStatus["hotspot"];
+
+async function nmcli(args: string[], timeoutMs = 20_000): Promise<string> {
+  const { stdout } = await execFileAsync("/usr/bin/nmcli", args, { timeout: timeoutMs, maxBuffer: 32 * 1024 });
+  return stdout.trim();
+}
+
+async function field(device: string, name: string): Promise<string | null> {
+  try {
+    return (await nmcli(["--get-values", name, "device", "show", device])).split("\n")[0]?.trim() || null;
+  } catch {
+    return null;
+  }
+}
+
+async function inspectAdapter(device: string): Promise<Adapter> {
+  const type = await field(device, "GENERAL.TYPE");
+  if (type !== "wifi") return { interface: device, available: false, connected: false, connection: null, address: null };
+  const state = await field(device, "GENERAL.STATE");
+  return {
+    interface: device,
+    available: true,
+    connected: state?.startsWith("100") ?? false,
+    connection: await field(device, "GENERAL.CONNECTION"),
+    address: await field(device, "IP4.ADDRESS")
+  };
+}
+
+async function connectivity(): Promise<NetworkStatus["connectivity"]> {
+  try {
+    const value = await nmcli(["networking", "connectivity", "check"], 10_000);
+    return ["full", "limited", "portal", "none", "unknown"].includes(value)
+      ? (value as NetworkStatus["connectivity"])
+      : "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
+class NetworkController {
+  private mode: NetworkMode = "starting";
+  private message = "Inspecting network adapters.";
+  private lastTransitionAt = new Date().toISOString();
+  private lastFailure: string | null = null;
+  private probeDeadline: string | null = null;
+  private fallbackAttempted = false;
+  private nextAwusAttemptAt = 0;
+  private stopped = false;
+
+  constructor(private readonly config: ControllerConfig) {}
+
+  stop(): void {
+    this.stopped = true;
+  }
+
+  private transition(mode: NetworkMode, message: string, failure: string | null = null): void {
+    if (this.mode !== mode) this.lastTransitionAt = new Date().toISOString();
+    this.mode = mode;
+    this.message = message;
+    this.lastFailure = failure;
+  }
+
+  private async publish(): Promise<{ hotspot: Adapter; upstream: Adapter }> {
+    const [hotspot, upstream, internet] = await Promise.all([
+      inspectAdapter(this.config.NETWORK_HOTSPOT_INTERFACE),
+      inspectAdapter(this.config.NETWORK_UPSTREAM_INTERFACE),
+      connectivity()
+    ]);
+    await writeNetworkStatus(this.config.NETWORK_STATUS_PATH, {
+      enabled: true,
+      mode: this.mode,
+      message: this.message,
+      updatedAt: new Date().toISOString(),
+      lastTransitionAt: this.lastTransitionAt,
+      probeDeadline: this.probeDeadline,
+      lastFailure: this.lastFailure,
+      connectivity: internet,
+      hotspot,
+      upstream
+    });
+    return { hotspot, upstream };
+  }
+
+  private async connectUpstreamAdapter(): Promise<boolean> {
+    try {
+      await nmcli(["--wait", "20", "device", "connect", this.config.NETWORK_UPSTREAM_INTERFACE], 25_000);
+      await sleep(this.config.NETWORK_STABILITY_SECONDS * 1000);
+      return (await inspectAdapter(this.config.NETWORK_UPSTREAM_INTERFACE)).connected;
+    } catch (error) {
+      this.lastFailure = error instanceof Error ? error.message.slice(0, 500) : "The upstream adapter could not connect.";
+      return false;
+    }
+  }
+
+  private async restoreHotspot(message = "Offline hotspot restored; the dashboard remains available without internet."): Promise<boolean> {
+    this.probeDeadline = null;
+    try {
+      await nmcli(
+        ["--wait", "20", "connection", "up", this.config.NETWORK_HOTSPOT_CONNECTION, "ifname", this.config.NETWORK_HOTSPOT_INTERFACE],
+        25_000
+      );
+      const upstream = await inspectAdapter(this.config.NETWORK_UPSTREAM_INTERFACE);
+      this.transition(upstream.connected ? "dual_radio" : "offline_hotspot", upstream.connected ? "Car hotspot is active with AWUS upstream internet." : message);
+      return true;
+    } catch (error) {
+      const failure = error instanceof Error ? error.message.slice(0, 500) : "The hotspot could not be restored.";
+      this.transition("error", "The recovery hotspot could not be activated. Local intervention is required.", failure);
+      return false;
+    }
+  }
+
+  private async probeBuiltIn(): Promise<void> {
+    const timeout = this.config.NETWORK_PROBE_TIMEOUT_SECONDS;
+    this.probeDeadline = new Date(Date.now() + timeout * 1000).toISOString();
+    this.transition("probing_built_in", `Trying saved upstream networks on the built-in adapter for up to ${timeout} seconds.`);
+    await this.publish();
+    try {
+      await nmcli(["--wait", "10", "connection", "down", this.config.NETWORK_HOTSPOT_CONNECTION], 15_000).catch(() => undefined);
+      await nmcli(["--wait", String(timeout), "device", "connect", this.config.NETWORK_HOTSPOT_INTERFACE], (timeout + 5) * 1000);
+      if ((await inspectAdapter(this.config.NETWORK_HOTSPOT_INTERFACE)).connected) {
+        this.probeDeadline = null;
+        this.transition("built_in_upstream", "The built-in adapter is connected upstream; the car hotspot is temporarily unavailable.");
+        this.fallbackAttempted = false;
+        return;
+      }
+      throw new Error("The built-in adapter did not activate a saved upstream profile.");
+    } catch (error) {
+      this.lastFailure = error instanceof Error ? error.message.slice(0, 500) : "No saved upstream network was available.";
+      this.fallbackAttempted = true;
+      await this.restoreHotspot();
+    }
+  }
+
+  private async handleAction(action: NetworkAction): Promise<void> {
+    if (action === "restore-hotspot") {
+      this.fallbackAttempted = true;
+      await this.restoreHotspot("The offline hotspot was restored manually.");
+      return;
+    }
+    this.fallbackAttempted = false;
+    const upstream = await inspectAdapter(this.config.NETWORK_UPSTREAM_INTERFACE);
+    if (upstream.available && (upstream.connected || (await this.connectUpstreamAdapter()))) {
+      await this.restoreHotspot();
+      return;
+    }
+    await this.probeBuiltIn();
+  }
+
+  async run(): Promise<void> {
+    try {
+      const prior = networkStatusSchema.parse(JSON.parse(await readFile(this.config.NETWORK_STATUS_PATH, "utf8")));
+      this.mode = prior.mode;
+      this.message = prior.message;
+      this.lastTransitionAt = prior.lastTransitionAt;
+      this.lastFailure = prior.lastFailure;
+      this.fallbackAttempted = prior.mode === "offline_hotspot";
+    } catch {
+      // A first start intentionally discovers the live NetworkManager state.
+    }
+    let previous = await this.publish();
+
+    while (!this.stopped) {
+      const command = await consumeNetworkCommand(this.config.NETWORK_COMMAND_PATH);
+      if (command) await this.handleAction(command.action);
+
+      let { hotspot, upstream } = await this.publish();
+      const hotspotActive = hotspot.connected && hotspot.connection === this.config.NETWORK_HOTSPOT_CONNECTION;
+      const builtInUpstream = hotspot.connected && !hotspotActive;
+
+      if (upstream.connected) {
+        if (!hotspotActive) await this.restoreHotspot();
+        else this.transition("dual_radio", "Car hotspot is active with AWUS upstream internet.");
+        this.fallbackAttempted = false;
+      } else if (builtInUpstream) {
+        this.transition("built_in_upstream", "The built-in adapter is connected upstream; the car hotspot is temporarily unavailable.");
+        if (upstream.available && Date.now() >= this.nextAwusAttemptAt) {
+          this.nextAwusAttemptAt = Date.now() + 60_000;
+          if (await this.connectUpstreamAdapter()) await this.restoreHotspot();
+        }
+      } else {
+        const awusJustFailed = previous.upstream.connected && !upstream.connected;
+        if (upstream.available && Date.now() >= this.nextAwusAttemptAt) {
+          this.nextAwusAttemptAt = Date.now() + 60_000;
+          if (await this.connectUpstreamAdapter()) {
+            await this.restoreHotspot();
+            ({ hotspot, upstream } = await this.publish());
+          }
+        }
+        if (!upstream.connected && !this.fallbackAttempted && (awusJustFailed || hotspotActive || !hotspot.connected)) {
+          await this.probeBuiltIn();
+        } else if (!upstream.connected && hotspotActive) {
+          this.transition("offline_hotspot", "Offline hotspot is active; no upstream network is currently connected.", this.lastFailure);
+        }
+      }
+
+      previous = await this.publish();
+      await sleep(this.config.NETWORK_POLL_SECONDS * 1000);
+    }
+  }
+}
+
+const config = controllerConfigSchema.parse(process.env);
+if (!config.NETWORK_MANAGER_ENABLED) {
+  process.exit(0);
+}
+const controller = new NetworkController(config);
+process.on("SIGINT", () => controller.stop());
+process.on("SIGTERM", () => controller.stop());
+await controller.run();

server/src/network.ts

@@ -0,0 +1,116 @@
+import { randomUUID } from "node:crypto";
+import { chmod, readFile, rename, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { z } from "zod";
+import type { AppConfig } from "./config.js";
+
+export const networkModeSchema = z.enum([
+  "disabled",
+  "starting",
+  "dual_radio",
+  "built_in_upstream",
+  "probing_built_in",
+  "offline_hotspot",
+  "degraded",
+  "error"
+]);
+export type NetworkMode = z.infer<typeof networkModeSchema>;
+
+const adapterSchema = z.object({
+  interface: z.string().max(64),
+  available: z.boolean(),
+  connected: z.boolean(),
+  connection: z.string().max(128).nullable(),
+  address: z.string().max(128).nullable()
+});
+
+export const networkStatusSchema = z.object({
+  enabled: z.boolean(),
+  mode: networkModeSchema,
+  message: z.string().max(500),
+  updatedAt: z.string(),
+  lastTransitionAt: z.string(),
+  probeDeadline: z.string().nullable(),
+  lastFailure: z.string().max(500).nullable(),
+  connectivity: z.enum(["full", "limited", "portal", "none", "unknown"]),
+  hotspot: adapterSchema,
+  upstream: adapterSchema
+});
+
+export type NetworkStatus = z.infer<typeof networkStatusSchema> & { supported: boolean };
+export const networkActionSchema = z.enum(["retry-upstream", "restore-hotspot"]);
+export type NetworkAction = z.infer<typeof networkActionSchema>;
+
+const commandSchema = z.object({
+  id: z.string().uuid(),
+  action: networkActionSchema,
+  requestedAt: z.string(),
+  actorUserId: z.number().int().positive()
+});
+export type NetworkCommand = z.infer<typeof commandSchema>;
+
+async function atomicJsonWrite(path: string, value: unknown): Promise<void> {
+  const temporary = join(dirname(path), `.${randomUUID()}.tmp`);
+  await writeFile(temporary, `${JSON.stringify(value)}\n`, { mode: 0o640 });
+  await chmod(temporary, 0o640);
+  await rename(temporary, path);
+}
+
+export async function readNetworkStatus(config: AppConfig): Promise<NetworkStatus> {
+  if (!config.network.enabled) {
+    return unsupportedStatus("Automatic network management is disabled until its hotspot and upstream profiles are configured.");
+  }
+  try {
+    return { ...networkStatusSchema.parse(JSON.parse(await readFile(config.network.statusPath, "utf8"))), supported: true };
+  } catch {
+    return {
+      ...unsupportedStatus("The network controller is starting or its status is unavailable."),
+      enabled: true,
+      mode: "starting"
+    };
+  }
+}
+
+export async function requestNetworkAction(
+  config: AppConfig,
+  action: NetworkAction,
+  actorUserId: number
+): Promise<NetworkCommand> {
+  if (!config.network.enabled) throw new Error("NETWORK_MANAGER_DISABLED");
+  const command = commandSchema.parse({ id: randomUUID(), action, requestedAt: new Date().toISOString(), actorUserId });
+  await atomicJsonWrite(config.network.commandPath, command);
+  return command;
+}
+
+export async function consumeNetworkCommand(path: string): Promise<NetworkCommand | null> {
+  try {
+    const parsed = commandSchema.parse(JSON.parse(await readFile(path, "utf8")));
+    await writeFile(path, "", { mode: 0o640 });
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+function unsupportedStatus(message: string): NetworkStatus {
+  const epoch = new Date(0).toISOString();
+  const adapter = { interface: "unconfigured", available: false, connected: false, connection: null, address: null };
+  return {
+    supported: false,
+    enabled: false,
+    mode: "disabled",
+    message,
+    updatedAt: epoch,
+    lastTransitionAt: epoch,
+    probeDeadline: null,
+    lastFailure: null,
+    connectivity: "unknown",
+    hotspot: adapter,
+    upstream: adapter
+  };
+}
+
+export async function writeNetworkStatus(path: string, status: z.infer<typeof networkStatusSchema>): Promise<void> {
+  await atomicJsonWrite(path, networkStatusSchema.parse(status));
+  await chmod(path, 0o644);
+}

server/tests/app.test.ts

@@ -17,6 +17,11 @@ function testConfig(): AppConfig {
     allowedOrigins: new Set(["http://mg4pi.local:8787"]),
     updateStatusPath: `/tmp/pi-car-companion-test-${process.pid}-missing-update.json`,
     versionFile: `/tmp/pi-car-companion-test-${process.pid}-missing-revision`,
+    network: {
+      enabled: false,
+      statusPath: `/tmp/pi-car-companion-test-${process.pid}-missing-network-status.json`,
+      commandPath: `/tmp/pi-car-companion-test-${process.pid}-network-command.json`
+    },
     adb: { enabled: false, executable: "adb", serial: null, timeoutMs: 3_000 }
   };
 }
@@ -180,6 +185,33 @@ describe("authentication boundary", () => {
     expect(response.statusCode).toBe(401);
   });
 
+  it("keeps network status and controls behind authentication and explicit configuration", async () => {
+    const app = await createApp();
+    expect((await app.inject({ method: "GET", url: "/api/network" })).statusCode).toBe(401);
+
+    const token = await csrf(app);
+    await setup(app, token);
+    const login = await app.inject({
+      method: "POST",
+      url: "/api/auth/login",
+      headers: mutationHeaders(token),
+      payload: { username: "owner", password: "Correct-horse1" }
+    });
+    const session = login.cookies.find((item) => item.name === SESSION_COOKIE)?.value ?? "";
+    const cookie = `${CSRF_COOKIE}=${token}; ${SESSION_COOKIE}=${session}`;
+    const status = await app.inject({ method: "GET", url: "/api/network", headers: { cookie } });
+    expect(status.statusCode).toBe(200);
+    expect(status.json()).toMatchObject({ supported: false, enabled: false, mode: "disabled" });
+
+    const action = await app.inject({
+      method: "POST",
+      url: "/api/network/actions",
+      headers: { ...mutationHeaders(token), cookie },
+      payload: { action: "retry-upstream" }
+    });
+    expect(action.statusCode).toBe(503);
+  });
+
   it("exposes only allowlisted jobs and authenticated run history", async () => {
     const app = await createApp();
     const token = await csrf(app);

server/tests/config.test.ts

@@ -21,4 +21,12 @@ describe("ADB configuration", () => {
       serial: "10.0.0.10:5555"
     });
   });
+
+  it("keeps automatic network management disabled by default", () => {
+    expect(loadConfig({ NODE_ENV: "test" }).network).toEqual({
+      enabled: false,
+      statusPath: "/var/lib/pi-car-companion/network-status.json",
+      commandPath: "/var/lib/pi-car-companion/network-command.json"
+    });
+  });
 });

server/tests/network.test.ts

@@ -0,0 +1,52 @@
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import type { AppConfig } from "../src/config.js";
+import { consumeNetworkCommand, readNetworkStatus, requestNetworkAction } from "../src/network.js";
+
+const directories: string[] = [];
+
+function config(directory: string, enabled = true): AppConfig {
+  return {
+    nodeEnv: "test",
+    host: "127.0.0.1",
+    port: 8787,
+    databasePath: ":memory:",
+    sessionTtlMs: 60_000,
+    cookieSecure: false,
+    allowedOrigins: new Set(),
+    updateStatusPath: join(directory, "update.json"),
+    versionFile: join(directory, "REVISION"),
+    network: { enabled, statusPath: join(directory, "status.json"), commandPath: join(directory, "command.json") },
+    adb: { enabled: false, executable: "adb", serial: null, timeoutMs: 3_000 }
+  };
+}
+
+afterEach(async () => {
+  await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })));
+});
+
+describe("network control boundary", () => {
+  it("writes and consumes only fixed action requests", async () => {
+    const directory = await mkdtemp(join(tmpdir(), "pi-car-network-test-"));
+    directories.push(directory);
+    const testConfig = config(directory);
+    const requested = await requestNetworkAction(testConfig, "retry-upstream", 7);
+    expect(JSON.parse(await readFile(testConfig.network.commandPath, "utf8"))).toMatchObject({
+      id: requested.id,
+      action: "retry-upstream",
+      actorUserId: 7
+    });
+    await expect(consumeNetworkCommand(testConfig.network.commandPath)).resolves.toMatchObject({ action: "retry-upstream" });
+    await expect(consumeNetworkCommand(testConfig.network.commandPath)).resolves.toBeNull();
+  });
+
+  it("rejects malformed controller status", async () => {
+    const directory = await mkdtemp(join(tmpdir(), "pi-car-network-test-"));
+    directories.push(directory);
+    const testConfig = config(directory);
+    await writeFile(testConfig.network.statusPath, '{"mode":"shell"}');
+    await expect(readNetworkStatus(testConfig)).resolves.toMatchObject({ supported: false, mode: "starting" });
+  });
+});

server/tests/update.test.ts

@@ -18,6 +18,11 @@ function config(updateStatusPath: string, versionFile: string): AppConfig {
     allowedOrigins: new Set(),
     updateStatusPath,
     versionFile,
+    network: {
+      enabled: false,
+      statusPath: join(tmpdir(), "missing-network-status.json"),
+      commandPath: join(tmpdir(), "network-command.json")
+    },
     adb: { enabled: false, executable: "adb", serial: null, timeoutMs: 3_000 }
   };
 }