feat(systemd): add shutdown/boot hang diagnostics module

Shutdown stalls are hard to debug because journald is torn down mid-way, so late hangs never reach the journal. `modules/shutdown-debug.nix` exposes `local.shutdownDebug.*`: verbose systemd logging routed to kmsg (recoverable from pstore after a forced power-off), an opt-in tty9 debug shell, and a shortened `DefaultTimeoutStopSec` preset (system + user) so a hung unit is killed and logged in seconds instead of spinning at 90s. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-12 14:47:30 +02:00
parent 98eb0f87ba
commit de6c3993cc
3 changed files with 98 additions and 0 deletions
@@ -0,0 +1,92 @@
+# Shutdown/boot hang diagnostics.
+#
+# Shutdown stalls are awkward to debug because journald is torn down mid-way,
+# so anything that wedges *after* it dies never reaches /var/log/journal — the
+# tail just stops with no explanation. These knobs make the next hang leave a
+# trail instead. All are opt-in except the shortened stop-timeout, which is a
+# baseline preset (overridable / disable with null).
+#
+# Reading the evidence after a hang:
+#   - On screen while it hangs: watch for "A stop job is running for <unit>
+#     (Xs / 1min 30s)". That line names the culprit; note whether its counter
+#     advances (named unit = easy) or the screen is frozen with no counter
+#     (late systemd-shutdown phase = needs pstore, below).
+#   - After a *clean* next boot: `journalctl -b -1 -e` shows the previous
+#     boot's shutdown tail (NixOS journald is persistent by default).
+#   - After a *forced* power-off: journald never flushed, so check the kernel
+#     ring buffer preserved across reboot in `/sys/fs/pstore/` (needs pstore
+#     backing — ACPI ERST on most laptops, else ramoops). `verboseLogging`
+#     below routes systemd's own messages to kmsg so they land there too.
+#   - With `debugShell` on: switch to tty9 (Ctrl+Alt+F9) while hung and run
+#     `systemctl list-jobs` to see which job is still pending.
+
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.local.shutdownDebug;
+in
+{
+  options.local.shutdownDebug = {
+    stopTimeout = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = "20s";
+      example = null;
+      description = ''
+        DefaultTimeoutStopSec for both the system and per-user managers. The
+        default shortens systemd's 90s so a hung unit is killed — and logged as
+        `timed out` — in seconds rather than spinning at a black screen. Set to
+        null to leave systemd's default untouched. Any systemd time span, e.g.
+        "20s", "45s", "1min".
+      '';
+    };
+
+    verboseLogging = lib.mkEnableOption ''
+      verbose systemd shutdown logging. Adds the kernel params
+      `systemd.log_level=debug systemd.log_target=kmsg`: bumps systemd to debug
+      verbosity and routes its messages into the kernel ring buffer, which
+      outlives journald and can be recovered from /sys/fs/pstore after a forced
+      power-off. Noisy on every boot and mildly slows it — enable only while
+      actively hunting a hang'';
+
+    debugShell = lib.mkEnableOption ''
+      an unauthenticated root shell on tty9 (Ctrl+Alt+F9), available through
+      shutdown, for running `systemctl list-jobs` while the system is wedged.
+      SECURITY: anyone with physical console access gets root — leave off
+      except while investigating'';
+  };
+
+  config = lib.mkMerge [
+    (lib.mkIf (cfg.stopTimeout != null) {
+      systemd.settings.Manager.DefaultTimeoutStopSec = cfg.stopTimeout;
+      systemd.user.extraConfig = "DefaultTimeoutStopSec=${cfg.stopTimeout}";
+    })
+
+    (lib.mkIf cfg.verboseLogging {
+      boot.kernelParams = [ "systemd.log_level=debug" "systemd.log_target=kmsg" ];
+    })
+
+    # Uniquely named rather than enabling the upstream (static) debug-shell.service,
+    # whose enablement via systemd.services would generate a shadowing unit.
+    (lib.mkIf cfg.debugShell {
+      systemd.services.shutdown-debug-shell = {
+        description = "Root shell on tty9 for boot/shutdown debugging";
+        unitConfig = {
+          DefaultDependencies = false;
+          ConditionPathExists = "/dev/tty9";
+        };
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig = {
+          Environment = "TERM=linux";
+          ExecStart = "${pkgs.bashInteractive}/bin/bash";
+          Restart = "always";
+          RestartSec = 0;
+          StandardInput = "tty";
+          TTYPath = "/dev/tty9";
+          TTYReset = true;
+          TTYVHangup = true;
+          KillMode = "process";
+        };
+      };
+    })
+  ];
+}