feat(systemd): add shutdown/boot hang diagnostics module

Shutdown stalls are hard to debug because journald is torn down mid-way,
so late hangs never reach the journal. `modules/shutdown-debug.nix`
exposes `local.shutdownDebug.*`: verbose systemd logging routed to kmsg
(recoverable from pstore after a forced power-off), an opt-in tty9 debug
shell, and a shortened `DefaultTimeoutStopSec` preset (system + user) so
a hung unit is killed and logged in seconds instead of spinning at 90s.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

README.md

@@ -71,3 +71,7 @@ hosts/example/       # the template host (placeholder hardware-configuration.nix
 ```
 
 Shared `*.nix` / `*.lock` live at the top level; per-host files under `hosts/<host>/`.
+
+Chasing a shutdown/boot hang? `modules/shutdown-debug.nix` exposes `local.shutdownDebug.*`
+(verbose systemd logging to kmsg, a tty9 debug shell, a shortened stop-timeout) — see its
+header comment for how to read the evidence.

modules/nixos.nix

@@ -5,6 +5,8 @@
 { config, lib, pkgs, ... }:
 
 {
+  imports = [ ./shutdown-debug.nix ];
+
   nix.settings.experimental-features = [ "nix-command" "flakes" ];
   # Deduplicate identical store paths via hardlinks.
   nix.settings.auto-optimise-store = true;

modules/shutdown-debug.nix

@@ -0,0 +1,92 @@
+# Shutdown/boot hang diagnostics.
+#
+# Shutdown stalls are awkward to debug because journald is torn down mid-way,
+# so anything that wedges *after* it dies never reaches /var/log/journal — the
+# tail just stops with no explanation. These knobs make the next hang leave a
+# trail instead. All are opt-in except the shortened stop-timeout, which is a
+# baseline preset (overridable / disable with null).
+#
+# Reading the evidence after a hang:
+#   - On screen while it hangs: watch for "A stop job is running for <unit>
+#     (Xs / 1min 30s)". That line names the culprit; note whether its counter
+#     advances (named unit = easy) or the screen is frozen with no counter
+#     (late systemd-shutdown phase = needs pstore, below).
+#   - After a *clean* next boot: `journalctl -b -1 -e` shows the previous
+#     boot's shutdown tail (NixOS journald is persistent by default).
+#   - After a *forced* power-off: journald never flushed, so check the kernel
+#     ring buffer preserved across reboot in `/sys/fs/pstore/` (needs pstore
+#     backing — ACPI ERST on most laptops, else ramoops). `verboseLogging`
+#     below routes systemd's own messages to kmsg so they land there too.
+#   - With `debugShell` on: switch to tty9 (Ctrl+Alt+F9) while hung and run
+#     `systemctl list-jobs` to see which job is still pending.
+
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.local.shutdownDebug;
+in
+{
+  options.local.shutdownDebug = {
+    stopTimeout = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = "20s";
+      example = null;
+      description = ''
+        DefaultTimeoutStopSec for both the system and per-user managers. The
+        default shortens systemd's 90s so a hung unit is killed — and logged as
+        `timed out` — in seconds rather than spinning at a black screen. Set to
+        null to leave systemd's default untouched. Any systemd time span, e.g.
+        "20s", "45s", "1min".
+      '';
+    };
+
+    verboseLogging = lib.mkEnableOption ''
+      verbose systemd shutdown logging. Adds the kernel params
+      `systemd.log_level=debug systemd.log_target=kmsg`: bumps systemd to debug
+      verbosity and routes its messages into the kernel ring buffer, which
+      outlives journald and can be recovered from /sys/fs/pstore after a forced
+      power-off. Noisy on every boot and mildly slows it — enable only while
+      actively hunting a hang'';
+
+    debugShell = lib.mkEnableOption ''
+      an unauthenticated root shell on tty9 (Ctrl+Alt+F9), available through
+      shutdown, for running `systemctl list-jobs` while the system is wedged.
+      SECURITY: anyone with physical console access gets root — leave off
+      except while investigating'';
+  };
+
+  config = lib.mkMerge [
+    (lib.mkIf (cfg.stopTimeout != null) {
+      systemd.settings.Manager.DefaultTimeoutStopSec = cfg.stopTimeout;
+      systemd.user.extraConfig = "DefaultTimeoutStopSec=${cfg.stopTimeout}";
+    })
+
+    (lib.mkIf cfg.verboseLogging {
+      boot.kernelParams = [ "systemd.log_level=debug" "systemd.log_target=kmsg" ];
+    })
+
+    # Uniquely named rather than enabling the upstream (static) debug-shell.service,
+    # whose enablement via systemd.services would generate a shadowing unit.
+    (lib.mkIf cfg.debugShell {
+      systemd.services.shutdown-debug-shell = {
+        description = "Root shell on tty9 for boot/shutdown debugging";
+        unitConfig = {
+          DefaultDependencies = false;
+          ConditionPathExists = "/dev/tty9";
+        };
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig = {
+          Environment = "TERM=linux";
+          ExecStart = "${pkgs.bashInteractive}/bin/bash";
+          Restart = "always";
+          RestartSec = 0;
+          StandardInput = "tty";
+          TTYPath = "/dev/tty9";
+          TTYReset = true;
+          TTYVHangup = true;
+          KillMode = "process";
+        };
+      };
+    })
+  ];
+}