diff --git a/README.md b/README.md index 4fd8f91..a6dd693 100644 --- a/README.md +++ b/README.md @@ -71,3 +71,7 @@ hosts/example/ # the template host (placeholder hardware-configuration.nix ``` Shared `*.nix` / `*.lock` live at the top level; per-host files under `hosts//`. + +Chasing a shutdown/boot hang? `modules/shutdown-debug.nix` exposes `local.shutdownDebug.*` +(verbose systemd logging to kmsg, a tty9 debug shell, a shortened stop-timeout) — see its +header comment for how to read the evidence. diff --git a/modules/nixos.nix b/modules/nixos.nix index 0616f1d..4b7f4d9 100644 --- a/modules/nixos.nix +++ b/modules/nixos.nix @@ -5,6 +5,8 @@ { config, lib, pkgs, ... }: { + imports = [ ./shutdown-debug.nix ]; + nix.settings.experimental-features = [ "nix-command" "flakes" ]; # Deduplicate identical store paths via hardlinks. nix.settings.auto-optimise-store = true; diff --git a/modules/shutdown-debug.nix b/modules/shutdown-debug.nix new file mode 100644 index 0000000..d25d5e1 --- /dev/null +++ b/modules/shutdown-debug.nix @@ -0,0 +1,92 @@ +# Shutdown/boot hang diagnostics. +# +# Shutdown stalls are awkward to debug because journald is torn down mid-way, +# so anything that wedges *after* it dies never reaches /var/log/journal — the +# tail just stops with no explanation. These knobs make the next hang leave a +# trail instead. All are opt-in except the shortened stop-timeout, which is a +# baseline preset (overridable / disable with null). +# +# Reading the evidence after a hang: +# - On screen while it hangs: watch for "A stop job is running for +# (Xs / 1min 30s)". That line names the culprit; note whether its counter +# advances (named unit = easy) or the screen is frozen with no counter +# (late systemd-shutdown phase = needs pstore, below). +# - After a *clean* next boot: `journalctl -b -1 -e` shows the previous +# boot's shutdown tail (NixOS journald is persistent by default). +# - After a *forced* power-off: journald never flushed, so check the kernel +# ring buffer preserved across reboot in `/sys/fs/pstore/` (needs pstore +# backing — ACPI ERST on most laptops, else ramoops). `verboseLogging` +# below routes systemd's own messages to kmsg so they land there too. +# - With `debugShell` on: switch to tty9 (Ctrl+Alt+F9) while hung and run +# `systemctl list-jobs` to see which job is still pending. + +{ config, lib, pkgs, ... }: + +let + cfg = config.local.shutdownDebug; +in +{ + options.local.shutdownDebug = { + stopTimeout = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = "20s"; + example = null; + description = '' + DefaultTimeoutStopSec for both the system and per-user managers. The + default shortens systemd's 90s so a hung unit is killed — and logged as + `timed out` — in seconds rather than spinning at a black screen. Set to + null to leave systemd's default untouched. Any systemd time span, e.g. + "20s", "45s", "1min". + ''; + }; + + verboseLogging = lib.mkEnableOption '' + verbose systemd shutdown logging. Adds the kernel params + `systemd.log_level=debug systemd.log_target=kmsg`: bumps systemd to debug + verbosity and routes its messages into the kernel ring buffer, which + outlives journald and can be recovered from /sys/fs/pstore after a forced + power-off. Noisy on every boot and mildly slows it — enable only while + actively hunting a hang''; + + debugShell = lib.mkEnableOption '' + an unauthenticated root shell on tty9 (Ctrl+Alt+F9), available through + shutdown, for running `systemctl list-jobs` while the system is wedged. + SECURITY: anyone with physical console access gets root — leave off + except while investigating''; + }; + + config = lib.mkMerge [ + (lib.mkIf (cfg.stopTimeout != null) { + systemd.settings.Manager.DefaultTimeoutStopSec = cfg.stopTimeout; + systemd.user.extraConfig = "DefaultTimeoutStopSec=${cfg.stopTimeout}"; + }) + + (lib.mkIf cfg.verboseLogging { + boot.kernelParams = [ "systemd.log_level=debug" "systemd.log_target=kmsg" ]; + }) + + # Uniquely named rather than enabling the upstream (static) debug-shell.service, + # whose enablement via systemd.services would generate a shadowing unit. + (lib.mkIf cfg.debugShell { + systemd.services.shutdown-debug-shell = { + description = "Root shell on tty9 for boot/shutdown debugging"; + unitConfig = { + DefaultDependencies = false; + ConditionPathExists = "/dev/tty9"; + }; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Environment = "TERM=linux"; + ExecStart = "${pkgs.bashInteractive}/bin/bash"; + Restart = "always"; + RestartSec = 0; + StandardInput = "tty"; + TTYPath = "/dev/tty9"; + TTYReset = true; + TTYVHangup = true; + KillMode = "process"; + }; + }; + }) + ]; +}