daniil-berg
de6c3993cc
Shutdown stalls are hard to debug because journald is torn down mid-way, so late hangs never reach the journal. `modules/shutdown-debug.nix` exposes `local.shutdownDebug.*`: verbose systemd logging routed to kmsg (recoverable from pstore after a forced power-off), an opt-in tty9 debug shell, and a shortened `DefaultTimeoutStopSec` preset (system + user) so a hung unit is killed and logged in seconds instead of spinning at 90s. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
93 lines
3.8 KiB
Nix
93 lines
3.8 KiB
Nix
# Shutdown/boot hang diagnostics.
|
|
#
|
|
# Shutdown stalls are awkward to debug because journald is torn down mid-way,
|
|
# so anything that wedges *after* it dies never reaches /var/log/journal — the
|
|
# tail just stops with no explanation. These knobs make the next hang leave a
|
|
# trail instead. All are opt-in except the shortened stop-timeout, which is a
|
|
# baseline preset (overridable / disable with null).
|
|
#
|
|
# Reading the evidence after a hang:
|
|
# - On screen while it hangs: watch for "A stop job is running for <unit>
|
|
# (Xs / 1min 30s)". That line names the culprit; note whether its counter
|
|
# advances (named unit = easy) or the screen is frozen with no counter
|
|
# (late systemd-shutdown phase = needs pstore, below).
|
|
# - After a *clean* next boot: `journalctl -b -1 -e` shows the previous
|
|
# boot's shutdown tail (NixOS journald is persistent by default).
|
|
# - After a *forced* power-off: journald never flushed, so check the kernel
|
|
# ring buffer preserved across reboot in `/sys/fs/pstore/` (needs pstore
|
|
# backing — ACPI ERST on most laptops, else ramoops). `verboseLogging`
|
|
# below routes systemd's own messages to kmsg so they land there too.
|
|
# - With `debugShell` on: switch to tty9 (Ctrl+Alt+F9) while hung and run
|
|
# `systemctl list-jobs` to see which job is still pending.
|
|
|
|
{ config, lib, pkgs, ... }:
|
|
|
|
let
|
|
cfg = config.local.shutdownDebug;
|
|
in
|
|
{
|
|
options.local.shutdownDebug = {
|
|
stopTimeout = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = "20s";
|
|
example = null;
|
|
description = ''
|
|
DefaultTimeoutStopSec for both the system and per-user managers. The
|
|
default shortens systemd's 90s so a hung unit is killed — and logged as
|
|
`timed out` — in seconds rather than spinning at a black screen. Set to
|
|
null to leave systemd's default untouched. Any systemd time span, e.g.
|
|
"20s", "45s", "1min".
|
|
'';
|
|
};
|
|
|
|
verboseLogging = lib.mkEnableOption ''
|
|
verbose systemd shutdown logging. Adds the kernel params
|
|
`systemd.log_level=debug systemd.log_target=kmsg`: bumps systemd to debug
|
|
verbosity and routes its messages into the kernel ring buffer, which
|
|
outlives journald and can be recovered from /sys/fs/pstore after a forced
|
|
power-off. Noisy on every boot and mildly slows it — enable only while
|
|
actively hunting a hang'';
|
|
|
|
debugShell = lib.mkEnableOption ''
|
|
an unauthenticated root shell on tty9 (Ctrl+Alt+F9), available through
|
|
shutdown, for running `systemctl list-jobs` while the system is wedged.
|
|
SECURITY: anyone with physical console access gets root — leave off
|
|
except while investigating'';
|
|
};
|
|
|
|
config = lib.mkMerge [
|
|
(lib.mkIf (cfg.stopTimeout != null) {
|
|
systemd.settings.Manager.DefaultTimeoutStopSec = cfg.stopTimeout;
|
|
systemd.user.extraConfig = "DefaultTimeoutStopSec=${cfg.stopTimeout}";
|
|
})
|
|
|
|
(lib.mkIf cfg.verboseLogging {
|
|
boot.kernelParams = [ "systemd.log_level=debug" "systemd.log_target=kmsg" ];
|
|
})
|
|
|
|
# Uniquely named rather than enabling the upstream (static) debug-shell.service,
|
|
# whose enablement via systemd.services would generate a shadowing unit.
|
|
(lib.mkIf cfg.debugShell {
|
|
systemd.services.shutdown-debug-shell = {
|
|
description = "Root shell on tty9 for boot/shutdown debugging";
|
|
unitConfig = {
|
|
DefaultDependencies = false;
|
|
ConditionPathExists = "/dev/tty9";
|
|
};
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = {
|
|
Environment = "TERM=linux";
|
|
ExecStart = "${pkgs.bashInteractive}/bin/bash";
|
|
Restart = "always";
|
|
RestartSec = 0;
|
|
StandardInput = "tty";
|
|
TTYPath = "/dev/tty9";
|
|
TTYReset = true;
|
|
TTYVHangup = true;
|
|
KillMode = "process";
|
|
};
|
|
};
|
|
})
|
|
];
|
|
}
|