ringofstorms/dotfiles
ringofstorms/dotfiles
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Persist OpenBao secrets in /var/lib and make readiness non-blocking

Browse source
This commit is contained in:
Joshua Bell 2026-01-27 17:28:39 -06:00
parent 8b54a94c54
commit 15fccd2ff4
6 changed files with 123 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

4
flakes/secrets-bao/flake.nix
View file

@ -42,7 +42,7 @@
fragments = builtins.attrValues (builtins.mapAttrs ( fragments = builtins.attrValues (builtins.mapAttrs (
name: s: name: s:
let let
secretPath = s.path or ("/run/secrets/" + name); secretPath = s.path or ("/var/lib/openbao-secrets/" + name);
in in
substitute secretPath (s.configChanges or { }) substitute secretPath (s.configChanges or { })
) secrets); ) secrets);
@ -85,7 +85,7 @@
fragments = builtins.attrValues (builtins.mapAttrs ( fragments = builtins.attrValues (builtins.mapAttrs (
name: s: name: s:
let let
secretPath = s.path or ("/run/secrets/" + name); secretPath = s.path or ("/var/lib/openbao-secrets/" + name);
in in
substitute secretPath (s.hmChanges or { }) substitute secretPath (s.hmChanges or { })
) secrets); ) secrets);

196
flakes/secrets-bao/nixos-module.nix
View file

@ -318,6 +318,12 @@ in
options.ringofstorms.secretsBao = { options.ringofstorms.secretsBao = {
enable = lib.mkEnableOption "Fetch runtime secrets via OpenBao"; enable = lib.mkEnableOption "Fetch runtime secrets via OpenBao";
secretsBasePath = lib.mkOption {
type = lib.types.str;
default = "/var/lib/openbao-secrets";
description = "Base directory for rendered secrets. Use /var/lib/openbao-secrets for persistence across reboots, or /run/secrets for ephemeral.";
};
zitadelKeyPath = lib.mkOption { zitadelKeyPath = lib.mkOption {
type = lib.types.str; type = lib.types.str;
default = "/machine-key.json"; default = "/machine-key.json";
@ -404,7 +410,7 @@ in
options = { options = {
path = lib.mkOption { path = lib.mkOption {
type = lib.types.str; type = lib.types.str;
default = "/run/secrets/${name}"; default = "${cfg.secretsBasePath}/${name}";
}; };
owner = lib.mkOption { owner = lib.mkOption {
@ -488,30 +494,35 @@ in
sec sec
]; ];
systemd.tmpfiles.rules = # Persistent secrets directory - survives reboots
[ systemd.tmpfiles.rules =
"d /run/openbao 0700 root root - -" [
"f /run/openbao/zitadel.jwt 0400 root root - -" "d /run/openbao 0700 root root - -"
"d /run/secrets 0711 root root - -" "f /run/openbao/zitadel.jwt 0400 root root - -"
] # Create base secrets directory (persistent by default)
# Create empty placeholder files for all secret destinations so "d ${cfg.secretsBasePath} 0711 root root - -"
# services that reference env files don't fail when offline. ]
++ (lib.unique ( # Create empty placeholder files for all secret destinations so
lib.concatLists ( # services that reference env files don't fail when offline.
lib.mapAttrsToList ( # Important: we do NOT recreate files that already have content (the '-' at end)
_: secret: ++ (lib.unique (
let lib.concatLists (
dir = builtins.dirOf secret.path; lib.mapAttrsToList (
in _: secret:
# Ensure the parent dir exists if a custom path is used. let
[ "d ${dir} 0755 root root - -" ] dir = builtins.dirOf secret.path;
) cfg.secrets in
) # Ensure the parent dir exists if a custom path is used.
)) [ "d ${dir} 0755 root root - -" ]
++ (lib.mapAttrsToList ( ) cfg.secrets
_: secret: )
"f ${secret.path} ${secret.mode} ${secret.owner} ${secret.group} - -" ))
) cfg.secrets); # Only create placeholder if file doesn't exist (preserves persisted secrets)
# Using 'f' with '-' for argument means create if not exists, don't truncate if exists
++ (lib.mapAttrsToList (
_: secret:
"f ${secret.path} ${secret.mode} ${secret.owner} ${secret.group} - -"
) cfg.secrets);
systemd.paths = systemd.paths =
@ -542,17 +553,17 @@ in
}; };
}; };
openbao-secrets-ready = { openbao-secrets-ready = {
description = "Re-check OpenBao secrets readiness"; description = "Re-check OpenBao secrets readiness";
wantedBy = [ "multi-user.target" ]; wantedBy = [ "multi-user.target" ];
pathConfig = { pathConfig = {
PathChanged = "/run/secrets"; PathChanged = cfg.secretsBasePath;
Unit = "openbao-secrets-ready.service"; Unit = "openbao-secrets-ready.service";
TriggerLimitIntervalSec = 30; TriggerLimitIntervalSec = 30;
TriggerLimitBurst = 3; TriggerLimitBurst = 3;
}; };
}; };
}; };
systemd.timers.zitadel-mint-jwt = { systemd.timers.zitadel-mint-jwt = {
@ -586,35 +597,43 @@ in
) )
) )
{ {
openbao-secrets-ready = { openbao-secrets-ready = {
description = "OpenBao: all configured secrets present"; description = "OpenBao: all configured secrets present (informational)";
wantedBy = [ "multi-user.target" ]; # NOT in wantedBy - this is a passive check, not a startup requirement
wants = [ "vault-agent.service" ]; # It gets triggered by path units when secrets change
after = [ "vault-agent.service" ]; wants = [ "vault-agent.service" ];
after = [ "vault-agent.service" ];
serviceConfig = { serviceConfig = {
Type = "oneshot"; Type = "oneshot";
RemainAfterExit = true; RemainAfterExit = true;
User = "root"; User = "root";
Group = "root"; Group = "root";
UMask = "0077"; UMask = "0077";
ExecStart = pkgs.writeShellScript "openbao-secrets-ready" '' ExecStart = pkgs.writeShellScript "openbao-secrets-ready" ''
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -uo pipefail
${lib.concatStringsSep "\n" ( missing=0
lib.mapAttrsToList (name: secret: '' ${lib.concatStringsSep "\n" (
if [ ! -s ${lib.escapeShellArg secret.path} ]; then lib.mapAttrsToList (name: secret: ''
echo "Missing secret: ${secret.path}" >&2 if [ ! -s ${lib.escapeShellArg secret.path} ]; then
exit 1 echo "Missing secret: ${secret.path}" >&2
fi missing=1
'') cfg.secrets fi
)} '') cfg.secrets
)}
echo "All configured OpenBao secrets present." >&2 if [ "$missing" -eq 1 ]; then
''; echo "Some secrets are missing (may be stale or not yet fetched)" >&2
}; # Don't exit 1 - this is informational only
}; else
echo "All configured OpenBao secrets present." >&2
fi
exit 0
'';
};
};
openbao-jwt-changed = { openbao-jwt-changed = {
description = "Restart vault-agent after Zitadel JWT refresh"; description = "Restart vault-agent after Zitadel JWT refresh";
@ -637,6 +656,9 @@ in
zitadel-mint-jwt = { zitadel-mint-jwt = {
description = "Mint Zitadel access token (JWT) for OpenBao"; description = "Mint Zitadel access token (JWT) for OpenBao";
# Non-blocking: don't add to wantedBy, let timer handle it
# The timer is wantedBy timers.target, so this runs periodically
startLimitIntervalSec = 0; startLimitIntervalSec = 0;
after = [ after = [
@ -656,17 +678,15 @@ in
Type = "oneshot"; Type = "oneshot";
User = "root"; User = "root";
Group = "root"; Group = "root";
Restart = "on-failure"; # Don't restart on failure - timer will retry later
RestartSec = "30s"; # This prevents blocking activation if Zitadel is down
TimeoutStartSec = "2min"; TimeoutStartSec = "2min";
UMask = "0077"; UMask = "0077";
ExecStart = pkgs.writeShellScript "zitadel-mint-jwt-service" '' ExecStart = pkgs.writeShellScript "zitadel-mint-jwt-service" ''
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -uo pipefail
if [ ! -d "/run/openbao" ]; then if [ ! -d "/run/openbao" ]; then
${pkgs.coreutils}/bin/mkdir -p /run/openbao ${pkgs.coreutils}/bin/mkdir -p /run/openbao
@ -724,30 +744,46 @@ in
return 1 return 1
} }
# Check if existing token is still valid
if [ -s "${cfg.zitadelJwtPath}" ] && jwt_is_valid "$(cat "${cfg.zitadelJwtPath}")"; then if [ -s "${cfg.zitadelJwtPath}" ] && jwt_is_valid "$(cat "${cfg.zitadelJwtPath}")"; then
echo "zitadel-mint-jwt: existing token still valid; skipping" >&2 echo "zitadel-mint-jwt: existing token still valid; skipping" >&2
exit 0 exit 0
fi fi
jwt="$(${zitadelMintJwt}/bin/zitadel-mint-jwt)" # Try to mint a new token
jwt=""
if jwt="$(${zitadelMintJwt}/bin/zitadel-mint-jwt 2>&1)"; then
if [ -n "$jwt" ] && [ "$jwt" != "null" ]; then
tmp="$(${pkgs.coreutils}/bin/mktemp)"
trap '${pkgs.coreutils}/bin/rm -f "$tmp"' EXIT
${pkgs.coreutils}/bin/printf '%s' "$jwt" > "$tmp"
if [ -z "$jwt" ] || [ "$jwt" = "null" ]; then if [ -s "${cfg.zitadelJwtPath}" ] && ${pkgs.diffutils}/bin/cmp -s "$tmp" "${cfg.zitadelJwtPath}"; then
echo "Failed to mint Zitadel access token" >&2 echo "zitadel-mint-jwt: token unchanged; skipping" >&2
exit 1 exit 0
fi
# Update the token file (the agent watches it).
${pkgs.coreutils}/bin/cat "$tmp" > "${cfg.zitadelJwtPath}"
${pkgs.coreutils}/bin/chmod 0400 "${cfg.zitadelJwtPath}" || true
echo "zitadel-mint-jwt: successfully refreshed token" >&2
exit 0
fi
fi fi
tmp="$(${pkgs.coreutils}/bin/mktemp)" # Failed to mint new token - check if we have a stale but existing token
trap '${pkgs.coreutils}/bin/rm -f "$tmp"' EXIT if [ -s "${cfg.zitadelJwtPath}" ]; then
${pkgs.coreutils}/bin/printf '%s' "$jwt" > "$tmp" echo "zitadel-mint-jwt: failed to refresh, but existing token present (may be stale)" >&2
echo "zitadel-mint-jwt: continuing with stale token; will retry on next timer" >&2
if [ -s "${cfg.zitadelJwtPath}" ] && ${pkgs.diffutils}/bin/cmp -s "$tmp" "${cfg.zitadelJwtPath}"; then # Exit 0 so we don't block activation - timer will retry
echo "zitadel-mint-jwt: token unchanged; skipping" >&2
exit 0 exit 0
fi fi
# Update the token file (the agent watches it). # No existing token and failed to mint - this is a problem but don't block
${pkgs.coreutils}/bin/cat "$tmp" > "${cfg.zitadelJwtPath}" echo "zitadel-mint-jwt: failed to mint token and no existing token available" >&2
${pkgs.coreutils}/bin/chmod 0400 "${cfg.zitadelJwtPath}" || true echo "zitadel-mint-jwt: services requiring secrets will wait; timer will retry" >&2
# Exit 0 to not block activation - the timer will keep retrying
exit 0
''; '';
}; };
}; };
@ -775,6 +811,8 @@ in
RestartSec = "10s"; RestartSec = "10s";
TimeoutStartSec = "30s"; TimeoutStartSec = "30s";
UMask = "0077"; UMask = "0077";
# Only start if JWT exists (path unit will trigger us when it appears)
ExecCondition = "${pkgs.coreutils}/bin/test -s ${cfg.zitadelJwtPath}";
ExecStart = "${pkgs.openbao}/bin/bao agent -log-level=${lib.escapeShellArg cfg.vaultAgentLogLevel} -config=${mkAgentConfig}"; ExecStart = "${pkgs.openbao}/bin/bao agent -log-level=${lib.escapeShellArg cfg.vaultAgentLogLevel} -config=${mkAgentConfig}";
}; };

2
hosts/h001/flake.nix
View file

@ -90,7 +90,7 @@
owner = "root"; owner = "root";
group = "root"; group = "root";
mode = "0400"; mode = "0400";
path = "/run/secrets/litellm.env"; # Uses default: /var/lib/openbao-secrets/litellm-env
softDepend = [ "litellm" ]; softDepend = [ "litellm" ];
template = '' template = ''
{{- with secret "kv/data/machines/home/openrouter" -}} {{- with secret "kv/data/machines/home/openrouter" -}}

2
hosts/h001/mods/litellm.nix
View file

@ -29,7 +29,7 @@ in
host = "0.0.0.0"; host = "0.0.0.0";
openFirewall = false; openFirewall = false;
package = pkgsLitellm.litellm; package = pkgsLitellm.litellm;
environmentFile = "/run/secrets/litellm.env"; environmentFile = "/var/lib/openbao-secrets/litellm-env";
environment = { environment = {
SCARF_NO_ANALYTICS = "True"; SCARF_NO_ANALYTICS = "True";
DO_NOT_TRACK = "True"; DO_NOT_TRACK = "True";

2
hosts/juni/flake.nix
View file

@ -349,7 +349,7 @@
exit 0 exit 0
fi fi
secret="/run/secrets/atuin-key-josh" secret="/var/lib/openbao-secrets/atuin-key-josh"
if [ ! -s "$secret" ]; then if [ ! -s "$secret" ]; then
echo "Missing atuin secret at $secret" >&2 echo "Missing atuin secret at $secret" >&2
exit 1 exit 1

2
hosts/juni/impermanence.nix
View file

@ -27,7 +27,7 @@
# bao secrets # bao secrets
"/run/openbao" "/run/openbao"
"/run/secrets" "/var/lib/openbao-secrets"
]; ];
files = [ files = [
"/machine-key.json" "/machine-key.json"