Persist OpenBao secrets in /var/lib and make readiness non-blocking
This commit is contained in:
Joshua Bell
parent
8b54a94c54
commit
15fccd2ff4
6 changed files with 123 additions and 85 deletions
|
|
@ -42,7 +42,7 @@
|
|||
fragments = builtins.attrValues (builtins.mapAttrs (
|
||||
name: s:
|
||||
let
|
||||
secretPath = s.path or ("/run/secrets/" + name);
|
||||
secretPath = s.path or ("/var/lib/openbao-secrets/" + name);
|
||||
in
|
||||
substitute secretPath (s.configChanges or { })
|
||||
) secrets);
|
||||
|
|
@ -85,7 +85,7 @@
|
|||
fragments = builtins.attrValues (builtins.mapAttrs (
|
||||
name: s:
|
||||
let
|
||||
secretPath = s.path or ("/run/secrets/" + name);
|
||||
secretPath = s.path or ("/var/lib/openbao-secrets/" + name);
|
||||
in
|
||||
substitute secretPath (s.hmChanges or { })
|
||||
) secrets);
|
||||
|
|
|
|||
|
|
@ -318,6 +318,12 @@ in
|
|||
options.ringofstorms.secretsBao = {
|
||||
enable = lib.mkEnableOption "Fetch runtime secrets via OpenBao";
|
||||
|
||||
secretsBasePath = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/var/lib/openbao-secrets";
|
||||
description = "Base directory for rendered secrets. Use /var/lib/openbao-secrets for persistence across reboots, or /run/secrets for ephemeral.";
|
||||
};
|
||||
|
||||
zitadelKeyPath = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/machine-key.json";
|
||||
|
|
@ -404,7 +410,7 @@ in
|
|||
options = {
|
||||
path = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/run/secrets/${name}";
|
||||
default = "${cfg.secretsBasePath}/${name}";
|
||||
};
|
||||
|
||||
owner = lib.mkOption {
|
||||
|
|
@ -488,14 +494,17 @@ in
|
|||
sec
|
||||
];
|
||||
|
||||
# Persistent secrets directory - survives reboots
|
||||
systemd.tmpfiles.rules =
|
||||
[
|
||||
"d /run/openbao 0700 root root - -"
|
||||
"f /run/openbao/zitadel.jwt 0400 root root - -"
|
||||
"d /run/secrets 0711 root root - -"
|
||||
# Create base secrets directory (persistent by default)
|
||||
"d ${cfg.secretsBasePath} 0711 root root - -"
|
||||
]
|
||||
# Create empty placeholder files for all secret destinations so
|
||||
# services that reference env files don't fail when offline.
|
||||
# Important: we do NOT recreate files that already have content (the '-' at end)
|
||||
++ (lib.unique (
|
||||
lib.concatLists (
|
||||
lib.mapAttrsToList (
|
||||
|
|
@ -508,6 +517,8 @@ in
|
|||
) cfg.secrets
|
||||
)
|
||||
))
|
||||
# Only create placeholder if file doesn't exist (preserves persisted secrets)
|
||||
# Using 'f' with '-' for argument means create if not exists, don't truncate if exists
|
||||
++ (lib.mapAttrsToList (
|
||||
_: secret:
|
||||
"f ${secret.path} ${secret.mode} ${secret.owner} ${secret.group} - -"
|
||||
|
|
@ -547,7 +558,7 @@ in
|
|||
wantedBy = [ "multi-user.target" ];
|
||||
|
||||
pathConfig = {
|
||||
PathChanged = "/run/secrets";
|
||||
PathChanged = cfg.secretsBasePath;
|
||||
Unit = "openbao-secrets-ready.service";
|
||||
TriggerLimitIntervalSec = 30;
|
||||
TriggerLimitBurst = 3;
|
||||
|
|
@ -587,8 +598,9 @@ in
|
|||
)
|
||||
{
|
||||
openbao-secrets-ready = {
|
||||
description = "OpenBao: all configured secrets present";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
description = "OpenBao: all configured secrets present (informational)";
|
||||
# NOT in wantedBy - this is a passive check, not a startup requirement
|
||||
# It gets triggered by path units when secrets change
|
||||
wants = [ "vault-agent.service" ];
|
||||
after = [ "vault-agent.service" ];
|
||||
|
||||
|
|
@ -600,18 +612,25 @@ in
|
|||
UMask = "0077";
|
||||
ExecStart = pkgs.writeShellScript "openbao-secrets-ready" ''
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set -uo pipefail
|
||||
|
||||
missing=0
|
||||
${lib.concatStringsSep "\n" (
|
||||
lib.mapAttrsToList (name: secret: ''
|
||||
if [ ! -s ${lib.escapeShellArg secret.path} ]; then
|
||||
echo "Missing secret: ${secret.path}" >&2
|
||||
exit 1
|
||||
missing=1
|
||||
fi
|
||||
'') cfg.secrets
|
||||
)}
|
||||
|
||||
if [ "$missing" -eq 1 ]; then
|
||||
echo "Some secrets are missing (may be stale or not yet fetched)" >&2
|
||||
# Don't exit 1 - this is informational only
|
||||
else
|
||||
echo "All configured OpenBao secrets present." >&2
|
||||
fi
|
||||
exit 0
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
|
@ -637,6 +656,9 @@ in
|
|||
zitadel-mint-jwt = {
|
||||
description = "Mint Zitadel access token (JWT) for OpenBao";
|
||||
|
||||
# Non-blocking: don't add to wantedBy, let timer handle it
|
||||
# The timer is wantedBy timers.target, so this runs periodically
|
||||
|
||||
startLimitIntervalSec = 0;
|
||||
|
||||
after = [
|
||||
|
|
@ -656,17 +678,15 @@ in
|
|||
Type = "oneshot";
|
||||
User = "root";
|
||||
Group = "root";
|
||||
Restart = "on-failure";
|
||||
RestartSec = "30s";
|
||||
# Don't restart on failure - timer will retry later
|
||||
# This prevents blocking activation if Zitadel is down
|
||||
TimeoutStartSec = "2min";
|
||||
UMask = "0077";
|
||||
|
||||
|
||||
ExecStart = pkgs.writeShellScript "zitadel-mint-jwt-service" ''
|
||||
|
||||
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set -uo pipefail
|
||||
|
||||
if [ ! -d "/run/openbao" ]; then
|
||||
${pkgs.coreutils}/bin/mkdir -p /run/openbao
|
||||
|
|
@ -724,18 +744,16 @@ in
|
|||
return 1
|
||||
}
|
||||
|
||||
# Check if existing token is still valid
|
||||
if [ -s "${cfg.zitadelJwtPath}" ] && jwt_is_valid "$(cat "${cfg.zitadelJwtPath}")"; then
|
||||
echo "zitadel-mint-jwt: existing token still valid; skipping" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
jwt="$(${zitadelMintJwt}/bin/zitadel-mint-jwt)"
|
||||
|
||||
if [ -z "$jwt" ] || [ "$jwt" = "null" ]; then
|
||||
echo "Failed to mint Zitadel access token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Try to mint a new token
|
||||
jwt=""
|
||||
if jwt="$(${zitadelMintJwt}/bin/zitadel-mint-jwt 2>&1)"; then
|
||||
if [ -n "$jwt" ] && [ "$jwt" != "null" ]; then
|
||||
tmp="$(${pkgs.coreutils}/bin/mktemp)"
|
||||
trap '${pkgs.coreutils}/bin/rm -f "$tmp"' EXIT
|
||||
${pkgs.coreutils}/bin/printf '%s' "$jwt" > "$tmp"
|
||||
|
|
@ -748,6 +766,24 @@ in
|
|||
# Update the token file (the agent watches it).
|
||||
${pkgs.coreutils}/bin/cat "$tmp" > "${cfg.zitadelJwtPath}"
|
||||
${pkgs.coreutils}/bin/chmod 0400 "${cfg.zitadelJwtPath}" || true
|
||||
echo "zitadel-mint-jwt: successfully refreshed token" >&2
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Failed to mint new token - check if we have a stale but existing token
|
||||
if [ -s "${cfg.zitadelJwtPath}" ]; then
|
||||
echo "zitadel-mint-jwt: failed to refresh, but existing token present (may be stale)" >&2
|
||||
echo "zitadel-mint-jwt: continuing with stale token; will retry on next timer" >&2
|
||||
# Exit 0 so we don't block activation - timer will retry
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# No existing token and failed to mint - this is a problem but don't block
|
||||
echo "zitadel-mint-jwt: failed to mint token and no existing token available" >&2
|
||||
echo "zitadel-mint-jwt: services requiring secrets will wait; timer will retry" >&2
|
||||
# Exit 0 to not block activation - the timer will keep retrying
|
||||
exit 0
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
|
@ -775,6 +811,8 @@ in
|
|||
RestartSec = "10s";
|
||||
TimeoutStartSec = "30s";
|
||||
UMask = "0077";
|
||||
# Only start if JWT exists (path unit will trigger us when it appears)
|
||||
ExecCondition = "${pkgs.coreutils}/bin/test -s ${cfg.zitadelJwtPath}";
|
||||
ExecStart = "${pkgs.openbao}/bin/bao agent -log-level=${lib.escapeShellArg cfg.vaultAgentLogLevel} -config=${mkAgentConfig}";
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -90,7 +90,7 @@
|
|||
owner = "root";
|
||||
group = "root";
|
||||
mode = "0400";
|
||||
path = "/run/secrets/litellm.env";
|
||||
# Uses default: /var/lib/openbao-secrets/litellm-env
|
||||
softDepend = [ "litellm" ];
|
||||
template = ''
|
||||
{{- with secret "kv/data/machines/home/openrouter" -}}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ in
|
|||
host = "0.0.0.0";
|
||||
openFirewall = false;
|
||||
package = pkgsLitellm.litellm;
|
||||
environmentFile = "/run/secrets/litellm.env";
|
||||
environmentFile = "/var/lib/openbao-secrets/litellm-env";
|
||||
environment = {
|
||||
SCARF_NO_ANALYTICS = "True";
|
||||
DO_NOT_TRACK = "True";
|
||||
|
|
|
|||
|
|
@ -349,7 +349,7 @@
|
|||
exit 0
|
||||
fi
|
||||
|
||||
secret="/run/secrets/atuin-key-josh"
|
||||
secret="/var/lib/openbao-secrets/atuin-key-josh"
|
||||
if [ ! -s "$secret" ]; then
|
||||
echo "Missing atuin secret at $secret" >&2
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@
|
|||
|
||||
# bao secrets
|
||||
"/run/openbao"
|
||||
"/run/secrets"
|
||||
"/var/lib/openbao-secrets"
|
||||
];
|
||||
files = [
|
||||
"/machine-key.json"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue