ringofstorms/dotfiles
ringofstorms/dotfiles
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Switch litellm public to API key, move secret to o001, use module

Browse source
This commit is contained in:
Joshua Bell 2026-01-21 19:54:14 -06:00
parent d706c122b1
commit eebf354173
9 changed files with 141 additions and 313 deletions
Download patch file Download diff file Expand all files Collapse all files

1
hosts/h001/containers/default.nix
View file

@ -4,7 +4,6 @@
{
imports = [
./forgejo.nix
./litellm-public.nix
./opengist.nix
./zitadel.nix
];

295
hosts/h001/containers/litellm-public.nix
View file

@ -1,295 +0,0 @@
{
config,
lib,
inputs,
...
}:
let
name = "litellm-public";
hostDataDir = "/var/lib/${name}";
hostAddress = "10.0.0.1";
containerAddress = "10.0.0.4";
hostAddress6 = "fc00::1";
containerAddress6 = "fc00::4";
litellmNixpkgs = inputs.litellm-nixpkgs;
pkgsLitellm = import litellmNixpkgs {
system = "x86_64-linux";
config.allowUnfree = true;
};
containerPort = 4000;
externalPort = 8095;
hasSecret =
secret:
let
secrets = config.age.secrets or { };
in
secrets ? ${secret} && secrets.${secret} != null;
binds = [
{
host = "${hostDataDir}/postgres";
container = "/var/lib/postgresql/17";
user = "postgres";
uid = config.ids.uids.postgres;
gid = config.ids.gids.postgres;
}
{
host = "${hostDataDir}/backups/postgres";
container = "/var/backup/postgresql";
user = "postgres";
uid = config.ids.uids.postgres;
gid = config.ids.gids.postgres;
}
]
++ lib.optionals (hasSecret "litellm_public_master_key") [
{
host = config.age.secrets.litellm_public_master_key.path;
container = "/var/secrets/litellm_master_key";
readOnly = true;
}
];
bindsWithUsers = lib.filter (b: b ? user) binds;
uniqueUsers = lib.foldl' (
acc: bind: if lib.lists.any (item: item.user == bind.user) acc then acc else acc ++ [ bind ]
) [ ] bindsWithUsers;
users = {
users = lib.listToAttrs (
lib.map (u: {
name = u.user;
value = {
isSystemUser = true;
name = u.user;
uid = u.uid;
group = u.user;
};
}) uniqueUsers
);
groups = lib.listToAttrs (
lib.map (g: {
name = g.user;
value.gid = g.gid;
}) uniqueUsers
);
};
azureModels = [
"gpt-5.2-2025-12-11"
"gpt-5.1-2025-11-13"
"gpt-4o-2024-05-13"
"gpt-4.1-2025-04-14"
"gpt-4.1-mini-2025-04-14"
"gpt-5-nano-2025-08-07"
"gpt-5-mini-2025-08-07"
"gpt-5-2025-08-07"
];
azureReasoningAliases = [
{
model_name = "azure-gpt-5.2-low";
litellm_params = {
model = "azure/gpt-5.2-2025-12-11";
api_base = "http://100.64.0.8:9010/azure";
api_version = "2025-04-01-preview";
api_key = "na";
extra_body = {
reasoning_effort = "low";
};
};
}
{
model_name = "azure-gpt-5.2-medium";
litellm_params = {
model = "azure/gpt-5.2-2025-12-11";
api_base = "http://100.64.0.8:9010/azure";
api_version = "2025-04-01-preview";
api_key = "na";
extra_body = {
reasoning_effort = "medium";
};
};
}
{
model_name = "azure-gpt-5.2-high";
litellm_params = {
model = "azure/gpt-5.2-2025-12-11";
api_base = "http://100.64.0.8:9010/azure";
api_version = "2025-04-01-preview";
api_key = "na";
extra_body = {
reasoning_effort = "high";
};
};
}
];
in
{
options = { };
config = {
networking.firewall.interfaces."tailscale0".allowedTCPPorts = [ externalPort ];
services.nginx.virtualHosts."llm.joshuabell.xyz" = {
listen = [
{
addr = "0.0.0.0";
port = externalPort;
}
];
locations = {
"/" = {
proxyWebsockets = true;
recommendedProxySettings = true;
proxyPass = "http://${containerAddress}:${toString containerPort}";
};
};
};
inherit users;
system.activationScripts."createDirsFor${name}" = ''
${lib.concatStringsSep "\n" (
lib.map (bind: ''
mkdir -p ${bind.host}
chown -R ${toString bind.user}:${toString bind.gid} ${bind.host}
chmod -R 750 ${bind.host}
'') bindsWithUsers
)}
'';
containers.${name} = {
ephemeral = true;
autoStart = true;
privateNetwork = true;
hostAddress = hostAddress;
localAddress = containerAddress;
hostAddress6 = hostAddress6;
localAddress6 = containerAddress6;
bindMounts = lib.foldl (
acc: bind:
{
"${bind.container}" = {
hostPath = bind.host;
isReadOnly = bind.readOnly or false;
};
}
// acc
) { } binds;
nixpkgs = litellmNixpkgs;
config =
{
config,
pkgs,
lib,
...
}:
{
config = {
system.stateVersion = "25.05";
networking = {
firewall = {
enable = true;
allowedTCPPorts = [ containerPort ];
};
useHostResolvConf = lib.mkForce false;
};
services.resolved.enable = true;
inherit users;
services.postgresql = {
enable = true;
package = pkgs.postgresql_17.withJIT;
enableJIT = true;
authentication = ''
local all all trust
host all all 127.0.0.1/8 trust
host all all ::1/128 trust
'';
ensureDatabases = [ "litellm" ];
ensureUsers = [
{
name = "litellm";
ensureDBOwnership = true;
ensureClauses.login = true;
}
];
};
services.postgresqlBackup = {
enable = true;
};
systemd.services.litellm = {
description = "LiteLLM Public Proxy";
after = [
"network.target"
"postgresql.service"
];
requires = [ "postgresql.service" ];
wantedBy = [ "multi-user.target" ];
environment = {
DATABASE_URL = "postgresql://litellm@/litellm?host=/var/run/postgresql";
SCARF_NO_ANALYTICS = "True";
DO_NOT_TRACK = "True";
ANONYMIZED_TELEMETRY = "False";
};
script = ''
export LITELLM_MASTER_KEY="$(cat /var/secrets/litellm_master_key)"
exec ${pkgsLitellm.litellm}/bin/litellm --config /etc/litellm/config.yaml --host 0.0.0.0 --port ${toString containerPort}
'';
serviceConfig = {
Type = "simple";
User = "litellm";
Group = "litellm";
Restart = "always";
RestartSec = 5;
};
};
users.users.litellm = {
isSystemUser = true;
group = "litellm";
extraGroups = [ "keys" ];
};
users.groups.litellm = { };
users.groups.keys = { };
environment.etc."litellm/config.yaml".text = builtins.toJSON {
general_settings = {
master_key = "os.environ/LITELLM_MASTER_KEY";
database_url = "os.environ/DATABASE_URL";
};
litellm_settings = {
check_provider_endpoints = true;
drop_params = true;
modify_params = true;
};
model_list =
(builtins.map (m: {
model_name = "azure-${m}";
litellm_params = {
model = "azure/${m}";
api_base = "http://100.64.0.8:9010/azure";
api_version = "2025-04-01-preview";
api_key = "na";
};
}) azureModels)
++ azureReasoningAliases;
};
};
};
};
};
}

1
hosts/h001/mods/default.nix
View file

@ -4,6 +4,7 @@
{
imports = [
./litellm.nix
./litellm-public.nix
./nixarr.nix
./hardware-transcoding.nix
./monitoring_hub.nix

116
hosts/h001/mods/litellm-public.nix Normal file
View file

@ -0,0 +1,116 @@
{
inputs,
pkgs,
lib,
...
}:
let
declaration = "services/misc/litellm.nix";
nixpkgsLitellm = inputs.litellm-nixpkgs;
pkgsLitellm = import nixpkgsLitellm {
inherit (pkgs) system;
config.allowUnfree = true;
};
port = 8095;
azureModels = [
"gpt-5.2-2025-12-11"
"gpt-5.1-2025-11-13"
"gpt-4o-2024-05-13"
"gpt-4.1-2025-04-14"
"gpt-4.1-mini-2025-04-14"
"gpt-5-nano-2025-08-07"
"gpt-5-mini-2025-08-07"
"gpt-5-2025-08-07"
];
in
{
options = { };
config = {
networking.firewall.interfaces."tailscale0".allowedTCPPorts = [ port ];
systemd.services.litellm-public = {
description = "LiteLLM Public Proxy (Azure models only)";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
environment = {
SCARF_NO_ANALYTICS = "True";
DO_NOT_TRACK = "True";
ANONYMIZED_TELEMETRY = "False";
};
serviceConfig = {
Type = "simple";
User = "litellm-public";
Group = "litellm-public";
StateDirectory = "litellm-public";
ExecStart = "${pkgsLitellm.litellm}/bin/litellm --config /etc/litellm-public/config.yaml --host 0.0.0.0 --port ${toString port}";
Restart = "always";
RestartSec = 5;
};
};
users.users.litellm-public = {
isSystemUser = true;
group = "litellm-public";
};
users.groups.litellm-public = { };
environment.etc."litellm-public/config.yaml".text = lib.generators.toYAML { } {
litellm_settings = {
check_provider_endpoints = true;
drop_params = true;
modify_params = true;
};
model_list =
(builtins.map (m: {
model_name = "azure-${m}";
litellm_params = {
model = "azure/${m}";
api_base = "http://100.64.0.8:9010/azure";
api_version = "2025-04-01-preview";
api_key = "na";
};
}) azureModels)
++ [
{
model_name = "azure-gpt-5.2-low";
litellm_params = {
model = "azure/gpt-5.2-2025-12-11";
api_base = "http://100.64.0.8:9010/azure";
api_version = "2025-04-01-preview";
api_key = "na";
extra_body = {
reasoning_effort = "low";
};
};
}
{
model_name = "azure-gpt-5.2-medium";
litellm_params = {
model = "azure/gpt-5.2-2025-12-11";
api_base = "http://100.64.0.8:9010/azure";
api_version = "2025-04-01-preview";
api_key = "na";
extra_body = {
reasoning_effort = "medium";
};
};
}
{
model_name = "azure-gpt-5.2-high";
litellm_params = {
model = "azure/gpt-5.2-2025-12-11";
api_base = "http://100.64.0.8:9010/azure";
api_version = "2025-04-01-preview";
api_key = "na";
extra_body = {
reasoning_effort = "high";
};
};
}
];
};
};
}