From 9a95841ff04c2b3be722d9c4595cb7c98bb4121f Mon Sep 17 00:00:00 2001 From: Joshua Bell Date: Wed, 21 Jan 2026 18:46:09 -0600 Subject: [PATCH] Add litellm-public container, age master key, and nginx update --- flakes/secrets/litellm_public_master_key.age | 15 + flakes/secrets/secrets.nix | 3 + hosts/h001/containers/default.nix | 1 + hosts/h001/containers/litellm-public.nix | 295 +++++++++++++++++++ hosts/oracle/o001/nginx.nix | 8 + 5 files changed, 322 insertions(+) create mode 100644 flakes/secrets/litellm_public_master_key.age create mode 100644 hosts/h001/containers/litellm-public.nix diff --git a/flakes/secrets/litellm_public_master_key.age b/flakes/secrets/litellm_public_master_key.age new file mode 100644 index 00000000..952a8d97 --- /dev/null +++ b/flakes/secrets/litellm_public_master_key.age @@ -0,0 +1,15 @@ +-----BEGIN AGE ENCRYPTED FILE----- +YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IDd6MzN5USBHaTlH +dm5OTFdJN0JoazBHWktoS1N0OEQ4YXlUV0E0WkVISkpIRnI4dUZnClg3cjllUFZ2 +NTdyRDNlNUNacW9CYmJESC80RWZ5QnhXQ2tBVzhBMDkyak0KLT4gc3NoLWVkMjU1 +MTkgcGVGQlFnIGs4ck94aXd3MEVURHF4dUgrNGtTYURnanRDaC9CWlU3ZXZQWVVo +MjFmeW8KWlA2Wlorb2hwZTROTEl6SFVwRXpGbG8vZFNoOVVLS3I0Z1BwdGJKZHha +awotPiBzc2gtZWQyNTUxOSA5di8ySEEgNGhSRmI0WnFCWmJQUGNTdmwzYzh2SElR +UWdkZGVldmdkb1pYL0p0MFZDZwp2aU4ra3YvUXB5T3JoWWRsNDBLaU5FNzNoZTll +L3JYSjhWRFE5SW5aeUpvCi0+IGgmZGJXX0ctZ3JlYXNlIFdZaiB8JygjPFgKWHFw +RHZQdDZidzU4Z0dFb0ZXelQ2QVZZdEhKQVYyRjhoOUk4dFN0Slc4VXdPNFgvRllP +SjNGcDNwMlRINStmZwphL082UDlWZ2J6SHVjdGpUN0RvUXRxTjQKLS0tIExFVmt6 +bm0rNHBJTTRZT0V0ckFndklOa3RZcHI2TzQ5VmdROVZjeVlxMjQKIECYbwvAxSuY +6Fwl6t8NTFePG345p8J/n24jzL+/Ndj36GuFu3/PGN1bb1BAsaVEsJSXERY324Ym +ksYm6Tuuf2CQjY9nA2kAF78hnXnWJ1h9RG/dTMudF68BgpypsPRSFemu3g== +-----END AGE ENCRYPTED FILE----- diff --git a/flakes/secrets/secrets.nix b/flakes/secrets/secrets.nix index a51e3a45..2917246d 100644 --- a/flakes/secrets/secrets.nix +++ b/flakes/secrets/secrets.nix @@ -134,6 +134,9 @@ in "openwebui_env.age" = { publicKeys = authorityKey ++ h001; }; + "litellm_public_master_key.age" = { + publicKeys = authorityKey ++ h001; + }; "vaultwarden_env.age" = { publicKeys = authorityKey ++ o001; }; diff --git a/hosts/h001/containers/default.nix b/hosts/h001/containers/default.nix index 37c50cdb..62fd314b 100644 --- a/hosts/h001/containers/default.nix +++ b/hosts/h001/containers/default.nix @@ -4,6 +4,7 @@ { imports = [ ./forgejo.nix + ./litellm-public.nix ./opengist.nix ./zitadel.nix ]; diff --git a/hosts/h001/containers/litellm-public.nix b/hosts/h001/containers/litellm-public.nix new file mode 100644 index 00000000..b9736537 --- /dev/null +++ b/hosts/h001/containers/litellm-public.nix @@ -0,0 +1,295 @@ +{ + config, + lib, + inputs, + ... +}: +let + name = "litellm-public"; + + hostDataDir = "/var/lib/${name}"; + + hostAddress = "10.0.0.1"; + containerAddress = "10.0.0.4"; + hostAddress6 = "fc00::1"; + containerAddress6 = "fc00::4"; + + litellmNixpkgs = inputs.litellm-nixpkgs; + pkgsLitellm = import litellmNixpkgs { + system = "x86_64-linux"; + config.allowUnfree = true; + }; + + containerPort = 4000; + externalPort = 8095; + + hasSecret = + secret: + let + secrets = config.age.secrets or { }; + in + secrets ? ${secret} && secrets.${secret} != null; + + binds = [ + { + host = "${hostDataDir}/postgres"; + container = "/var/lib/postgresql/17"; + user = "postgres"; + uid = config.ids.uids.postgres; + gid = config.ids.gids.postgres; + } + { + host = "${hostDataDir}/backups/postgres"; + container = "/var/backup/postgresql"; + user = "postgres"; + uid = config.ids.uids.postgres; + gid = config.ids.gids.postgres; + } + ] + ++ lib.optionals (hasSecret "litellm_public_master_key") [ + { + host = config.age.secrets.litellm_public_master_key.path; + container = "/var/secrets/litellm_master_key"; + readOnly = true; + } + ]; + + bindsWithUsers = lib.filter (b: b ? user) binds; + uniqueUsers = lib.foldl' ( + acc: bind: if lib.lists.any (item: item.user == bind.user) acc then acc else acc ++ [ bind ] + ) [ ] bindsWithUsers; + + users = { + users = lib.listToAttrs ( + lib.map (u: { + name = u.user; + value = { + isSystemUser = true; + name = u.user; + uid = u.uid; + group = u.user; + }; + }) uniqueUsers + ); + + groups = lib.listToAttrs ( + lib.map (g: { + name = g.user; + value.gid = g.gid; + }) uniqueUsers + ); + }; + + azureModels = [ + "gpt-5.2-2025-12-11" + "gpt-5.1-2025-11-13" + "gpt-4o-2024-05-13" + "gpt-4.1-2025-04-14" + "gpt-4.1-mini-2025-04-14" + "gpt-5-nano-2025-08-07" + "gpt-5-mini-2025-08-07" + "gpt-5-2025-08-07" + ]; + + azureReasoningAliases = [ + { + model_name = "azure-gpt-5.2-low"; + litellm_params = { + model = "azure/gpt-5.2-2025-12-11"; + api_base = "http://100.64.0.8:9010/azure"; + api_version = "2025-04-01-preview"; + api_key = "na"; + extra_body = { + reasoning_effort = "low"; + }; + }; + } + { + model_name = "azure-gpt-5.2-medium"; + litellm_params = { + model = "azure/gpt-5.2-2025-12-11"; + api_base = "http://100.64.0.8:9010/azure"; + api_version = "2025-04-01-preview"; + api_key = "na"; + extra_body = { + reasoning_effort = "medium"; + }; + }; + } + { + model_name = "azure-gpt-5.2-high"; + litellm_params = { + model = "azure/gpt-5.2-2025-12-11"; + api_base = "http://100.64.0.8:9010/azure"; + api_version = "2025-04-01-preview"; + api_key = "na"; + extra_body = { + reasoning_effort = "high"; + }; + }; + } + ]; + +in +{ + options = { }; + config = { + networking.firewall.interfaces."tailscale0".allowedTCPPorts = [ externalPort ]; + + services.nginx.virtualHosts."litellm-public" = { + listen = [ + { + addr = "0.0.0.0"; + port = externalPort; + } + ]; + locations = { + "/" = { + proxyWebsockets = true; + recommendedProxySettings = true; + proxyPass = "http://${containerAddress}:${toString containerPort}"; + }; + }; + }; + + inherit users; + + system.activationScripts."createDirsFor${name}" = '' + ${lib.concatStringsSep "\n" ( + lib.map (bind: '' + mkdir -p ${bind.host} + chown -R ${toString bind.user}:${toString bind.gid} ${bind.host} + chmod -R 750 ${bind.host} + '') bindsWithUsers + )} + ''; + + containers.${name} = { + ephemeral = true; + autoStart = true; + privateNetwork = true; + hostAddress = hostAddress; + localAddress = containerAddress; + hostAddress6 = hostAddress6; + localAddress6 = containerAddress6; + bindMounts = lib.foldl ( + acc: bind: + { + "${bind.container}" = { + hostPath = bind.host; + isReadOnly = bind.readOnly or false; + }; + } + // acc + ) { } binds; + nixpkgs = litellmNixpkgs; + config = + { + config, + pkgs, + lib, + ... + }: + { + config = { + system.stateVersion = "25.05"; + + networking = { + firewall = { + enable = true; + allowedTCPPorts = [ containerPort ]; + }; + useHostResolvConf = lib.mkForce false; + }; + services.resolved.enable = true; + + inherit users; + + services.postgresql = { + enable = true; + package = pkgs.postgresql_17.withJIT; + enableJIT = true; + authentication = '' + local all all trust + host all all 127.0.0.1/8 trust + host all all ::1/128 trust + ''; + ensureDatabases = [ "litellm" ]; + ensureUsers = [ + { + name = "litellm"; + ensureDBOwnership = true; + ensureClauses.login = true; + } + ]; + }; + + services.postgresqlBackup = { + enable = true; + }; + + systemd.services.litellm = { + description = "LiteLLM Public Proxy"; + after = [ + "network.target" + "postgresql.service" + ]; + requires = [ "postgresql.service" ]; + wantedBy = [ "multi-user.target" ]; + + environment = { + DATABASE_URL = "postgresql://litellm@/litellm?host=/var/run/postgresql"; + SCARF_NO_ANALYTICS = "True"; + DO_NOT_TRACK = "True"; + ANONYMIZED_TELEMETRY = "False"; + }; + + script = '' + export LITELLM_MASTER_KEY="$(cat /var/secrets/litellm_master_key)" + exec ${pkgsLitellm.litellm}/bin/litellm --config /etc/litellm/config.yaml --host 0.0.0.0 --port ${toString containerPort} + ''; + + serviceConfig = { + Type = "simple"; + User = "litellm"; + Group = "litellm"; + Restart = "always"; + RestartSec = 5; + }; + }; + + users.users.litellm = { + isSystemUser = true; + group = "litellm"; + extraGroups = [ "keys" ]; + }; + users.groups.litellm = { }; + users.groups.keys = { }; + + environment.etc."litellm/config.yaml".text = builtins.toJSON { + general_settings = { + master_key = "os.environ/LITELLM_MASTER_KEY"; + database_url = "os.environ/DATABASE_URL"; + }; + litellm_settings = { + check_provider_endpoints = true; + drop_params = true; + modify_params = true; + }; + model_list = + (builtins.map (m: { + model_name = "azure-${m}"; + litellm_params = { + model = "azure/${m}"; + api_base = "http://100.64.0.8:9010/azure"; + api_version = "2025-04-01-preview"; + api_key = "na"; + }; + }) azureModels) + ++ azureReasoningAliases; + }; + }; + }; + }; + }; +} diff --git a/hosts/oracle/o001/nginx.nix b/hosts/oracle/o001/nginx.nix index 20f2eb59..2f47aa3e 100644 --- a/hosts/oracle/o001/nginx.nix +++ b/hosts/oracle/o001/nginx.nix @@ -228,6 +228,14 @@ proxyPass = "http://100.64.0.13"; }; }; + "llm.joshuabell.xyz" = { + enableACME = true; + forceSSL = true; + locations."/" = { + proxyWebsockets = true; + proxyPass = "http://100.64.0.13:8095"; + }; + }; "_" = { rejectSSL = true;