From eebf3541735e5a2a46ab02ce4ccb76f56d61693b Mon Sep 17 00:00:00 2001 From: Joshua Bell Date: Wed, 21 Jan 2026 19:54:14 -0600 Subject: [PATCH] Switch litellm public to API key, move secret to o001, use module --- flakes/secrets/flake.nix | 1 + flakes/secrets/litellm_public_api_key.age | 13 + flakes/secrets/litellm_public_master_key.age | 15 - flakes/secrets/secrets.nix | 4 +- hosts/h001/containers/default.nix | 1 - hosts/h001/containers/litellm-public.nix | 295 ------------------- hosts/h001/mods/default.nix | 1 + hosts/h001/mods/litellm-public.nix | 116 ++++++++ hosts/oracle/o001/nginx.nix | 8 + 9 files changed, 141 insertions(+), 313 deletions(-) create mode 100644 flakes/secrets/litellm_public_api_key.age delete mode 100644 flakes/secrets/litellm_public_master_key.age delete mode 100644 hosts/h001/containers/litellm-public.nix create mode 100644 hosts/h001/mods/litellm-public.nix diff --git a/flakes/secrets/flake.nix b/flakes/secrets/flake.nix index ffaad6ca..75b6c98d 100644 --- a/flakes/secrets/flake.nix +++ b/flakes/secrets/flake.nix @@ -52,6 +52,7 @@ "zitadel_master_key" "openwebui_env" "vaultwarden_env" + "litellm_public_api_key" ]; # Keep only secrets intended for this host (or that include the authority key) diff --git a/flakes/secrets/litellm_public_api_key.age b/flakes/secrets/litellm_public_api_key.age new file mode 100644 index 00000000..62d19136 --- /dev/null +++ b/flakes/secrets/litellm_public_api_key.age @@ -0,0 +1,13 @@ +-----BEGIN AGE ENCRYPTED FILE----- +YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IDd6MzN5USBSS0VL +YmZpZVg0SGZNY0d6ZHBQTDNsbk5vTjlETk42MnpuQ25CNDNPcTI0CklzM0dLZTdX +ZmdyY2JiNFk1MVlDV3pKNFFrOGduRktVaVRNNmdRSzFlTmsKLT4gc3NoLWVkMjU1 +MTkgc2EwSmpnIGJJLzdWeUtWTUloenp1STd5WWZyTFBuRkdsRkdXRTR1N1VINVEy +UkhJd1UKK2doWXpBRkdIbE1FampiWlozTUZiMFBsL3hPR0hzWHpmTU0wbEFDVnFP +MAotPiApcio8LWdyZWFzZSAzRFElICk5NW9yTFAgJwpxcFpUQ0JWMVRQMVEvdGRw +MUpWbmdsOElqa0MzTGVZOXg0amEKLS0tIGRPQ3JzV0VjOEQzWjQyM3l5c1ZXbmNW +cHRWRjMzSlZsa25OcGczUm1uWTQKFr4yE3vA4vwExj4F+foRxOQTDPms1y5uXtDl +oWchhi4y5bx+1jz02mZcSBixntm4h0edhDsrGFgRlbOW2YxfX8tMDXtljQfc5nP1 +uxWC2XdUwLuOzAbjtM2bJT9Bqx3XHDTkY6Fbs50pz2Fo0b1RnYMMp5P8dDhi2lUl +ghyArRxnecAbp5ad/7bQLrfpadh6N4ySe/HwA4MJF/i1fQ== +-----END AGE ENCRYPTED FILE----- diff --git a/flakes/secrets/litellm_public_master_key.age b/flakes/secrets/litellm_public_master_key.age deleted file mode 100644 index 952a8d97..00000000 --- a/flakes/secrets/litellm_public_master_key.age +++ /dev/null @@ -1,15 +0,0 @@ ------BEGIN AGE ENCRYPTED FILE----- -YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IDd6MzN5USBHaTlH -dm5OTFdJN0JoazBHWktoS1N0OEQ4YXlUV0E0WkVISkpIRnI4dUZnClg3cjllUFZ2 -NTdyRDNlNUNacW9CYmJESC80RWZ5QnhXQ2tBVzhBMDkyak0KLT4gc3NoLWVkMjU1 -MTkgcGVGQlFnIGs4ck94aXd3MEVURHF4dUgrNGtTYURnanRDaC9CWlU3ZXZQWVVo -MjFmeW8KWlA2Wlorb2hwZTROTEl6SFVwRXpGbG8vZFNoOVVLS3I0Z1BwdGJKZHha -awotPiBzc2gtZWQyNTUxOSA5di8ySEEgNGhSRmI0WnFCWmJQUGNTdmwzYzh2SElR -UWdkZGVldmdkb1pYL0p0MFZDZwp2aU4ra3YvUXB5T3JoWWRsNDBLaU5FNzNoZTll -L3JYSjhWRFE5SW5aeUpvCi0+IGgmZGJXX0ctZ3JlYXNlIFdZaiB8JygjPFgKWHFw -RHZQdDZidzU4Z0dFb0ZXelQ2QVZZdEhKQVYyRjhoOUk4dFN0Slc4VXdPNFgvRllP -SjNGcDNwMlRINStmZwphL082UDlWZ2J6SHVjdGpUN0RvUXRxTjQKLS0tIExFVmt6 -bm0rNHBJTTRZT0V0ckFndklOa3RZcHI2TzQ5VmdROVZjeVlxMjQKIECYbwvAxSuY -6Fwl6t8NTFePG345p8J/n24jzL+/Ndj36GuFu3/PGN1bb1BAsaVEsJSXERY324Ym -ksYm6Tuuf2CQjY9nA2kAF78hnXnWJ1h9RG/dTMudF68BgpypsPRSFemu3g== ------END AGE ENCRYPTED FILE----- diff --git a/flakes/secrets/secrets.nix b/flakes/secrets/secrets.nix index 2917246d..c4d6037b 100644 --- a/flakes/secrets/secrets.nix +++ b/flakes/secrets/secrets.nix @@ -134,8 +134,8 @@ in "openwebui_env.age" = { publicKeys = authorityKey ++ h001; }; - "litellm_public_master_key.age" = { - publicKeys = authorityKey ++ h001; + "litellm_public_api_key.age" = { + publicKeys = authorityKey ++ o001; }; "vaultwarden_env.age" = { publicKeys = authorityKey ++ o001; diff --git a/hosts/h001/containers/default.nix b/hosts/h001/containers/default.nix index 62fd314b..37c50cdb 100644 --- a/hosts/h001/containers/default.nix +++ b/hosts/h001/containers/default.nix @@ -4,7 +4,6 @@ { imports = [ ./forgejo.nix - ./litellm-public.nix ./opengist.nix ./zitadel.nix ]; diff --git a/hosts/h001/containers/litellm-public.nix b/hosts/h001/containers/litellm-public.nix deleted file mode 100644 index 93797752..00000000 --- a/hosts/h001/containers/litellm-public.nix +++ /dev/null @@ -1,295 +0,0 @@ -{ - config, - lib, - inputs, - ... -}: -let - name = "litellm-public"; - - hostDataDir = "/var/lib/${name}"; - - hostAddress = "10.0.0.1"; - containerAddress = "10.0.0.4"; - hostAddress6 = "fc00::1"; - containerAddress6 = "fc00::4"; - - litellmNixpkgs = inputs.litellm-nixpkgs; - pkgsLitellm = import litellmNixpkgs { - system = "x86_64-linux"; - config.allowUnfree = true; - }; - - containerPort = 4000; - externalPort = 8095; - - hasSecret = - secret: - let - secrets = config.age.secrets or { }; - in - secrets ? ${secret} && secrets.${secret} != null; - - binds = [ - { - host = "${hostDataDir}/postgres"; - container = "/var/lib/postgresql/17"; - user = "postgres"; - uid = config.ids.uids.postgres; - gid = config.ids.gids.postgres; - } - { - host = "${hostDataDir}/backups/postgres"; - container = "/var/backup/postgresql"; - user = "postgres"; - uid = config.ids.uids.postgres; - gid = config.ids.gids.postgres; - } - ] - ++ lib.optionals (hasSecret "litellm_public_master_key") [ - { - host = config.age.secrets.litellm_public_master_key.path; - container = "/var/secrets/litellm_master_key"; - readOnly = true; - } - ]; - - bindsWithUsers = lib.filter (b: b ? user) binds; - uniqueUsers = lib.foldl' ( - acc: bind: if lib.lists.any (item: item.user == bind.user) acc then acc else acc ++ [ bind ] - ) [ ] bindsWithUsers; - - users = { - users = lib.listToAttrs ( - lib.map (u: { - name = u.user; - value = { - isSystemUser = true; - name = u.user; - uid = u.uid; - group = u.user; - }; - }) uniqueUsers - ); - - groups = lib.listToAttrs ( - lib.map (g: { - name = g.user; - value.gid = g.gid; - }) uniqueUsers - ); - }; - - azureModels = [ - "gpt-5.2-2025-12-11" - "gpt-5.1-2025-11-13" - "gpt-4o-2024-05-13" - "gpt-4.1-2025-04-14" - "gpt-4.1-mini-2025-04-14" - "gpt-5-nano-2025-08-07" - "gpt-5-mini-2025-08-07" - "gpt-5-2025-08-07" - ]; - - azureReasoningAliases = [ - { - model_name = "azure-gpt-5.2-low"; - litellm_params = { - model = "azure/gpt-5.2-2025-12-11"; - api_base = "http://100.64.0.8:9010/azure"; - api_version = "2025-04-01-preview"; - api_key = "na"; - extra_body = { - reasoning_effort = "low"; - }; - }; - } - { - model_name = "azure-gpt-5.2-medium"; - litellm_params = { - model = "azure/gpt-5.2-2025-12-11"; - api_base = "http://100.64.0.8:9010/azure"; - api_version = "2025-04-01-preview"; - api_key = "na"; - extra_body = { - reasoning_effort = "medium"; - }; - }; - } - { - model_name = "azure-gpt-5.2-high"; - litellm_params = { - model = "azure/gpt-5.2-2025-12-11"; - api_base = "http://100.64.0.8:9010/azure"; - api_version = "2025-04-01-preview"; - api_key = "na"; - extra_body = { - reasoning_effort = "high"; - }; - }; - } - ]; - -in -{ - options = { }; - config = { - networking.firewall.interfaces."tailscale0".allowedTCPPorts = [ externalPort ]; - - services.nginx.virtualHosts."llm.joshuabell.xyz" = { - listen = [ - { - addr = "0.0.0.0"; - port = externalPort; - } - ]; - locations = { - "/" = { - proxyWebsockets = true; - recommendedProxySettings = true; - proxyPass = "http://${containerAddress}:${toString containerPort}"; - }; - }; - }; - - inherit users; - - system.activationScripts."createDirsFor${name}" = '' - ${lib.concatStringsSep "\n" ( - lib.map (bind: '' - mkdir -p ${bind.host} - chown -R ${toString bind.user}:${toString bind.gid} ${bind.host} - chmod -R 750 ${bind.host} - '') bindsWithUsers - )} - ''; - - containers.${name} = { - ephemeral = true; - autoStart = true; - privateNetwork = true; - hostAddress = hostAddress; - localAddress = containerAddress; - hostAddress6 = hostAddress6; - localAddress6 = containerAddress6; - bindMounts = lib.foldl ( - acc: bind: - { - "${bind.container}" = { - hostPath = bind.host; - isReadOnly = bind.readOnly or false; - }; - } - // acc - ) { } binds; - nixpkgs = litellmNixpkgs; - config = - { - config, - pkgs, - lib, - ... - }: - { - config = { - system.stateVersion = "25.05"; - - networking = { - firewall = { - enable = true; - allowedTCPPorts = [ containerPort ]; - }; - useHostResolvConf = lib.mkForce false; - }; - services.resolved.enable = true; - - inherit users; - - services.postgresql = { - enable = true; - package = pkgs.postgresql_17.withJIT; - enableJIT = true; - authentication = '' - local all all trust - host all all 127.0.0.1/8 trust - host all all ::1/128 trust - ''; - ensureDatabases = [ "litellm" ]; - ensureUsers = [ - { - name = "litellm"; - ensureDBOwnership = true; - ensureClauses.login = true; - } - ]; - }; - - services.postgresqlBackup = { - enable = true; - }; - - systemd.services.litellm = { - description = "LiteLLM Public Proxy"; - after = [ - "network.target" - "postgresql.service" - ]; - requires = [ "postgresql.service" ]; - wantedBy = [ "multi-user.target" ]; - - environment = { - DATABASE_URL = "postgresql://litellm@/litellm?host=/var/run/postgresql"; - SCARF_NO_ANALYTICS = "True"; - DO_NOT_TRACK = "True"; - ANONYMIZED_TELEMETRY = "False"; - }; - - script = '' - export LITELLM_MASTER_KEY="$(cat /var/secrets/litellm_master_key)" - exec ${pkgsLitellm.litellm}/bin/litellm --config /etc/litellm/config.yaml --host 0.0.0.0 --port ${toString containerPort} - ''; - - serviceConfig = { - Type = "simple"; - User = "litellm"; - Group = "litellm"; - Restart = "always"; - RestartSec = 5; - }; - }; - - users.users.litellm = { - isSystemUser = true; - group = "litellm"; - extraGroups = [ "keys" ]; - }; - users.groups.litellm = { }; - users.groups.keys = { }; - - environment.etc."litellm/config.yaml".text = builtins.toJSON { - general_settings = { - master_key = "os.environ/LITELLM_MASTER_KEY"; - database_url = "os.environ/DATABASE_URL"; - }; - litellm_settings = { - check_provider_endpoints = true; - drop_params = true; - modify_params = true; - }; - model_list = - (builtins.map (m: { - model_name = "azure-${m}"; - litellm_params = { - model = "azure/${m}"; - api_base = "http://100.64.0.8:9010/azure"; - api_version = "2025-04-01-preview"; - api_key = "na"; - }; - }) azureModels) - ++ azureReasoningAliases; - }; - }; - }; - }; - }; -} diff --git a/hosts/h001/mods/default.nix b/hosts/h001/mods/default.nix index ee6f6a9f..54795945 100644 --- a/hosts/h001/mods/default.nix +++ b/hosts/h001/mods/default.nix @@ -4,6 +4,7 @@ { imports = [ ./litellm.nix + ./litellm-public.nix ./nixarr.nix ./hardware-transcoding.nix ./monitoring_hub.nix diff --git a/hosts/h001/mods/litellm-public.nix b/hosts/h001/mods/litellm-public.nix new file mode 100644 index 00000000..41097604 --- /dev/null +++ b/hosts/h001/mods/litellm-public.nix @@ -0,0 +1,116 @@ +{ + inputs, + pkgs, + lib, + ... +}: +let + declaration = "services/misc/litellm.nix"; + nixpkgsLitellm = inputs.litellm-nixpkgs; + pkgsLitellm = import nixpkgsLitellm { + inherit (pkgs) system; + config.allowUnfree = true; + }; + port = 8095; + + azureModels = [ + "gpt-5.2-2025-12-11" + "gpt-5.1-2025-11-13" + "gpt-4o-2024-05-13" + "gpt-4.1-2025-04-14" + "gpt-4.1-mini-2025-04-14" + "gpt-5-nano-2025-08-07" + "gpt-5-mini-2025-08-07" + "gpt-5-2025-08-07" + ]; +in +{ + options = { }; + config = { + networking.firewall.interfaces."tailscale0".allowedTCPPorts = [ port ]; + + systemd.services.litellm-public = { + description = "LiteLLM Public Proxy (Azure models only)"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = { + SCARF_NO_ANALYTICS = "True"; + DO_NOT_TRACK = "True"; + ANONYMIZED_TELEMETRY = "False"; + }; + + serviceConfig = { + Type = "simple"; + User = "litellm-public"; + Group = "litellm-public"; + StateDirectory = "litellm-public"; + ExecStart = "${pkgsLitellm.litellm}/bin/litellm --config /etc/litellm-public/config.yaml --host 0.0.0.0 --port ${toString port}"; + Restart = "always"; + RestartSec = 5; + }; + }; + + users.users.litellm-public = { + isSystemUser = true; + group = "litellm-public"; + }; + users.groups.litellm-public = { }; + + environment.etc."litellm-public/config.yaml".text = lib.generators.toYAML { } { + litellm_settings = { + check_provider_endpoints = true; + drop_params = true; + modify_params = true; + }; + model_list = + (builtins.map (m: { + model_name = "azure-${m}"; + litellm_params = { + model = "azure/${m}"; + api_base = "http://100.64.0.8:9010/azure"; + api_version = "2025-04-01-preview"; + api_key = "na"; + }; + }) azureModels) + ++ [ + { + model_name = "azure-gpt-5.2-low"; + litellm_params = { + model = "azure/gpt-5.2-2025-12-11"; + api_base = "http://100.64.0.8:9010/azure"; + api_version = "2025-04-01-preview"; + api_key = "na"; + extra_body = { + reasoning_effort = "low"; + }; + }; + } + { + model_name = "azure-gpt-5.2-medium"; + litellm_params = { + model = "azure/gpt-5.2-2025-12-11"; + api_base = "http://100.64.0.8:9010/azure"; + api_version = "2025-04-01-preview"; + api_key = "na"; + extra_body = { + reasoning_effort = "medium"; + }; + }; + } + { + model_name = "azure-gpt-5.2-high"; + litellm_params = { + model = "azure/gpt-5.2-2025-12-11"; + api_base = "http://100.64.0.8:9010/azure"; + api_version = "2025-04-01-preview"; + api_key = "na"; + extra_body = { + reasoning_effort = "high"; + }; + }; + } + ]; + }; + }; +} diff --git a/hosts/oracle/o001/nginx.nix b/hosts/oracle/o001/nginx.nix index 2f47aa3e..fc44d9af 100644 --- a/hosts/oracle/o001/nginx.nix +++ b/hosts/oracle/o001/nginx.nix @@ -1,6 +1,10 @@ { + config, ... }: +let + apiKeyFile = config.age.secrets.litellm_public_api_key.path; +in { # JUST A TEST TODO remove containers.wasabi = { @@ -234,6 +238,10 @@ locations."/" = { proxyWebsockets = true; proxyPass = "http://100.64.0.13:8095"; + extraConfig = '' + # API key auth - secret file contains: if ($http_authorization != "Bearer sk-xxx") { return 401; } + include ${apiKeyFile}; + ''; }; };