Switch litellm public to API key, move secret to o001, use module
This commit is contained in:
Joshua Bell
parent
d706c122b1
commit
eebf354173
9 changed files with 141 additions and 313 deletions
|
|
@ -4,7 +4,6 @@
|
|||
{
|
||||
imports = [
|
||||
./forgejo.nix
|
||||
./litellm-public.nix
|
||||
./opengist.nix
|
||||
./zitadel.nix
|
||||
];
|
||||
|
|
|
|||
|
|
@ -1,295 +0,0 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
inputs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
name = "litellm-public";
|
||||
|
||||
hostDataDir = "/var/lib/${name}";
|
||||
|
||||
hostAddress = "10.0.0.1";
|
||||
containerAddress = "10.0.0.4";
|
||||
hostAddress6 = "fc00::1";
|
||||
containerAddress6 = "fc00::4";
|
||||
|
||||
litellmNixpkgs = inputs.litellm-nixpkgs;
|
||||
pkgsLitellm = import litellmNixpkgs {
|
||||
system = "x86_64-linux";
|
||||
config.allowUnfree = true;
|
||||
};
|
||||
|
||||
containerPort = 4000;
|
||||
externalPort = 8095;
|
||||
|
||||
hasSecret =
|
||||
secret:
|
||||
let
|
||||
secrets = config.age.secrets or { };
|
||||
in
|
||||
secrets ? ${secret} && secrets.${secret} != null;
|
||||
|
||||
binds = [
|
||||
{
|
||||
host = "${hostDataDir}/postgres";
|
||||
container = "/var/lib/postgresql/17";
|
||||
user = "postgres";
|
||||
uid = config.ids.uids.postgres;
|
||||
gid = config.ids.gids.postgres;
|
||||
}
|
||||
{
|
||||
host = "${hostDataDir}/backups/postgres";
|
||||
container = "/var/backup/postgresql";
|
||||
user = "postgres";
|
||||
uid = config.ids.uids.postgres;
|
||||
gid = config.ids.gids.postgres;
|
||||
}
|
||||
]
|
||||
++ lib.optionals (hasSecret "litellm_public_master_key") [
|
||||
{
|
||||
host = config.age.secrets.litellm_public_master_key.path;
|
||||
container = "/var/secrets/litellm_master_key";
|
||||
readOnly = true;
|
||||
}
|
||||
];
|
||||
|
||||
bindsWithUsers = lib.filter (b: b ? user) binds;
|
||||
uniqueUsers = lib.foldl' (
|
||||
acc: bind: if lib.lists.any (item: item.user == bind.user) acc then acc else acc ++ [ bind ]
|
||||
) [ ] bindsWithUsers;
|
||||
|
||||
users = {
|
||||
users = lib.listToAttrs (
|
||||
lib.map (u: {
|
||||
name = u.user;
|
||||
value = {
|
||||
isSystemUser = true;
|
||||
name = u.user;
|
||||
uid = u.uid;
|
||||
group = u.user;
|
||||
};
|
||||
}) uniqueUsers
|
||||
);
|
||||
|
||||
groups = lib.listToAttrs (
|
||||
lib.map (g: {
|
||||
name = g.user;
|
||||
value.gid = g.gid;
|
||||
}) uniqueUsers
|
||||
);
|
||||
};
|
||||
|
||||
azureModels = [
|
||||
"gpt-5.2-2025-12-11"
|
||||
"gpt-5.1-2025-11-13"
|
||||
"gpt-4o-2024-05-13"
|
||||
"gpt-4.1-2025-04-14"
|
||||
"gpt-4.1-mini-2025-04-14"
|
||||
"gpt-5-nano-2025-08-07"
|
||||
"gpt-5-mini-2025-08-07"
|
||||
"gpt-5-2025-08-07"
|
||||
];
|
||||
|
||||
azureReasoningAliases = [
|
||||
{
|
||||
model_name = "azure-gpt-5.2-low";
|
||||
litellm_params = {
|
||||
model = "azure/gpt-5.2-2025-12-11";
|
||||
api_base = "http://100.64.0.8:9010/azure";
|
||||
api_version = "2025-04-01-preview";
|
||||
api_key = "na";
|
||||
extra_body = {
|
||||
reasoning_effort = "low";
|
||||
};
|
||||
};
|
||||
}
|
||||
{
|
||||
model_name = "azure-gpt-5.2-medium";
|
||||
litellm_params = {
|
||||
model = "azure/gpt-5.2-2025-12-11";
|
||||
api_base = "http://100.64.0.8:9010/azure";
|
||||
api_version = "2025-04-01-preview";
|
||||
api_key = "na";
|
||||
extra_body = {
|
||||
reasoning_effort = "medium";
|
||||
};
|
||||
};
|
||||
}
|
||||
{
|
||||
model_name = "azure-gpt-5.2-high";
|
||||
litellm_params = {
|
||||
model = "azure/gpt-5.2-2025-12-11";
|
||||
api_base = "http://100.64.0.8:9010/azure";
|
||||
api_version = "2025-04-01-preview";
|
||||
api_key = "na";
|
||||
extra_body = {
|
||||
reasoning_effort = "high";
|
||||
};
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
in
|
||||
{
|
||||
options = { };
|
||||
config = {
|
||||
networking.firewall.interfaces."tailscale0".allowedTCPPorts = [ externalPort ];
|
||||
|
||||
services.nginx.virtualHosts."llm.joshuabell.xyz" = {
|
||||
listen = [
|
||||
{
|
||||
addr = "0.0.0.0";
|
||||
port = externalPort;
|
||||
}
|
||||
];
|
||||
locations = {
|
||||
"/" = {
|
||||
proxyWebsockets = true;
|
||||
recommendedProxySettings = true;
|
||||
proxyPass = "http://${containerAddress}:${toString containerPort}";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
inherit users;
|
||||
|
||||
system.activationScripts."createDirsFor${name}" = ''
|
||||
${lib.concatStringsSep "\n" (
|
||||
lib.map (bind: ''
|
||||
mkdir -p ${bind.host}
|
||||
chown -R ${toString bind.user}:${toString bind.gid} ${bind.host}
|
||||
chmod -R 750 ${bind.host}
|
||||
'') bindsWithUsers
|
||||
)}
|
||||
'';
|
||||
|
||||
containers.${name} = {
|
||||
ephemeral = true;
|
||||
autoStart = true;
|
||||
privateNetwork = true;
|
||||
hostAddress = hostAddress;
|
||||
localAddress = containerAddress;
|
||||
hostAddress6 = hostAddress6;
|
||||
localAddress6 = containerAddress6;
|
||||
bindMounts = lib.foldl (
|
||||
acc: bind:
|
||||
{
|
||||
"${bind.container}" = {
|
||||
hostPath = bind.host;
|
||||
isReadOnly = bind.readOnly or false;
|
||||
};
|
||||
}
|
||||
// acc
|
||||
) { } binds;
|
||||
nixpkgs = litellmNixpkgs;
|
||||
config =
|
||||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
{
|
||||
config = {
|
||||
system.stateVersion = "25.05";
|
||||
|
||||
networking = {
|
||||
firewall = {
|
||||
enable = true;
|
||||
allowedTCPPorts = [ containerPort ];
|
||||
};
|
||||
useHostResolvConf = lib.mkForce false;
|
||||
};
|
||||
services.resolved.enable = true;
|
||||
|
||||
inherit users;
|
||||
|
||||
services.postgresql = {
|
||||
enable = true;
|
||||
package = pkgs.postgresql_17.withJIT;
|
||||
enableJIT = true;
|
||||
authentication = ''
|
||||
local all all trust
|
||||
host all all 127.0.0.1/8 trust
|
||||
host all all ::1/128 trust
|
||||
'';
|
||||
ensureDatabases = [ "litellm" ];
|
||||
ensureUsers = [
|
||||
{
|
||||
name = "litellm";
|
||||
ensureDBOwnership = true;
|
||||
ensureClauses.login = true;
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
services.postgresqlBackup = {
|
||||
enable = true;
|
||||
};
|
||||
|
||||
systemd.services.litellm = {
|
||||
description = "LiteLLM Public Proxy";
|
||||
after = [
|
||||
"network.target"
|
||||
"postgresql.service"
|
||||
];
|
||||
requires = [ "postgresql.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
||||
environment = {
|
||||
DATABASE_URL = "postgresql://litellm@/litellm?host=/var/run/postgresql";
|
||||
SCARF_NO_ANALYTICS = "True";
|
||||
DO_NOT_TRACK = "True";
|
||||
ANONYMIZED_TELEMETRY = "False";
|
||||
};
|
||||
|
||||
script = ''
|
||||
export LITELLM_MASTER_KEY="$(cat /var/secrets/litellm_master_key)"
|
||||
exec ${pkgsLitellm.litellm}/bin/litellm --config /etc/litellm/config.yaml --host 0.0.0.0 --port ${toString containerPort}
|
||||
'';
|
||||
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
User = "litellm";
|
||||
Group = "litellm";
|
||||
Restart = "always";
|
||||
RestartSec = 5;
|
||||
};
|
||||
};
|
||||
|
||||
users.users.litellm = {
|
||||
isSystemUser = true;
|
||||
group = "litellm";
|
||||
extraGroups = [ "keys" ];
|
||||
};
|
||||
users.groups.litellm = { };
|
||||
users.groups.keys = { };
|
||||
|
||||
environment.etc."litellm/config.yaml".text = builtins.toJSON {
|
||||
general_settings = {
|
||||
master_key = "os.environ/LITELLM_MASTER_KEY";
|
||||
database_url = "os.environ/DATABASE_URL";
|
||||
};
|
||||
litellm_settings = {
|
||||
check_provider_endpoints = true;
|
||||
drop_params = true;
|
||||
modify_params = true;
|
||||
};
|
||||
model_list =
|
||||
(builtins.map (m: {
|
||||
model_name = "azure-${m}";
|
||||
litellm_params = {
|
||||
model = "azure/${m}";
|
||||
api_base = "http://100.64.0.8:9010/azure";
|
||||
api_version = "2025-04-01-preview";
|
||||
api_key = "na";
|
||||
};
|
||||
}) azureModels)
|
||||
++ azureReasoningAliases;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@
|
|||
{
|
||||
imports = [
|
||||
./litellm.nix
|
||||
./litellm-public.nix
|
||||
./nixarr.nix
|
||||
./hardware-transcoding.nix
|
||||
./monitoring_hub.nix
|
||||
|
|
|
|||
116
hosts/h001/mods/litellm-public.nix
Normal file
116
hosts/h001/mods/litellm-public.nix
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
{
|
||||
inputs,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
declaration = "services/misc/litellm.nix";
|
||||
nixpkgsLitellm = inputs.litellm-nixpkgs;
|
||||
pkgsLitellm = import nixpkgsLitellm {
|
||||
inherit (pkgs) system;
|
||||
config.allowUnfree = true;
|
||||
};
|
||||
port = 8095;
|
||||
|
||||
azureModels = [
|
||||
"gpt-5.2-2025-12-11"
|
||||
"gpt-5.1-2025-11-13"
|
||||
"gpt-4o-2024-05-13"
|
||||
"gpt-4.1-2025-04-14"
|
||||
"gpt-4.1-mini-2025-04-14"
|
||||
"gpt-5-nano-2025-08-07"
|
||||
"gpt-5-mini-2025-08-07"
|
||||
"gpt-5-2025-08-07"
|
||||
];
|
||||
in
|
||||
{
|
||||
options = { };
|
||||
config = {
|
||||
networking.firewall.interfaces."tailscale0".allowedTCPPorts = [ port ];
|
||||
|
||||
systemd.services.litellm-public = {
|
||||
description = "LiteLLM Public Proxy (Azure models only)";
|
||||
after = [ "network.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
||||
environment = {
|
||||
SCARF_NO_ANALYTICS = "True";
|
||||
DO_NOT_TRACK = "True";
|
||||
ANONYMIZED_TELEMETRY = "False";
|
||||
};
|
||||
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
User = "litellm-public";
|
||||
Group = "litellm-public";
|
||||
StateDirectory = "litellm-public";
|
||||
ExecStart = "${pkgsLitellm.litellm}/bin/litellm --config /etc/litellm-public/config.yaml --host 0.0.0.0 --port ${toString port}";
|
||||
Restart = "always";
|
||||
RestartSec = 5;
|
||||
};
|
||||
};
|
||||
|
||||
users.users.litellm-public = {
|
||||
isSystemUser = true;
|
||||
group = "litellm-public";
|
||||
};
|
||||
users.groups.litellm-public = { };
|
||||
|
||||
environment.etc."litellm-public/config.yaml".text = lib.generators.toYAML { } {
|
||||
litellm_settings = {
|
||||
check_provider_endpoints = true;
|
||||
drop_params = true;
|
||||
modify_params = true;
|
||||
};
|
||||
model_list =
|
||||
(builtins.map (m: {
|
||||
model_name = "azure-${m}";
|
||||
litellm_params = {
|
||||
model = "azure/${m}";
|
||||
api_base = "http://100.64.0.8:9010/azure";
|
||||
api_version = "2025-04-01-preview";
|
||||
api_key = "na";
|
||||
};
|
||||
}) azureModels)
|
||||
++ [
|
||||
{
|
||||
model_name = "azure-gpt-5.2-low";
|
||||
litellm_params = {
|
||||
model = "azure/gpt-5.2-2025-12-11";
|
||||
api_base = "http://100.64.0.8:9010/azure";
|
||||
api_version = "2025-04-01-preview";
|
||||
api_key = "na";
|
||||
extra_body = {
|
||||
reasoning_effort = "low";
|
||||
};
|
||||
};
|
||||
}
|
||||
{
|
||||
model_name = "azure-gpt-5.2-medium";
|
||||
litellm_params = {
|
||||
model = "azure/gpt-5.2-2025-12-11";
|
||||
api_base = "http://100.64.0.8:9010/azure";
|
||||
api_version = "2025-04-01-preview";
|
||||
api_key = "na";
|
||||
extra_body = {
|
||||
reasoning_effort = "medium";
|
||||
};
|
||||
};
|
||||
}
|
||||
{
|
||||
model_name = "azure-gpt-5.2-high";
|
||||
litellm_params = {
|
||||
model = "azure/gpt-5.2-2025-12-11";
|
||||
api_base = "http://100.64.0.8:9010/azure";
|
||||
api_version = "2025-04-01-preview";
|
||||
api_key = "na";
|
||||
extra_body = {
|
||||
reasoning_effort = "high";
|
||||
};
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -1,6 +1,10 @@
|
|||
{
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
apiKeyFile = config.age.secrets.litellm_public_api_key.path;
|
||||
in
|
||||
{
|
||||
# JUST A TEST TODO remove
|
||||
containers.wasabi = {
|
||||
|
|
@ -234,6 +238,10 @@
|
|||
locations."/" = {
|
||||
proxyWebsockets = true;
|
||||
proxyPass = "http://100.64.0.13:8095";
|
||||
extraConfig = ''
|
||||
# API key auth - secret file contains: if ($http_authorization != "Bearer sk-xxx") { return 401; }
|
||||
include ${apiKeyFile};
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue