diff --git a/hosts/h001/mods/monitoring.nix b/hosts/h001/mods/monitoring.nix index 04cc962..f9b31bc 100644 --- a/hosts/h001/mods/monitoring.nix +++ b/hosts/h001/mods/monitoring.nix @@ -1,4 +1,5 @@ { + config, ... }: { @@ -9,7 +10,14 @@ { job_name = "node"; static_configs = [ - { targets = [ "localhost:9100" ]; } + { + targets = [ "localhost:9100" ]; + labels.instance = config.networking.hostName; # h001 + } + # { + # targets = [ "http://lio.net.joshuabell.xyz:9100" ]; + # labels.instance = "lio"; + # } ]; } ]; @@ -17,9 +25,11 @@ services.grafana = { enable = true; + dataDir = "/var/lib/grafana"; settings = { server = { http_port = 3001; + http_addr = "127.0.0.1"; serve_from_sub_path = true; domain = "h001.net.joshuabell.xyz"; root_url = "http://h001.net.joshuabell.xyz/grafana/"; @@ -27,6 +37,105 @@ enable_gzip = true; }; }; + provision = { + datasources.settings.datasources = [ + { + name = "Prometheus"; + type = "prometheus"; + url = "http://localhost:9090"; + access = "proxy"; + isDefault = true; # Set as default, if you want + } + { + name = "Loki"; + type = "loki"; + url = "http://localhost:3100"; + access = "proxy"; + isDefault = false; + } + ]; + }; + }; + + # Loki for log aggregation + systemd.tmpfiles.rules = [ + "d /var/lib/loki 0755 loki loki -" + "d /var/lib/loki/chunks 0755 loki loki -" + "d /var/lib/loki/rules 0755 loki loki -" + "d /var/lib/loki/compactor 0755 loki loki -" + ]; + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + + server = { + http_listen_port = 3100; + }; + + common = { + path_prefix = "/var/lib/loki"; + storage = { + filesystem = { + chunks_directory = "/var/lib/loki/chunks"; + rules_directory = "/var/lib/loki/rules"; + }; + }; + replication_factor = 1; + ring = { + kvstore = { + store = "inmemory"; + }; + }; + }; + + schema_config = { + configs = [ + { + from = "2023-01-01"; + store = "boltdb-shipper"; + object_store = "filesystem"; + schema = "v12"; # Updated schema version + index = { + prefix = "index_"; + period = "24h"; # Set to 24h period as recommended + }; + } + ]; + }; + + limits_config = { + allow_structured_metadata = false; # Disable structured metadata until we upgrade to v13 + }; + + ruler = { + alertmanager_url = "http://localhost:9093"; + storage = { + type = "local"; + local = { + directory = "/var/lib/loki/rules"; + }; + }; + rule_path = "/var/lib/loki/rules"; + ring = { + kvstore = { + store = "inmemory"; + }; + }; + }; + + compactor = { + working_directory = "/var/lib/loki/compactor"; # Set working directory + retention_enabled = true; + compaction_interval = "5m"; + delete_request_store = "filesystem"; # Add this line for retention configuration + delete_request_cancel_period = "24h"; + }; + + analytics = { + reporting_enabled = false; + }; + }; }; }; } diff --git a/hosts/h001/mods/reporting.nix b/hosts/h001/mods/reporting.nix index 48e19e1..dc6a3a0 100644 --- a/hosts/h001/mods/reporting.nix +++ b/hosts/h001/mods/reporting.nix @@ -1,8 +1,69 @@ { + config, ... }: { config = { - services.prometheus.exporters.node.enable = true; + services.prometheus.exporters.node.enable = true; # port 9080 + # Create necessary directories with appropriate permissions + systemd.tmpfiles.rules = [ + "d /tmp/positions 1777 - - -" # World-writable directory for positions file + "f /tmp/positions.yaml 0666 - - -" # World-writable positions file + ]; + users.groups.systemd-journal.members = [ "promtail" ]; + services.promtail = { + enable = true; + extraFlags = [ + "-config.expand-env=true" + ]; + configuration = { + server = { + http_listen_port = 9080; + grpc_listen_port = 0; + }; + positions = { + filename = "/tmp/positions.yaml"; # Changed from /var/lib/promtail/positions.yaml + }; + clients = [ + { + url = "http://localhost:3100/loki/api/v1/push"; # Points to your Loki instance + } + ]; + scrape_configs = [ + { + job_name = "journal"; + journal = { + json = false; + max_age = "12h"; + path = "/var/log/journal"; + labels = { + job = "systemd-journal"; + host = "${config.networking.hostName}"; + }; + }; + relabel_configs = [ + { + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + } + ]; + } + # Simple file-based logs as a fallback + { + job_name = "system"; + static_configs = [ + { + targets = [ "localhost" ]; + labels = { + job = "syslog"; + host = "${config.networking.hostName}"; + __path__ = "/var/log/syslog"; + }; + } + ]; + } + ]; + }; + }; }; }