monitoring :3

hosts/ocelot/default.nix

@@ -12,6 +12,7 @@
     internal_IPv4 = "10.69.0.5";
     internal_IPv6 = "fe80::8400:ff:fe8e:470d";
   };
+  cherrykitten.prometheus.client.enable = true;
 
   networking = {
     nameservers = [

hosts/serval/default.nix

@@ -3,6 +3,14 @@
 
   fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; };
   cherrykitten.backups.enable = true;
+  cherrykitten.prometheus = {
+    server.enable = true;
+    client.enable = true;
+  };
+  cherrykitten.grafana = {
+    enable = true;
+    hostname = "graph.cherrykitten.dev";
+  };
   cherrykitten.network = {
     public_IPv4 = "116.203.116.228";
     public_IPv6 = "2a01:4f8:1c1b:5db9::1";

modules/nixos/cherrykitten/default.nix

@@ -1,4 +1,4 @@
-{ config, lib, ... }:
+{ config, lib, nodes, ... }:
 let
   cfg = config.cherrykitten;
 in
@@ -23,4 +23,10 @@ in
       };
     };
   };
+  config = {
+    networking.hosts = {
+      "${nodes.ocelot.config.cherrykitten.network.internal_IPv4 }" = [ "ocelot" ];
+      "${nodes.serval.config.cherrykitten.network.internal_IPv4 }" = [ "serval" ];
+    };
+  };
 }

modules/nixos/grafana/default.nix

@@ -0,0 +1,40 @@
+{ lib, config, ... }:
+
+let cfg = config.cherrykitten.grafana;
+
+in with lib; {
+  options.cherrykitten.grafana = {
+    enable = mkEnableOption "Grafana";
+    hostname = mkOption {
+      type = types.str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.grafana = {
+      enable = true;
+      settings = {
+
+        server = {
+          domain = cfg.hostname;
+          http_port = 8571;
+        };
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [ 80 443 ];
+    services.nginx.enable = true;
+    services.nginx.virtualHosts.${cfg.hostname} = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:8571";
+        proxyWebsockets = true;
+        recommendedProxySettings = true;
+      };
+      extraConfig = ''
+        access_log /var/log/nginx/grafana.access.log;
+      '';
+    };
+  };
+}

modules/nixos/prometheus/client.nix

@@ -0,0 +1,21 @@
+{ lib, config, ... }:
+
+let cfg = config.cherrykitten.prometheus.client;
+
+in {
+  options.cherrykitten.prometheus.client = {
+    enable = lib.mkEnableOption "Prometheus client";
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.prometheus.exporters = {
+      node = {
+        enable = true;
+        port = 9100;
+        enabledCollectors = [ "systemd" ];
+        openFirewall = true;
+        listenAddress = config.cherrykitten.network.internal_IPv4;
+      };
+    };
+  };
+}

modules/nixos/prometheus/default.nix

@@ -0,0 +1,6 @@
+{ ... }: {
+  imports = [
+    ./client.nix
+    ./server.nix
+  ];
+}

modules/nixos/prometheus/server.nix

@@ -0,0 +1,34 @@
+{ nodes, lib, config, ... }:
+
+let cfg = config.cherrykitten.prometheus.server;
+
+in {
+  options.cherrykitten.prometheus.server = {
+    enable = lib.mkEnableOption "Prometheus server";
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    services.prometheus = {
+      enable = true;
+      retentionTime = "30d";
+
+      scrapeConfigs = [
+        {
+          job_name = "node";
+          honor_labels = true;
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "instance";
+              regex = "([^:]+)(:[0-9]+)?";
+            }
+          ];
+          static_configs = [{
+            targets = [ "serval:9100" "ocelot:9100" ];
+          }];
+        }
+      ];
+    };
+  };
+}