{ self, config, pkgs, ... }: {
  age.secrets.smtp_password.file = "${self}/secrets/nullhex-smtp.age";
  age.secrets.smtp_password.owner = "nobody";

  # grafana configuration
  services.grafana = {
    enable = true;
    settings = {
      smtp = {
        enabled = false;
        password = "$__file{/run/secrets/smtp_password}";
        host = "smtp.nullhex.com";
        user = "ops@sealight.xyz";
        from_address = "ops@sealight.xyz";
      };
      server = {
        domain = "stats.mossnet.lan";
        http_port = 2342;
        http_addr = "127.0.0.1";
      };
    };
    # TODO enable
    # provision.alerting.rules.path = "${self}/hosts/profiles/monitoring/alert-rules.yaml";
  };

  # nginx reverse proxy
  services.nginx.recommendedProxySettings = true; # Needed for new grafana versions 
  services.nginx.virtualHosts.${config.services.grafana.settings.server.domain} = {
    locations."/" = {
      proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
      proxyWebsockets = true;
    };
  };

  services.prometheus = {
    enable = true;
    port = 9001;
    exporters = {
      node = {
        enable = true;
        enabledCollectors = [ "systemd" ];
        port = 9002;
      };
      dnsmasq = {
        enable = false;
        port = 9153;
      };
    };
    scrapeConfigs = [
      {
        job_name = "box";
        static_configs = [{ targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ]; }];
      }
      {
        job_name = "dns";
        static_configs = [{ targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.dnsmasq.port}" ]; }];
      }
    ];
  };

  # services.loki = {
  #   enable = true;
  #   configFile = /var/loki-config.yaml;
  # };

  #systemd.services.promtail = {
  #  description = "Promtail service for Loki";
  #  wantedBy = [ "multi-user.target" ];

  #  serviceConfig = {
  #    ExecStart = ''
  #      ${pkgs.grafana-loki}/bin/promtail --config.file ${/var/promtail.yaml}
  #    '';
  #  };
  #};
}