{ self, config, pkgs, ... }: {
  age.secrets.nullhex-smtp.file = "${self}/secrets/nullhex-smtp.age";
  age.secrets.nullhex-smtp.owner = "grafana";

  # grafana configuration
  services.grafana = {
    enable = true;
    settings = {
      smtp = {
        enabled = true;
        password = "$__file{/run/agenix/nullhex-smtp}";
        host = "smtp.nullhex.com";
        user = "ops@sealight.xyz";
        from_address = "ops@sealight.xyz";
      };
      server = {
        domain = "stats.mossnet.lan";
        http_addr = "127.0.0.1";
        http_port = 2342;
        root_url = "http://stats.mossnet.lan/";
      };
    };
    # TODO enable
    # provision.alerting.rules.path = "${self}/hosts/profiles/monitoring/alert-rules.yaml";
  };

  # nginx reverse proxy
  # services.nginx.recommendedProxySettings = true; # Needed for new grafana versions 
  services.nginx.virtualHosts.${config.services.grafana.settings.server.domain} = {
    locations."/" = {
      proxyPass = "http://127.0.0.1:2342";
      proxyWebsockets = true;
    };
  };

    services.postgresql = {
      ensureUsers = [{
        name = "grafana";
        # TODO this is deprecated 
        # Need to translate this to 
        # systemd.services.postgresql.postStart
        # or initialScript 
        ensurePermissions = {
          "ALL TABLES IN SCHEMA public" = "SELECT";
          "DATABASE wallabag" = "CONNECT";
          "DATABASE ulogger" = "CONNECT";
          "DATABASE photoprism" = "CONNECT";
        };
      }];
    };

  services.prometheus = {
    enable = true;
    port = 9001;
    exporters = {
      node = {
        enable = true;
        enabledCollectors = [ "systemd" ];
        port = 9002;
      };
      dnsmasq = {
        enable = false;
        port = 9153;
      };
    };
    scrapeConfigs = [
      {
        job_name = "box";
        static_configs = [{ targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ]; }];
      }
      {
        job_name = "dns";
        static_configs = [{ targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.dnsmasq.port}" ]; }];
      }
    ];
  };

  # services.loki = {
  #   enable = true;
  #   configFile = /var/loki-config.yaml;
  # };

  #systemd.services.promtail = {
  #  description = "Promtail service for Loki";
  #  wantedBy = [ "multi-user.target" ];

  #  serviceConfig = {
  #    ExecStart = ''
  #      ${pkgs.grafana-loki}/bin/promtail --config.file ${/var/promtail.yaml}
  #    '';
  #  };
  #};
}