updating some stuff, trying to get woodpecker, and grafana alerts off

the ground

hosts/curve/default.nix

@@ -24,7 +24,8 @@
     services.NetworkManager-wait-online.wantedBy = pkgs.lib.mkForce [ ]; # Normally ["network-online.target"]
   };
 
-  programs.gnupg.agent.pinentryFlavor = "gtk2";
+  programs.gnupg.agent.enable = true;
+  programs.gnupg.agent.pinentryFlavor = "gnome3";
 
   fileSystems."/mnt/ftp" = {
     device = "192.168.1.240:/home/ftp";

hosts/profiles/desktop/default.nix

@@ -58,6 +58,8 @@ in
     unstable.sublime-music
     unstable.nheko
     unstable.tootle
+    newsflash
+    liferea
 
     scrot
     ripcord
@@ -69,7 +71,6 @@ in
     calibre
     fractal
     mpv
-    newsflash
     zeal
     xclip
     xdotool

hosts/profiles/monitoring/alert-rules.yaml

@@ -0,0 +1,300 @@
+groups:
+- name: hosts
+  rules:
+  - alert: HostOutOfMemory
+    expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host out of memory (instance {{ $labels.instance }})
+      description: "Node memory is filling up (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostMemoryUnderMemoryPressure
+    expr: rate(node_vmstat_pgmajfault[1m]) > 1000
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host memory under memory pressure (instance {{ $labels.instance }})
+      description: "The node is under heavy memory pressure. High rate of major page faults\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostUnusualNetworkThroughputIn
+    expr: sum by (instance) (rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host unusual network throughput in (instance {{ $labels.instance }})
+      description: "Host network interfaces are probably receiving too much data (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostUnusualNetworkThroughputOut
+    expr: sum by (instance) (rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host unusual network throughput out (instance {{ $labels.instance }})
+      description: "Host network interfaces are probably sending too much data (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+#  - alert: HostUnusualDiskReadRate
+#    expr: sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 100
+#    for: 5m
+#    labels:
+#      severity: warning
+#    annotations:
+#      summary: Host unusual disk read rate (instance {{ $labels.instance }})
+#      description: "Disk is probably reading too much data (> 50 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+#
+#  - alert: HostUnusualDiskWriteRate
+#    expr: sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50
+#    for: 2m
+#    labels:
+#      severity: warning
+#    annotations:
+#      summary: Host unusual disk write rate (instance {{ $labels.instance }})
+#      description: "Disk is probably writing too much data (> 50 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  # Please add ignored mountpoints in node_exporter parameters like
+  # "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)".
+  # Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users.
+  - alert: HostOutOfDiskSpace
+    expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host out of disk space (instance {{ $labels.instance }})
+      description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  # Please add ignored mountpoints in node_exporter parameters like
+  # "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)".
+  # Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users.
+  - alert: HostDiskWillFillIn24Hours
+    expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
+      description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostOutOfInodes
+    expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint="/rootfs"} * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly{mountpoint="/rootfs"} == 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host out of inodes (instance {{ $labels.instance }})
+      description: "Disk is almost running out of available inodes (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostInodesWillFillIn24Hours
+    expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint="/rootfs"} * 100 < 10 and predict_linear(node_filesystem_files_free{mountpoint="/rootfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly{mountpoint="/rootfs"} == 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host inodes will fill in 24 hours (instance {{ $labels.instance }})
+      description: "Filesystem is predicted to run out of inodes within the next 24 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+#  - alert: HostUnusualDiskReadLatency
+#    expr: rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0
+#    for: 2m
+#    labels:
+#      severity: warning
+#    annotations:
+#      summary: Host unusual disk read latency (instance {{ $labels.instance }})
+#      description: "Disk latency is growing (read operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+#
+#  - alert: HostUnusualDiskWriteLatency
+#    expr: rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 2 and rate(node_disk_writes_completed_total[1m]) > 0
+#    for: 2m
+#    labels:
+#      severity: warning
+#    annotations:
+#      summary: Host unusual disk write latency (instance {{ $labels.instance }})
+#      description: "Disk latency is growing (write operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostHighCpuLoad
+    expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 90
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host high CPU load (instance {{ $labels.instance }})
+      description: "CPU load is > 90%\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostCpuStealNoisyNeighbor
+    expr: avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
+      description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  # 10000 context switches is an arbitrary number.
+  # Alert threshold depends on nature of application.
+  # Please read: https://github.com/samber/awesome-prometheus-alerts/issues/58
+  - alert: HostContextSwitching
+    expr: (rate(node_context_switches_total[5m])) / (count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 10000
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host context switching (instance {{ $labels.instance }})
+      description: "Context switching is growing on node (> 1000 / s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostSwapIsFillingUp
+    expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host swap is filling up (instance {{ $labels.instance }})
+      description: "Swap is filling up (>80%)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostSystemdServiceCrashed
+    expr: node_systemd_unit_state{state="failed"} == 1
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host systemd service crashed (instance {{ $labels.instance }})
+      description: "systemd service crashed: {{ $labels.name }} on {{ $labels.job }}"
+
+  - alert: HostPhysicalComponentTooHot
+    expr: node_hwmon_temp_celsius > 75
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host physical component too hot (instance {{ $labels.instance }})
+      description: "Physical hardware component too hot\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostNodeOvertemperatureAlarm
+    expr: node_hwmon_temp_crit_alarm_celsius == 1
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Host node overtemperature alarm (instance {{ $labels.instance }})
+      description: "Physical node temperature alarm triggered\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostRaidArrayGotInactive
+    expr: node_md_state{state="inactive"} > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Host RAID array got inactive (instance {{ $labels.instance }})
+      description: "RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostRaidDiskFailure
+    expr: node_md_disks{state="failed"} > 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host RAID disk failure (instance {{ $labels.instance }})
+      description: "At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostKernelVersionDeviations
+    expr: count(sum(label_replace(node_uname_info, "kernel", "$1", "release", "([0-9]+.[0-9]+.[0-9]+).*")) by (kernel)) > 2
+    for: 6h
+    labels:
+      severity: warning
+    annotations:
+      summary: Host kernel version deviations (instance {{ $labels.instance }})
+      description: "Different kernel versions are running\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostOomKillDetected
+    expr: increase(node_vmstat_oom_kill[1m]) > 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host OOM kill detected (instance {{ $labels.instance }})
+      description: "OOM kill detected\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostEdacCorrectableErrorsDetected
+    expr: increase(node_edac_correctable_errors_total[1m]) > 0
+    for: 0m
+    labels:
+      severity: info
+    annotations:
+      summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }})
+      description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} correctable memory errors reported by EDAC in the last 5 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostEdacUncorrectableErrorsDetected
+    expr: node_edac_uncorrectable_errors_total > 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})
+      description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostNetworkReceiveErrors
+    expr: rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host Network Receive Errors (instance {{ $labels.instance }})
+      description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last five minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostNetworkTransmitErrors
+    expr: rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host Network Transmit Errors (instance {{ $labels.instance }})
+      description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last five minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostNetworkInterfaceSaturated
+    expr: (rate(node_network_receive_bytes_total{device!~"^tap.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*"} > 0.8
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host Network Interface Saturated (instance {{ $labels.instance }})
+      description: "The network interface \"{{ $labels.interface }}\" on \"{{ $labels.instance }}\" is getting overloaded.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostConntrackLimit
+    expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host conntrack limit (instance {{ $labels.instance }})
+      description: "The number of conntrack is approching limit\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostClockSkew
+    expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host clock skew (instance {{ $labels.instance }})
+      description: "Clock skew detected. Clock is out of sync.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: HostClockNotSynchronising
+    expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host clock not synchronising (instance {{ $labels.instance }})
+      description: "Clock not synchronising.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    # 26 hours since last backup
+  - alert: NoRecentBackup
+    expr: (time()*1000 - borgbackup_last_backup) > (1000 * 60 * 60 * 26) 
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: No backup in the last 26 hours (instance {{ $labels.exported_job }})
+      description: "to repository {{ $labels.repo }}"

hosts/profiles/monitoring/default.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, ... }: {
+{ self, config, pkgs, ... }: {
   # grafana configuration
   services.grafana = {
     enable = true;
@@ -7,6 +7,8 @@
       http_port = 2342;
       http_addr = "127.0.0.1";
     };
+    # TODO enable
+    # provision.alerting.rules.path = "${self}/hosts/profiles/monitoring/alert-rules.yaml";
   };
 
   # nginx reverse proxy

hosts/profiles/woodpecker-agent/default.nix

@@ -0,0 +1,65 @@
+{
+
+  users.users.woodpecker-runner = {
+    isSystemUser = true;
+    group = "woodpecker-runner";
+  };
+  users.groups.woodpecker-runner = { };
+  # Allow the exec runner to write to build with nix
+  nix.allowedUsers = [ "woodpecker-runner" ];
+
+  systemd.services.woodpecker-runner-exec = {
+    enable = true;
+    wantedBy = [ "multi-user.target" ];
+    ### MANUALLY RESTART SERVICE IF CHANGED
+    restartIfChanged = true;
+    confinement.enable = true;
+    confinement.packages = [
+      pkgs.git
+      pkgs.gnutar
+      pkgs.bash
+      pkgs.nixFlakes
+      pkgs.gzip
+    ];
+    path = [
+      pkgs.git
+      pkgs.gnutar
+      pkgs.bash
+      pkgs.nixFlakes
+      pkgs.gzip
+    ];
+    serviceConfig = {
+      Environment = [
+        "WOODPECKER_RUNNER_CAPACITY=2"
+        "WOODPECKER_RUNNER_NAME=woodpecker-agent"
+        "NIX_REMOTE=daemon"
+        "PAGER=cat"
+      ];
+      BindPaths = [
+        "/nix/var/nix/daemon-socket/socket"
+        "/run/nscd/socket"
+        # "/var/lib/woodpecker"
+      ];
+      BindReadOnlyPaths = [
+        "/etc/passwd:/etc/passwd"
+        "/etc/group:/etc/group"
+        "/nix/var/nix/profiles/system/etc/nix:/etc/nix"
+        "${config.environment.etc."ssl/certs/ca-certificates.crt".source}:/etc/ssl/certs/ca-certificates.crt"
+        "${config.environment.etc."ssh/ssh_known_hosts".source}:/etc/ssh/ssh_known_hosts"
+        "${builtins.toFile "ssh_config" ''
+          Host git.sealight.xyz
+          ForwardAgent yes
+        ''}:/etc/ssh/ssh_config"
+        "/etc/machine-id"
+        "/etc/resolv.conf"
+        "/nix/"
+      ];
+      EnvironmentFile = [
+        # /run/agenix/woodpecker-agent-secret
+      ];
+      ExecStart = "${pkgs.woodpecker-agent}/bin/woodpecker-agent";
+      User = "woodpecker-runner";
+      Group = "woodpecker-runner";
+    };
+  };
+}

hosts/profiles/woodpecker-server/default.nix

@@ -0,0 +1,52 @@
+{
+  users.users.woodpecker = {
+    group = woodpecker;
+    description = "woodpecker user";
+    home = "/var/lib/woodpecker";
+    createHome = true;
+  };
+
+  users.groups.woodpecker = { };
+
+  services.nginx.virtualHosts."ci.sealight.xyz" = {
+    enableACME = true;
+    forceSSL = true;
+    locations."/".proxyPass = "http://localhost:3030/";
+  };
+
+  services.postgresql = {
+    ensureDatabases = [ "woodpecker" ];
+    ensureUsers = [{
+      name = "woodpecker";
+      ensurePermissions = {
+        "DATABASE woodpecker" = "ALL PRIVILEGES";
+      };
+    }];
+  };
+
+  systemd.services.woodpecker-server = {
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      EnvironmentFile = [
+        # GITEA_CLIENT_SECRET etc
+        # /run/agenix/woodpecker
+      ];
+      Environment = [
+        "WOODPECKER_OPEN=true"
+        "WOODPECKER_AGENT_SECRET=${WOODPECKER_AGENT_SECRET}"
+
+        "WOODPECKER_GITEA=true"
+        "WOODPECKER_GITEA_URL=https://git.sealight.xyz"
+
+        "WOODPECKER_DATABASE_DATASOURCE=postgres:///woodpecker?host=/run/postgresql"
+        "WOODPECKER_DATABASE_DRIVER=postgres"
+        "WOODPECKER_SERVER_PORT=:3030"
+        "WOODPECKER_USER_CREATE=username:aynish,admin:true" # set your admin username
+      ];
+      ExecStart = "${pkgs.woodpecker-server}/bin/woodpecker-server";
+      User = woodpecker;
+      Group = woodpecker;
+    };
+  };
+
+}

modules/nixos/woodpecker.nix

@@ -0,0 +1,195 @@
+{ config, pkgs, lib, ... }:
+# I gave up writing this lol
+
+with lib;
+
+let
+  cfg = config.services.woodpecker;
+  droneserver = config.users.users.droneserver.name;
+
+in
+{
+  options = {
+    enable = mkEnableOption "Woodpecker CI/CD";
+
+    user = mkOption {
+      type = types.str;
+      default = "woodpeckerserver";
+      description = "User account under which gonic runs.";
+    };
+
+    group = mkOption {
+      type = types.str;
+      default = "woodpeckerserver";
+      description = "Group account under which gonic runs.";
+    };
+
+    hostname = mkOption {
+      type = types.str;
+      example = "woodpecker.example.com";
+      description = "VirtualHost created for nginx";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.users = optionalAttrs (cfg.user == name) {
+      ${name} = {
+        group = cfg.group;
+        description = "woodpecker user";
+        home = cfg.dataDir;
+        createHome = true;
+      };
+    };
+
+    users.groups = optionalAttrs (cfg.group == name) {
+      ${name}.gid = config.ids.gids.headphones;
+    };
+
+    services.nginx.virtualHosts."drone.my-server.tld" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/".proxyPass = "http://localhost:3030/";
+    };
+
+    services.postgresql = {
+      ensureDatabases = [ droneserver ];
+      ensureUsers = [{
+        name = droneserver;
+        ensurePermissions = {
+          "DATABASE ${droneserver}" = "ALL PRIVILEGES";
+        };
+      }];
+    };
+
+    # Secrets configured:
+    # - DRONE_GITEA_CLIENT_ID
+    # - DRONE_GITEA_CLIENT_SECRET
+    # - DRONE_RPC_SECRET
+    # To get these secrets, please check Drone's documentation for Gitea integration:
+    # https://docs.drone.io/server/provider/gitea/
+
+    sops.secrets.drone = {
+      sopsFile = ../.secrets/drone.yaml;
+    };
+
+    systemd.services.drone-server = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        EnvironmentFile = [
+          config.sops.secrets.drone.path
+        ];
+        Environment = [
+          "DRONE_DATABASE_DATASOURCE=postgres:///droneserver?host=/run/postgresql"
+          "DRONE_DATABASE_DRIVER=postgres"
+          "DRONE_SERVER_PORT=:3030"
+          "DRONE_USER_CREATE=username:viperML,admin:true" # set your admin username
+
+          "DRONE_GITEA_SERVER=https://git.my-domain.tld"
+          "DRONE_SERVER_HOST=drone.my-domain.tld"
+          "DRONE_SERVER_PROTO=https"
+        ];
+        ExecStart = "${pkgs.drone}/bin/drone-server";
+        User = droneserver;
+        Group = droneserver;
+      };
+    };
+
+    ### Docker runner
+
+    users.users.drone-runner-docker = {
+      isSystemUser = true;
+      group = "drone-runner-docker";
+    };
+    users.groups.drone-runner-docker = { };
+    # Allow the runner to use docker
+    users.groups.docker.members = [ "drone-runner-docker" ];
+
+    systemd.services.drone-runner-docker = {
+      enable = true;
+      wantedBy = [ "multi-user.target" ];
+      ### MANUALLY RESTART SERVICE IF CHANGED
+      restartIfChanged = false;
+      serviceConfig = {
+        Environment = [
+          "DRONE_RPC_PROTO=http"
+          "DRONE_RPC_HOST=localhost:3030"
+          "DRONE_RUNNER_CAPACITY=2"
+          "DRONE_RUNNER_NAME=drone-runner-docker"
+        ];
+        EnvironmentFile = [
+          config.sops.secrets.drone.path
+        ];
+        ExecStart = "${pkgs.drone-runner-docker}/bin/drone-runner-docker";
+        User = "drone-runner-docker";
+        Group = "drone-runner-docker";
+      };
+    };
+
+    ### Exec runner
+
+    users.users.drone-runner-exec = {
+      isSystemUser = true;
+      group = "drone-runner-exec";
+    };
+    users.groups.drone-runner-exec = { };
+    # Allow the exec runner to write to build with nix
+    nix.allowedUsers = [ "drone-runner-exec" ];
+
+    systemd.services.drone-runner-exec = {
+      enable = true;
+      wantedBy = [ "multi-user.target" ];
+      ### MANUALLY RESTART SERVICE IF CHANGED
+      restartIfChanged = true;
+      confinement.enable = true;
+      confinement.packages = [
+        pkgs.git
+        pkgs.gnutar
+        pkgs.bash
+        pkgs.nixFlakes
+        pkgs.gzip
+      ];
+      path = [
+        pkgs.git
+        pkgs.gnutar
+        pkgs.bash
+        pkgs.nixFlakes
+        pkgs.gzip
+      ];
+      serviceConfig = {
+        Environment = [
+          "DRONE_RPC_PROTO=http"
+          "DRONE_RPC_HOST=127.0.0.1:3030"
+          "DRONE_RUNNER_CAPACITY=2"
+          "DRONE_RUNNER_NAME=drone-runner-exec"
+          "NIX_REMOTE=daemon"
+          "PAGER=cat"
+          "DRONE_DEBUG=true"
+        ];
+        BindPaths = [
+          "/nix/var/nix/daemon-socket/socket"
+          "/run/nscd/socket"
+          # "/var/lib/drone"
+        ];
+        BindReadOnlyPaths = [
+          "/etc/passwd:/etc/passwd"
+          "/etc/group:/etc/group"
+          "/nix/var/nix/profiles/system/etc/nix:/etc/nix"
+          "${config.environment.etc."ssl/certs/ca-certificates.crt".source}:/etc/ssl/certs/ca-certificates.crt"
+          "${config.environment.etc."ssh/ssh_known_hosts".source}:/etc/ssh/ssh_known_hosts"
+          "${builtins.toFile "ssh_config" ''
+          Host git.ayats.org
+          ForwardAgent yes
+        ''}:/etc/ssh/ssh_config"
+          "/etc/machine-id"
+          "/etc/resolv.conf"
+          "/nix/"
+        ];
+        EnvironmentFile = [
+          config.sops.secrets.drone.path
+        ];
+        ExecStart = "${pkgs.drone-runner-exec}/bin/drone-runner-exec";
+        User = "drone-runner-exec";
+        Group = "drone-runner-exec";
+      };
+    };
+  };