updating some stuff, trying to get woodpecker, and grafana alerts off
the ground
This commit is contained in:
		
							parent
							
								
									12875ff891
								
							
						
					
					
						commit
						e7bfe88dcf
					
				|  | @ -24,7 +24,8 @@ | |||
|     services.NetworkManager-wait-online.wantedBy = pkgs.lib.mkForce [ ]; # Normally ["network-online.target"] | ||||
|   }; | ||||
| 
 | ||||
|   programs.gnupg.agent.pinentryFlavor = "gtk2"; | ||||
|   programs.gnupg.agent.enable = true; | ||||
|   programs.gnupg.agent.pinentryFlavor = "gnome3"; | ||||
| 
 | ||||
|   fileSystems."/mnt/ftp" = { | ||||
|     device = "192.168.1.240:/home/ftp"; | ||||
|  |  | |||
|  | @ -58,6 +58,8 @@ in | |||
|     unstable.sublime-music | ||||
|     unstable.nheko | ||||
|     unstable.tootle | ||||
|     newsflash | ||||
|     liferea | ||||
| 
 | ||||
|     scrot | ||||
|     ripcord | ||||
|  | @ -69,7 +71,6 @@ in | |||
|     calibre | ||||
|     fractal | ||||
|     mpv | ||||
|     newsflash | ||||
|     zeal | ||||
|     xclip | ||||
|     xdotool | ||||
|  |  | |||
							
								
								
									
										300
									
								
								hosts/profiles/monitoring/alert-rules.yaml
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										300
									
								
								hosts/profiles/monitoring/alert-rules.yaml
									
									
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,300 @@ | |||
| groups: | ||||
| - name: hosts | ||||
|   rules: | ||||
|   - alert: HostOutOfMemory | ||||
|     expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10 | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host out of memory (instance {{ $labels.instance }}) | ||||
|       description: "Node memory is filling up (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostMemoryUnderMemoryPressure | ||||
|     expr: rate(node_vmstat_pgmajfault[1m]) > 1000 | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host memory under memory pressure (instance {{ $labels.instance }}) | ||||
|       description: "The node is under heavy memory pressure. High rate of major page faults\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostUnusualNetworkThroughputIn | ||||
|     expr: sum by (instance) (rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100 | ||||
|     for: 5m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host unusual network throughput in (instance {{ $labels.instance }}) | ||||
|       description: "Host network interfaces are probably receiving too much data (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostUnusualNetworkThroughputOut | ||||
|     expr: sum by (instance) (rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100 | ||||
|     for: 5m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host unusual network throughput out (instance {{ $labels.instance }}) | ||||
|       description: "Host network interfaces are probably sending too much data (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
| #  - alert: HostUnusualDiskReadRate | ||||
| #    expr: sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 100 | ||||
| #    for: 5m | ||||
| #    labels: | ||||
| #      severity: warning | ||||
| #    annotations: | ||||
| #      summary: Host unusual disk read rate (instance {{ $labels.instance }}) | ||||
| #      description: "Disk is probably reading too much data (> 50 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| # | ||||
| #  - alert: HostUnusualDiskWriteRate | ||||
| #    expr: sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50 | ||||
| #    for: 2m | ||||
| #    labels: | ||||
| #      severity: warning | ||||
| #    annotations: | ||||
| #      summary: Host unusual disk write rate (instance {{ $labels.instance }}) | ||||
| #      description: "Disk is probably writing too much data (> 50 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   # Please add ignored mountpoints in node_exporter parameters like | ||||
|   # "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)". | ||||
|   # Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users. | ||||
|   - alert: HostOutOfDiskSpace | ||||
|     expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0 | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host out of disk space (instance {{ $labels.instance }}) | ||||
|       description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   # Please add ignored mountpoints in node_exporter parameters like | ||||
|   # "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)". | ||||
|   # Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users. | ||||
|   - alert: HostDiskWillFillIn24Hours | ||||
|     expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0 | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host disk will fill in 24 hours (instance {{ $labels.instance }}) | ||||
|       description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostOutOfInodes | ||||
|     expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint="/rootfs"} * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly{mountpoint="/rootfs"} == 0 | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host out of inodes (instance {{ $labels.instance }}) | ||||
|       description: "Disk is almost running out of available inodes (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostInodesWillFillIn24Hours | ||||
|     expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint="/rootfs"} * 100 < 10 and predict_linear(node_filesystem_files_free{mountpoint="/rootfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly{mountpoint="/rootfs"} == 0 | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host inodes will fill in 24 hours (instance {{ $labels.instance }}) | ||||
|       description: "Filesystem is predicted to run out of inodes within the next 24 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
| #  - alert: HostUnusualDiskReadLatency | ||||
| #    expr: rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0 | ||||
| #    for: 2m | ||||
| #    labels: | ||||
| #      severity: warning | ||||
| #    annotations: | ||||
| #      summary: Host unusual disk read latency (instance {{ $labels.instance }}) | ||||
| #      description: "Disk latency is growing (read operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| # | ||||
| #  - alert: HostUnusualDiskWriteLatency | ||||
| #    expr: rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 2 and rate(node_disk_writes_completed_total[1m]) > 0 | ||||
| #    for: 2m | ||||
| #    labels: | ||||
| #      severity: warning | ||||
| #    annotations: | ||||
| #      summary: Host unusual disk write latency (instance {{ $labels.instance }}) | ||||
| #      description: "Disk latency is growing (write operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostHighCpuLoad | ||||
|     expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 90 | ||||
|     for: 0m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host high CPU load (instance {{ $labels.instance }}) | ||||
|       description: "CPU load is > 90%\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostCpuStealNoisyNeighbor | ||||
|     expr: avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10 | ||||
|     for: 0m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }}) | ||||
|       description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   # 10000 context switches is an arbitrary number. | ||||
|   # Alert threshold depends on nature of application. | ||||
|   # Please read: https://github.com/samber/awesome-prometheus-alerts/issues/58 | ||||
|   - alert: HostContextSwitching | ||||
|     expr: (rate(node_context_switches_total[5m])) / (count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 10000 | ||||
|     for: 0m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host context switching (instance {{ $labels.instance }}) | ||||
|       description: "Context switching is growing on node (> 1000 / s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostSwapIsFillingUp | ||||
|     expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80 | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host swap is filling up (instance {{ $labels.instance }}) | ||||
|       description: "Swap is filling up (>80%)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostSystemdServiceCrashed | ||||
|     expr: node_systemd_unit_state{state="failed"} == 1 | ||||
|     for: 0m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host systemd service crashed (instance {{ $labels.instance }}) | ||||
|       description: "systemd service crashed: {{ $labels.name }} on {{ $labels.job }}" | ||||
| 
 | ||||
|   - alert: HostPhysicalComponentTooHot | ||||
|     expr: node_hwmon_temp_celsius > 75 | ||||
|     for: 5m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host physical component too hot (instance {{ $labels.instance }}) | ||||
|       description: "Physical hardware component too hot\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostNodeOvertemperatureAlarm | ||||
|     expr: node_hwmon_temp_crit_alarm_celsius == 1 | ||||
|     for: 0m | ||||
|     labels: | ||||
|       severity: critical | ||||
|     annotations: | ||||
|       summary: Host node overtemperature alarm (instance {{ $labels.instance }}) | ||||
|       description: "Physical node temperature alarm triggered\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostRaidArrayGotInactive | ||||
|     expr: node_md_state{state="inactive"} > 0 | ||||
|     for: 0m | ||||
|     labels: | ||||
|       severity: critical | ||||
|     annotations: | ||||
|       summary: Host RAID array got inactive (instance {{ $labels.instance }}) | ||||
|       description: "RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostRaidDiskFailure | ||||
|     expr: node_md_disks{state="failed"} > 0 | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host RAID disk failure (instance {{ $labels.instance }}) | ||||
|       description: "At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostKernelVersionDeviations | ||||
|     expr: count(sum(label_replace(node_uname_info, "kernel", "$1", "release", "([0-9]+.[0-9]+.[0-9]+).*")) by (kernel)) > 2 | ||||
|     for: 6h | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host kernel version deviations (instance {{ $labels.instance }}) | ||||
|       description: "Different kernel versions are running\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostOomKillDetected | ||||
|     expr: increase(node_vmstat_oom_kill[1m]) > 0 | ||||
|     for: 0m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host OOM kill detected (instance {{ $labels.instance }}) | ||||
|       description: "OOM kill detected\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostEdacCorrectableErrorsDetected | ||||
|     expr: increase(node_edac_correctable_errors_total[1m]) > 0 | ||||
|     for: 0m | ||||
|     labels: | ||||
|       severity: info | ||||
|     annotations: | ||||
|       summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }}) | ||||
|       description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} correctable memory errors reported by EDAC in the last 5 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostEdacUncorrectableErrorsDetected | ||||
|     expr: node_edac_uncorrectable_errors_total > 0 | ||||
|     for: 0m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }}) | ||||
|       description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostNetworkReceiveErrors | ||||
|     expr: rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01 | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host Network Receive Errors (instance {{ $labels.instance }}) | ||||
|       description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last five minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostNetworkTransmitErrors | ||||
|     expr: rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01 | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host Network Transmit Errors (instance {{ $labels.instance }}) | ||||
|       description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last five minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostNetworkInterfaceSaturated | ||||
|     expr: (rate(node_network_receive_bytes_total{device!~"^tap.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*"} > 0.8 | ||||
|     for: 1m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host Network Interface Saturated (instance {{ $labels.instance }}) | ||||
|       description: "The network interface \"{{ $labels.interface }}\" on \"{{ $labels.instance }}\" is getting overloaded.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostConntrackLimit | ||||
|     expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8 | ||||
|     for: 5m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host conntrack limit (instance {{ $labels.instance }}) | ||||
|       description: "The number of conntrack is approching limit\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostClockSkew | ||||
|     expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0) | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host clock skew (instance {{ $labels.instance }}) | ||||
|       description: "Clock skew detected. Clock is out of sync.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|   - alert: HostClockNotSynchronising | ||||
|     expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16 | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: Host clock not synchronising (instance {{ $labels.instance }}) | ||||
|       description: "Clock not synchronising.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}" | ||||
| 
 | ||||
|     # 26 hours since last backup | ||||
|   - alert: NoRecentBackup | ||||
|     expr: (time()*1000 - borgbackup_last_backup) > (1000 * 60 * 60 * 26)  | ||||
|     for: 2m | ||||
|     labels: | ||||
|       severity: warning | ||||
|     annotations: | ||||
|       summary: No backup in the last 26 hours (instance {{ $labels.exported_job }}) | ||||
|       description: "to repository {{ $labels.repo }}" | ||||
|  | @ -1,4 +1,4 @@ | |||
| { config, pkgs, ... }: { | ||||
| { self, config, pkgs, ... }: { | ||||
|   # grafana configuration | ||||
|   services.grafana = { | ||||
|     enable = true; | ||||
|  | @ -7,6 +7,8 @@ | |||
|       http_port = 2342; | ||||
|       http_addr = "127.0.0.1"; | ||||
|     }; | ||||
|     # TODO enable | ||||
|     # provision.alerting.rules.path = "${self}/hosts/profiles/monitoring/alert-rules.yaml"; | ||||
|   }; | ||||
| 
 | ||||
|   # nginx reverse proxy | ||||
|  |  | |||
							
								
								
									
										65
									
								
								hosts/profiles/woodpecker-agent/default.nix
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										65
									
								
								hosts/profiles/woodpecker-agent/default.nix
									
									
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,65 @@ | |||
| { | ||||
| 
 | ||||
|   users.users.woodpecker-runner = { | ||||
|     isSystemUser = true; | ||||
|     group = "woodpecker-runner"; | ||||
|   }; | ||||
|   users.groups.woodpecker-runner = { }; | ||||
|   # Allow the exec runner to write to build with nix | ||||
|   nix.allowedUsers = [ "woodpecker-runner" ]; | ||||
| 
 | ||||
|   systemd.services.woodpecker-runner-exec = { | ||||
|     enable = true; | ||||
|     wantedBy = [ "multi-user.target" ]; | ||||
|     ### MANUALLY RESTART SERVICE IF CHANGED | ||||
|     restartIfChanged = true; | ||||
|     confinement.enable = true; | ||||
|     confinement.packages = [ | ||||
|       pkgs.git | ||||
|       pkgs.gnutar | ||||
|       pkgs.bash | ||||
|       pkgs.nixFlakes | ||||
|       pkgs.gzip | ||||
|     ]; | ||||
|     path = [ | ||||
|       pkgs.git | ||||
|       pkgs.gnutar | ||||
|       pkgs.bash | ||||
|       pkgs.nixFlakes | ||||
|       pkgs.gzip | ||||
|     ]; | ||||
|     serviceConfig = { | ||||
|       Environment = [ | ||||
|         "WOODPECKER_RUNNER_CAPACITY=2" | ||||
|         "WOODPECKER_RUNNER_NAME=woodpecker-agent" | ||||
|         "NIX_REMOTE=daemon" | ||||
|         "PAGER=cat" | ||||
|       ]; | ||||
|       BindPaths = [ | ||||
|         "/nix/var/nix/daemon-socket/socket" | ||||
|         "/run/nscd/socket" | ||||
|         # "/var/lib/woodpecker" | ||||
|       ]; | ||||
|       BindReadOnlyPaths = [ | ||||
|         "/etc/passwd:/etc/passwd" | ||||
|         "/etc/group:/etc/group" | ||||
|         "/nix/var/nix/profiles/system/etc/nix:/etc/nix" | ||||
|         "${config.environment.etc."ssl/certs/ca-certificates.crt".source}:/etc/ssl/certs/ca-certificates.crt" | ||||
|         "${config.environment.etc."ssh/ssh_known_hosts".source}:/etc/ssh/ssh_known_hosts" | ||||
|         "${builtins.toFile "ssh_config" '' | ||||
|           Host git.sealight.xyz | ||||
|           ForwardAgent yes | ||||
|         ''}:/etc/ssh/ssh_config" | ||||
|         "/etc/machine-id" | ||||
|         "/etc/resolv.conf" | ||||
|         "/nix/" | ||||
|       ]; | ||||
|       EnvironmentFile = [ | ||||
|         # /run/agenix/woodpecker-agent-secret | ||||
|       ]; | ||||
|       ExecStart = "${pkgs.woodpecker-agent}/bin/woodpecker-agent"; | ||||
|       User = "woodpecker-runner"; | ||||
|       Group = "woodpecker-runner"; | ||||
|     }; | ||||
|   }; | ||||
| } | ||||
							
								
								
									
										52
									
								
								hosts/profiles/woodpecker-server/default.nix
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										52
									
								
								hosts/profiles/woodpecker-server/default.nix
									
									
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,52 @@ | |||
| { | ||||
|   users.users.woodpecker = { | ||||
|     group = woodpecker; | ||||
|     description = "woodpecker user"; | ||||
|     home = "/var/lib/woodpecker"; | ||||
|     createHome = true; | ||||
|   }; | ||||
| 
 | ||||
|   users.groups.woodpecker = { }; | ||||
| 
 | ||||
|   services.nginx.virtualHosts."ci.sealight.xyz" = { | ||||
|     enableACME = true; | ||||
|     forceSSL = true; | ||||
|     locations."/".proxyPass = "http://localhost:3030/"; | ||||
|   }; | ||||
| 
 | ||||
|   services.postgresql = { | ||||
|     ensureDatabases = [ "woodpecker" ]; | ||||
|     ensureUsers = [{ | ||||
|       name = "woodpecker"; | ||||
|       ensurePermissions = { | ||||
|         "DATABASE woodpecker" = "ALL PRIVILEGES"; | ||||
|       }; | ||||
|     }]; | ||||
|   }; | ||||
| 
 | ||||
|   systemd.services.woodpecker-server = { | ||||
|     wantedBy = [ "multi-user.target" ]; | ||||
|     serviceConfig = { | ||||
|       EnvironmentFile = [ | ||||
|         # GITEA_CLIENT_SECRET etc | ||||
|         # /run/agenix/woodpecker | ||||
|       ]; | ||||
|       Environment = [ | ||||
|         "WOODPECKER_OPEN=true" | ||||
|         "WOODPECKER_AGENT_SECRET=${WOODPECKER_AGENT_SECRET}" | ||||
| 
 | ||||
|         "WOODPECKER_GITEA=true" | ||||
|         "WOODPECKER_GITEA_URL=https://git.sealight.xyz" | ||||
| 
 | ||||
|         "WOODPECKER_DATABASE_DATASOURCE=postgres:///woodpecker?host=/run/postgresql" | ||||
|         "WOODPECKER_DATABASE_DRIVER=postgres" | ||||
|         "WOODPECKER_SERVER_PORT=:3030" | ||||
|         "WOODPECKER_USER_CREATE=username:aynish,admin:true" # set your admin username | ||||
|       ]; | ||||
|       ExecStart = "${pkgs.woodpecker-server}/bin/woodpecker-server"; | ||||
|       User = woodpecker; | ||||
|       Group = woodpecker; | ||||
|     }; | ||||
|   }; | ||||
| 
 | ||||
| } | ||||
							
								
								
									
										195
									
								
								modules/nixos/woodpecker.nix
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										195
									
								
								modules/nixos/woodpecker.nix
									
									
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,195 @@ | |||
| { config, pkgs, lib, ... }: | ||||
| # I gave up writing this lol | ||||
| 
 | ||||
| with lib; | ||||
| 
 | ||||
| let | ||||
|   cfg = config.services.woodpecker; | ||||
|   droneserver = config.users.users.droneserver.name; | ||||
| 
 | ||||
| in | ||||
| { | ||||
|   options = { | ||||
|     enable = mkEnableOption "Woodpecker CI/CD"; | ||||
| 
 | ||||
|     user = mkOption { | ||||
|       type = types.str; | ||||
|       default = "woodpeckerserver"; | ||||
|       description = "User account under which gonic runs."; | ||||
|     }; | ||||
| 
 | ||||
|     group = mkOption { | ||||
|       type = types.str; | ||||
|       default = "woodpeckerserver"; | ||||
|       description = "Group account under which gonic runs."; | ||||
|     }; | ||||
| 
 | ||||
|     hostname = mkOption { | ||||
|       type = types.str; | ||||
|       example = "woodpecker.example.com"; | ||||
|       description = "VirtualHost created for nginx"; | ||||
|     }; | ||||
|   }; | ||||
| 
 | ||||
|   config = mkIf cfg.enable { | ||||
|     users.users = optionalAttrs (cfg.user == name) { | ||||
|       ${name} = { | ||||
|         group = cfg.group; | ||||
|         description = "woodpecker user"; | ||||
|         home = cfg.dataDir; | ||||
|         createHome = true; | ||||
|       }; | ||||
|     }; | ||||
| 
 | ||||
|     users.groups = optionalAttrs (cfg.group == name) { | ||||
|       ${name}.gid = config.ids.gids.headphones; | ||||
|     }; | ||||
| 
 | ||||
|     services.nginx.virtualHosts."drone.my-server.tld" = { | ||||
|       enableACME = true; | ||||
|       forceSSL = true; | ||||
|       locations."/".proxyPass = "http://localhost:3030/"; | ||||
|     }; | ||||
| 
 | ||||
|     services.postgresql = { | ||||
|       ensureDatabases = [ droneserver ]; | ||||
|       ensureUsers = [{ | ||||
|         name = droneserver; | ||||
|         ensurePermissions = { | ||||
|           "DATABASE ${droneserver}" = "ALL PRIVILEGES"; | ||||
|         }; | ||||
|       }]; | ||||
|     }; | ||||
| 
 | ||||
|     # Secrets configured: | ||||
|     # - DRONE_GITEA_CLIENT_ID | ||||
|     # - DRONE_GITEA_CLIENT_SECRET | ||||
|     # - DRONE_RPC_SECRET | ||||
|     # To get these secrets, please check Drone's documentation for Gitea integration: | ||||
|     # https://docs.drone.io/server/provider/gitea/ | ||||
| 
 | ||||
|     sops.secrets.drone = { | ||||
|       sopsFile = ../.secrets/drone.yaml; | ||||
|     }; | ||||
| 
 | ||||
|     systemd.services.drone-server = { | ||||
|       wantedBy = [ "multi-user.target" ]; | ||||
|       serviceConfig = { | ||||
|         EnvironmentFile = [ | ||||
|           config.sops.secrets.drone.path | ||||
|         ]; | ||||
|         Environment = [ | ||||
|           "DRONE_DATABASE_DATASOURCE=postgres:///droneserver?host=/run/postgresql" | ||||
|           "DRONE_DATABASE_DRIVER=postgres" | ||||
|           "DRONE_SERVER_PORT=:3030" | ||||
|           "DRONE_USER_CREATE=username:viperML,admin:true" # set your admin username | ||||
| 
 | ||||
|           "DRONE_GITEA_SERVER=https://git.my-domain.tld" | ||||
|           "DRONE_SERVER_HOST=drone.my-domain.tld" | ||||
|           "DRONE_SERVER_PROTO=https" | ||||
|         ]; | ||||
|         ExecStart = "${pkgs.drone}/bin/drone-server"; | ||||
|         User = droneserver; | ||||
|         Group = droneserver; | ||||
|       }; | ||||
|     }; | ||||
| 
 | ||||
|     ### Docker runner | ||||
| 
 | ||||
|     users.users.drone-runner-docker = { | ||||
|       isSystemUser = true; | ||||
|       group = "drone-runner-docker"; | ||||
|     }; | ||||
|     users.groups.drone-runner-docker = { }; | ||||
|     # Allow the runner to use docker | ||||
|     users.groups.docker.members = [ "drone-runner-docker" ]; | ||||
| 
 | ||||
|     systemd.services.drone-runner-docker = { | ||||
|       enable = true; | ||||
|       wantedBy = [ "multi-user.target" ]; | ||||
|       ### MANUALLY RESTART SERVICE IF CHANGED | ||||
|       restartIfChanged = false; | ||||
|       serviceConfig = { | ||||
|         Environment = [ | ||||
|           "DRONE_RPC_PROTO=http" | ||||
|           "DRONE_RPC_HOST=localhost:3030" | ||||
|           "DRONE_RUNNER_CAPACITY=2" | ||||
|           "DRONE_RUNNER_NAME=drone-runner-docker" | ||||
|         ]; | ||||
|         EnvironmentFile = [ | ||||
|           config.sops.secrets.drone.path | ||||
|         ]; | ||||
|         ExecStart = "${pkgs.drone-runner-docker}/bin/drone-runner-docker"; | ||||
|         User = "drone-runner-docker"; | ||||
|         Group = "drone-runner-docker"; | ||||
|       }; | ||||
|     }; | ||||
| 
 | ||||
|     ### Exec runner | ||||
| 
 | ||||
|     users.users.drone-runner-exec = { | ||||
|       isSystemUser = true; | ||||
|       group = "drone-runner-exec"; | ||||
|     }; | ||||
|     users.groups.drone-runner-exec = { }; | ||||
|     # Allow the exec runner to write to build with nix | ||||
|     nix.allowedUsers = [ "drone-runner-exec" ]; | ||||
| 
 | ||||
|     systemd.services.drone-runner-exec = { | ||||
|       enable = true; | ||||
|       wantedBy = [ "multi-user.target" ]; | ||||
|       ### MANUALLY RESTART SERVICE IF CHANGED | ||||
|       restartIfChanged = true; | ||||
|       confinement.enable = true; | ||||
|       confinement.packages = [ | ||||
|         pkgs.git | ||||
|         pkgs.gnutar | ||||
|         pkgs.bash | ||||
|         pkgs.nixFlakes | ||||
|         pkgs.gzip | ||||
|       ]; | ||||
|       path = [ | ||||
|         pkgs.git | ||||
|         pkgs.gnutar | ||||
|         pkgs.bash | ||||
|         pkgs.nixFlakes | ||||
|         pkgs.gzip | ||||
|       ]; | ||||
|       serviceConfig = { | ||||
|         Environment = [ | ||||
|           "DRONE_RPC_PROTO=http" | ||||
|           "DRONE_RPC_HOST=127.0.0.1:3030" | ||||
|           "DRONE_RUNNER_CAPACITY=2" | ||||
|           "DRONE_RUNNER_NAME=drone-runner-exec" | ||||
|           "NIX_REMOTE=daemon" | ||||
|           "PAGER=cat" | ||||
|           "DRONE_DEBUG=true" | ||||
|         ]; | ||||
|         BindPaths = [ | ||||
|           "/nix/var/nix/daemon-socket/socket" | ||||
|           "/run/nscd/socket" | ||||
|           # "/var/lib/drone" | ||||
|         ]; | ||||
|         BindReadOnlyPaths = [ | ||||
|           "/etc/passwd:/etc/passwd" | ||||
|           "/etc/group:/etc/group" | ||||
|           "/nix/var/nix/profiles/system/etc/nix:/etc/nix" | ||||
|           "${config.environment.etc."ssl/certs/ca-certificates.crt".source}:/etc/ssl/certs/ca-certificates.crt" | ||||
|           "${config.environment.etc."ssh/ssh_known_hosts".source}:/etc/ssh/ssh_known_hosts" | ||||
|           "${builtins.toFile "ssh_config" '' | ||||
|           Host git.ayats.org | ||||
|           ForwardAgent yes | ||||
|         ''}:/etc/ssh/ssh_config" | ||||
|           "/etc/machine-id" | ||||
|           "/etc/resolv.conf" | ||||
|           "/nix/" | ||||
|         ]; | ||||
|         EnvironmentFile = [ | ||||
|           config.sops.secrets.drone.path | ||||
|         ]; | ||||
|         ExecStart = "${pkgs.drone-runner-exec}/bin/drone-runner-exec"; | ||||
|         User = "drone-runner-exec"; | ||||
|         Group = "drone-runner-exec"; | ||||
|       }; | ||||
|     }; | ||||
|   }; | ||||
		Loading…
	
		Reference in a new issue