homelab/nomad-job/prometheus.nomad
vincent e0f9190b76
Some checks failed
continuous-integration/drone/push Build is failing
feat: docker pull througt mirror
2024-03-17 18:58:24 +01:00

296 lines
8.6 KiB
HCL

job "prometheus" {
datacenters = ["homelab"]
priority = 50
type = "service"
constraint {
attribute = "${attr.cpu.arch}"
value = "amd64"
}
meta{
force_deploy= 1
}
constraint {
attribute = "${node.class}"
operator = "set_contains"
value = "cluster"
}
group "prometheus" {
count = 1
network {
port "prometheus_ui" {
static = 9090
}
}
restart {
attempts = 2
interval = "30m"
delay = "15s"
mode = "fail"
}
vault {
policies = ["prometheus"]
}
ephemeral_disk {
size = 300
}
task "prometheus" {
template {
change_mode = "noop"
destination = "local/prometheus.yml"
data = <<EOH
---
global:
scrape_interval: 10s
evaluation_interval: 10s
alerting:
alertmanagers:
- consul_sd_configs:
- server: 'consul.service.consul:8500'
services: ['alertmanager']
rule_files:
- "nomad-alert-rules.yml"
scrape_configs:
- job_name: 'nomad_metrics'
consul_sd_configs:
- server: 'consul.service.consul:8500'
services: ['nomad-client', 'nomad']
relabel_configs:
- source_labels: ['__meta_consul_tags']
regex: '(.*)http(.*)'
action: keep
- source_labels: ['__meta_consul_dc']
target_label: 'dc'
scrape_interval: 5s
metrics_path: /v1/metrics
params:
format: ['prometheus']
- job_name: 'traefik'
consul_sd_configs:
- server: 'consul.service.consul:8500'
services: ['traefik-local-admin','traefik-admin']
relabel_configs:
- source_labels: ['__meta_consul_service']
target_label: instance
- job_name: 'alertmanager'
consul_sd_configs:
- server: 'consul.service.consul:8500'
services: ['alertmanager']
relabel_configs:
- source_labels: ['__meta_consul_dc']
target_label: instance
- job_name: 'crowdsec'
consul_sd_configs:
- server: 'consul.service.consul:8500'
services: ['crowdsec-metrics']
relabel_configs:
- source_labels: [__meta_consul_node]
target_label: machine
- job_name: 'nodeexp'
static_configs:
consul_sd_configs:
- server: 'consul.service.consul:8500'
services: ['node-exporter']
relabel_configs:
- source_labels: [__meta_consul_node]
target_label: instance
- job_name: 'loki'
consul_sd_configs:
- server: consul.service.consul:8500
services: ['loki']
relabel_configs:
- source_labels: ['__meta_consul_dc']
target_label: instance
- job_name: 'HASS'
consul_sd_configs:
- server: 'consul.service.consul:8500'
services: ['hass']
relabel_configs:
- source_labels: ['__meta_consul_dc']
target_label: instance
scrape_interval: 60s
metrics_path: /api/prometheus
authorization:
credentials: {{ with secret "secrets/data/nomad/prometheus"}}'{{ .Data.data.hass_token }}'{{end}}
- job_name: 'nut'
consul_sd_configs:
- server: 'consul.service.consul:8500'
services: ['nutexporter']
metrics_path: /ups_metrics
relabel_configs:
- source_labels: ['__meta_consul_dc']
target_label: instance
EOH
}
template {
destination = "local/nomad-alert-rules.yml"
right_delimiter = "]]"
left_delimiter = "[["
data = <<EOH
---
groups:
- name: nomad_alerts
rules:
- alert: NomadBlockedEvaluation
expr: nomad_nomad_blocked_evals_total_blocked > 0
for: 0m
labels:
severity: warning
annotations:
summary: Nomad blocked evaluation (instance {{ $labels.instance }})
description: "Nomad blocked evaluation\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: NomadJobQueued
expr: nomad_nomad_job_summary_queued > 0
for: 2m
labels:
severity: warning
annotations:
summary: Nomad job queued (instance {{ $labels.instance }})
description: "Nomad job queued\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: NomadBatchError
expr: nomad_nomad_job_summary_failed{parent_id=~".+"}>0
labels:
severity: warning
annotations:
summary: Nomad batch {{ $labels.parent_id}} error
- alert: test gitea
expr: nomad_nomad_job_summary_running{exported_job="git"}==0
labels:
severity: warning
- name: nut_alerts
rules:
- alert: UPSonBattery
expr: network_ups_tools_ups_status{flag="OB"}==1
labels:
severity: warning
annotations:
summary: UPS switched on battery
- alert: UPSLowBattery
expr: network_ups_tools_ups_status{flag="LB"}==1
labels:
severity: critical
annotations:
summary: UPS is now on low battery please shutdown all device
- alert: "UPS Battery needed to be replaced"
expr: network_ups_tools_ups_status{flag="RB"}==1
labels:
severity: warning
annotations:
summary: UPS battery is detected to replace
- name: Node_alerts
rules:
- alert: HostOutOfMemory
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10)
for: 2m
labels:
severity: warning
annotations:
summary: Host out of memory (instance {{ $labels.instance }})
description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostMemoryUnderMemoryPressure
expr: (rate(node_vmstat_pgmajfault[1m]) > 600) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: critical
annotations:
summary: Host memory under memory pressure (instance {{ $labels.instance }})
description: "The node is under heavy memory pressure. High rate of major page faults\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostOutOfDiskSpace
expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0)
for: 2m
labels:
severity: warning
annotations:
summary: Host out of disk space (instance {{ $labels.instance }})
description: "Disk is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostHighCpuLoad
expr: (sum by (instance) (avg by (mode, instance) (rate(node_cpu_seconds_total{mode!="idle"}[2m]))) > 0.8)
for: 10m
labels:
severity: warning
annotations:
summary: Host high CPU load (instance {{ $labels.instance }})
description: "CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualDiskWriteLatency
expr: (rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0)
for: 2m
labels:
severity: warning
annotations:
summary: Host unusual disk write latency (instance {{ $labels.instance }})
description: "Disk latency is growing (write operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualDiskReadLatency
expr: (rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0)
for: 2m
labels:
severity: warning
annotations:
summary: Host unusual disk read latency (instance {{ $labels.instance }})
description: "Disk latency is growing (read operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
EOH
}
driver = "docker"
config {
image = "docker.service.consul:5000/prom/prometheus:latest"
args = [
"--config.file=/etc/prometheus/prometheus.yml",
"--storage.tsdb.path=/prometheus",
"--storage.tsdb.retention.time=40d",
]
volumes = [
"local/nomad-alert-rules.yml:/etc/prometheus/nomad-alert-rules.yml",
"local/prometheus.yml:/etc/prometheus/prometheus.yml",
"/mnt/diskstation/nomad/prometheus:/prometheus"
]
ports = ["prometheus_ui"]
}
service {
name = "prometheus"
tags = ["urlprefix-/",
"homer.enable=true",
"homer.name=Prometheus",
"homer.service=Monitoring",
"homer.type=Prometheus",
"homer.logo=https://upload.wikimedia.org/wikipedia/commons/thumb/3/38/Prometheus_software_logo.svg/173px-Prometheus_software_logo.svg.png",
"homer.target=_blank",
"homer.url=http://${NOMAD_ADDR_prometheus_ui}",
]
port = "prometheus_ui"
check {
name = "prometheus_ui port alive"
type = "http"
path = "/-/healthy"
interval = "10s"
timeout = "2s"
}
}
resources {
memory = 350
}
}
}
}