mirror of
git://f0xx.org/ac/ac-deploy
synced 2026-07-29 03:58:34 +03:00
added 'monitoring' path - ask devs why it's there
This commit is contained in:
Anton Afanasyeu
113
monitoring/cast04/rules/node_alerts.yml
Normal file
113
monitoring/cast04/rules/node_alerts.yml
Normal file
@@ -0,0 +1,113 @@
|
||||
# Node-level alert rules — hardware / OS
|
||||
# Thresholds per SPEC ac-docs/specs/20260626_cluster_monitoring.md §4.1
|
||||
|
||||
groups:
|
||||
- name: node_health
|
||||
rules:
|
||||
|
||||
# ── instance down ────────────────────────────────────────────────────────
|
||||
- alert: InstanceDown
|
||||
expr: up{job="node"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Instance {{ $labels.instance }} is down"
|
||||
description: "node_exporter on {{ $labels.instance }} has been unreachable for >2 min."
|
||||
|
||||
# ── CPU ─────────────────────────────────────────────────────────────────
|
||||
- alert: HighCpuWarning
|
||||
expr: >
|
||||
100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU on {{ $labels.instance }}"
|
||||
description: "CPU usage is {{ $value | printf \"%.1f\" }}% (threshold: 85%) on {{ $labels.instance }}."
|
||||
|
||||
- alert: HighCpuCritical
|
||||
expr: >
|
||||
100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical CPU on {{ $labels.instance }}"
|
||||
description: "CPU usage is {{ $value | printf \"%.1f\" }}% (threshold: 95%) on {{ $labels.instance }}."
|
||||
|
||||
# ── memory ──────────────────────────────────────────────────────────────
|
||||
- alert: LowMemoryWarning
|
||||
expr: >
|
||||
(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 < 15
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Low memory on {{ $labels.instance }}"
|
||||
description: "Available RAM is {{ $value | printf \"%.1f\" }}% (threshold: 15%) on {{ $labels.instance }}."
|
||||
|
||||
- alert: LowMemoryCritical
|
||||
expr: >
|
||||
(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 < 5
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical memory on {{ $labels.instance }}"
|
||||
description: "Available RAM is {{ $value | printf \"%.1f\" }}% (threshold: 5%) on {{ $labels.instance }}."
|
||||
|
||||
# ── disk ────────────────────────────────────────────────────────────────
|
||||
- alert: LowDiskWarning
|
||||
expr: >
|
||||
(node_filesystem_avail_bytes{fstype!~"tmpfs|devtmpfs|overlay"} /
|
||||
node_filesystem_size_bytes{fstype!~"tmpfs|devtmpfs|overlay"}) * 100 < 20
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Low disk on {{ $labels.instance }} mountpoint {{ $labels.mountpoint }}"
|
||||
description: "Free disk is {{ $value | printf \"%.1f\" }}% on {{ $labels.instance }}:{{ $labels.mountpoint }} (threshold: 20%)."
|
||||
|
||||
- alert: LowDiskCritical
|
||||
expr: >
|
||||
(node_filesystem_avail_bytes{fstype!~"tmpfs|devtmpfs|overlay"} /
|
||||
node_filesystem_size_bytes{fstype!~"tmpfs|devtmpfs|overlay"}) * 100 < 10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical disk on {{ $labels.instance }}:{{ $labels.mountpoint }}"
|
||||
description: "Free disk is {{ $value | printf \"%.1f\" }}% (threshold: 10%) on {{ $labels.instance }}:{{ $labels.mountpoint }}."
|
||||
|
||||
# ── load ────────────────────────────────────────────────────────────────
|
||||
- alert: HighLoadWarning
|
||||
expr: >
|
||||
node_load1 / on(instance) group_left() count(node_cpu_seconds_total{mode="idle"}) by (instance) > 1.5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High load on {{ $labels.instance }}"
|
||||
description: "Load average 1m is {{ $value | printf \"%.2f\" }}x per vCPU (threshold: 1.5×) on {{ $labels.instance }}."
|
||||
|
||||
- alert: HighLoadCritical
|
||||
expr: >
|
||||
node_load1 / on(instance) group_left() count(node_cpu_seconds_total{mode="idle"}) by (instance) > 3.0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical load on {{ $labels.instance }}"
|
||||
description: "Load average 1m is {{ $value | printf \"%.2f\" }}x per vCPU (threshold: 3.0×) on {{ $labels.instance }}."
|
||||
|
||||
# ── network anomaly ─────────────────────────────────────────────────────
|
||||
- alert: NetworkReceiveSaturation
|
||||
expr: >
|
||||
rate(node_network_receive_bytes_total{device!~"lo|dummy.*"}[5m]) > 100 * 1024 * 1024
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High inbound traffic on {{ $labels.instance }}"
|
||||
description: "{{ $labels.device }} on {{ $labels.instance }} receiving >100 MB/s for 5 min."
|
||||
87
monitoring/cast04/rules/service_alerts.yml
Normal file
87
monitoring/cast04/rules/service_alerts.yml
Normal file
@@ -0,0 +1,87 @@
|
||||
# Service-level alert rules — HTTP probes, DB
|
||||
# Alpine uses OpenRC (not systemd) — nginx availability covered by blackbox HTTP probes
|
||||
# Thresholds per SPEC ac-docs/specs/20260626_cluster_monitoring.md §4.2–4.3
|
||||
|
||||
groups:
|
||||
- name: blackbox_probes
|
||||
rules:
|
||||
|
||||
# ── HTTP endpoint down ───────────────────────────────────────────────────
|
||||
- alert: HttpEndpointDown
|
||||
expr: probe_success{job="blackbox_http"} == 0
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "HTTP endpoint down: {{ $labels.instance }}"
|
||||
description: "blackbox probe to {{ $labels.instance }} is failing for >3 min."
|
||||
|
||||
- alert: HttpSlowResponse
|
||||
expr: probe_duration_seconds{job="blackbox_http"} > 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Slow HTTP response: {{ $labels.instance }}"
|
||||
description: "HTTP probe to {{ $labels.instance }} takes >5s (current: {{ $value | printf \"%.1f\" }}s)."
|
||||
|
||||
- alert: SslCertExpiringSoon
|
||||
expr: probe_ssl_earliest_cert_expiry{job="blackbox_http"} - time() < 14 * 86400
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "SSL cert expiring soon: {{ $labels.instance }}"
|
||||
description: "TLS certificate for {{ $labels.instance }} expires in {{ $value | humanizeDuration }}."
|
||||
|
||||
- alert: SslCertExpiring7d
|
||||
expr: probe_ssl_earliest_cert_expiry{job="blackbox_http"} - time() < 7 * 86400
|
||||
for: 1h
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "SSL cert expires in <7 days: {{ $labels.instance }}"
|
||||
description: "TLS certificate for {{ $labels.instance }} expires in {{ $value | humanizeDuration }} — renew immediately."
|
||||
|
||||
# ── MariaDB ─────────────────────────────────────────────────────────────────
|
||||
- name: mariadb
|
||||
rules:
|
||||
- alert: MariaDBDown
|
||||
expr: mysql_up == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "MariaDB is down on {{ $labels.instance }}"
|
||||
description: "mysqld_exporter cannot connect to MariaDB on {{ $labels.instance }}."
|
||||
|
||||
- alert: MariaDBHighConnections
|
||||
expr: mysql_global_status_threads_connected / mysql_global_variables_max_connections > 0.80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "MariaDB connection pool high on {{ $labels.instance }}"
|
||||
description: "{{ $value | printf \"%.0f\" }}% of max_connections used on {{ $labels.instance }}."
|
||||
|
||||
- alert: MariaDBSlowQueries
|
||||
expr: rate(mysql_global_status_slow_queries[5m]) > 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "MariaDB slow queries on {{ $labels.instance }}"
|
||||
description: "More than 1 slow query/s detected on {{ $labels.instance }}."
|
||||
|
||||
# ── disk I/O saturation ──────────────────────────────────────────────────────
|
||||
- name: disk_io
|
||||
rules:
|
||||
- alert: DiskIoSaturation
|
||||
expr: >
|
||||
rate(node_disk_io_time_seconds_total[5m]) > 0.9
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Disk I/O saturation on {{ $labels.instance }}"
|
||||
description: "Disk {{ $labels.device }} on {{ $labels.instance }} is >90% I/O utilized for >10 min."
|
||||
Reference in New Issue
Block a user