mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 19:01:54 +00:00
108 lines
4.2 KiB
Django/Jinja
108 lines
4.2 KiB
Django/Jinja
# Prometheus alert rules — auto-generated by Ansible
|
|
# DO NOT edit manually; this Jinja2 template IS the source.
|
|
groups:
|
|
- name: app_availability
|
|
interval: 30s
|
|
rules:
|
|
- alert: AppDown
|
|
expr: probe_success{job="blackbox-healthz"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Application {{ '{{' }} $labels.app {{ '}}' }} is down"
|
|
description: >
|
|
{{ '{{' }} $labels.instance {{ '}}' }} has been unreachable for more than 2 minutes.
|
|
Check the application container and upstream connectivity.
|
|
|
|
- alert: TlsCertificateExpiringSoon
|
|
expr: (probe_ssl_earliest_cert_expiry{job="blackbox-healthz"} - time()) < 14 * 86400
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "TLS certificate for {{ '{{' }} $labels.app {{ '}}' }} expires in less than 14 days"
|
|
description: >
|
|
The certificate served at {{ '{{' }} $labels.instance {{ '}}' }}
|
|
(app: {{ '{{' }} $labels.app {{ '}}' }}) expires soon.
|
|
Check certificate renewal and sys-ctl-mtn-cert-deploy.
|
|
|
|
- alert: TlsCertificateExpiryCritical
|
|
expr: (probe_ssl_earliest_cert_expiry{job="blackbox-healthz"} - time()) < 3 * 86400
|
|
for: 30m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "TLS certificate for {{ '{{' }} $labels.app {{ '}}' }} expires in less than 3 days"
|
|
description: >
|
|
The certificate served at {{ '{{' }} $labels.instance {{ '}}' }}
|
|
(app: {{ '{{' }} $labels.app {{ '}}' }}) is about to expire.
|
|
Renew immediately; clients will start failing at expiry.
|
|
|
|
- alert: AppHighLatency
|
|
expr: >
|
|
histogram_quantile(0.95,
|
|
rate(nginx_http_request_duration_seconds_bucket[5m])
|
|
) > 2
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High latency on {{ '{{' }} $labels.app {{ '}}' }} ({{ '{{' }} $labels.host {{ '}}' }})"
|
|
description: >
|
|
95th percentile request latency on {{ '{{' }} $labels.host {{ '}}' }}
|
|
(app: {{ '{{' }} $labels.app {{ '}}' }}) exceeds 2s
|
|
(current: {{ '{{' }} $value | humanizeDuration {{ '}}' }}).
|
|
|
|
- alert: AppHighErrorRate
|
|
expr: >
|
|
rate(nginx_http_requests_total{status=~"5.."}[5m])
|
|
/
|
|
rate(nginx_http_requests_total[5m]) > 0.05
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High 5xx error rate on {{ '{{' }} $labels.app {{ '}}' }} ({{ '{{' }} $labels.host {{ '}}' }})"
|
|
description: >
|
|
More than 5% of requests on {{ '{{' }} $labels.host {{ '}}' }}
|
|
(app: {{ '{{' }} $labels.app {{ '}}' }}) are returning 5xx errors
|
|
(current: {{ '{{' }} $value | humanizePercentage {{ '}}' }}).
|
|
|
|
- name: communication_channels
|
|
rules:
|
|
{% set _comm_channels = lookup('active_alertmanager_channels') %}
|
|
{% if _comm_channels %}
|
|
- alert: CommunicationChannelDown
|
|
expr: >
|
|
probe_success{job="blackbox-healthz",
|
|
app=~"{{ _comm_channels | join('|') }}"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Communication channel {{ '{{' }} $labels.app {{ '}}' }} is down"
|
|
description: >
|
|
{{ '{{' }} $labels.instance {{ '}}' }} ({{ '{{' }} $labels.app {{ '}}' }})
|
|
has been unreachable for more than 2 minutes.
|
|
This is a communication-critical application.
|
|
{% endif %}
|
|
|
|
- name: prometheus_health
|
|
rules:
|
|
- alert: PrometheusTargetMissing
|
|
expr: up == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Prometheus target missing: {{ '{{' }} $labels.instance {{ '}}' }}"
|
|
description: >
|
|
Prometheus target {{ '{{' }} $labels.instance {{ '}}' }} (job: {{ '{{' }} $labels.job {{ '}}' }})
|
|
has been unreachable for more than 5 minutes.
|
|
|
|
{% for alert_rules_role_id in lookup('alert_rule_roles') %}
|
|
{% include 'roles/' + alert_rules_role_id + '/templates/prometheus/alert_rules.yml.j2' %}
|
|
|
|
{% endfor %}
|