# Prometheus alert rules — auto-generated by Ansible
# DO NOT edit manually; this Jinja2 template IS the source.
groups:
  - name: app_availability
    interval: 30s
    rules:
      - alert: AppDown
        expr: probe_success{job="blackbox-healthz"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Application {{ '{{' }} $labels.app {{ '}}' }} is down"
          description: >
            {{ '{{' }} $labels.instance {{ '}}' }} has been unreachable for more than 2 minutes.
            Check the application container and upstream connectivity.

      - alert: TlsCertificateExpiringSoon
        expr: (probe_ssl_earliest_cert_expiry{job="blackbox-healthz"} - time()) < 14 * 86400
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "TLS certificate for {{ '{{' }} $labels.app {{ '}}' }} expires in less than 14 days"
          description: >
            The certificate served at {{ '{{' }} $labels.instance {{ '}}' }}
            (app: {{ '{{' }} $labels.app {{ '}}' }}) expires soon.
            Check certificate renewal and sys-ctl-mtn-cert-deploy.

      - alert: TlsCertificateExpiryCritical
        expr: (probe_ssl_earliest_cert_expiry{job="blackbox-healthz"} - time()) < 3 * 86400
        for: 30m
        labels:
          severity: critical
        annotations:
          summary: "TLS certificate for {{ '{{' }} $labels.app {{ '}}' }} expires in less than 3 days"
          description: >
            The certificate served at {{ '{{' }} $labels.instance {{ '}}' }}
            (app: {{ '{{' }} $labels.app {{ '}}' }}) is about to expire.
            Renew immediately; clients will start failing at expiry.

      - alert: AppHighLatency
        expr: >
          histogram_quantile(0.95,
            rate(nginx_http_request_duration_seconds_bucket[5m])
          ) > 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High latency on {{ '{{' }} $labels.app {{ '}}' }} ({{ '{{' }} $labels.host {{ '}}' }})"
          description: >
            95th percentile request latency on {{ '{{' }} $labels.host {{ '}}' }}
            (app: {{ '{{' }} $labels.app {{ '}}' }}) exceeds 2s
            (current: {{ '{{' }} $value | humanizeDuration {{ '}}' }}).

      - alert: AppHighErrorRate
        expr: >
          rate(nginx_http_requests_total{status=~"5.."}[5m])
          /
          rate(nginx_http_requests_total[5m]) > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High 5xx error rate on {{ '{{' }} $labels.app {{ '}}' }} ({{ '{{' }} $labels.host {{ '}}' }})"
          description: >
            More than 5% of requests on {{ '{{' }} $labels.host {{ '}}' }}
            (app: {{ '{{' }} $labels.app {{ '}}' }}) are returning 5xx errors
            (current: {{ '{{' }} $value | humanizePercentage {{ '}}' }}).

  - name: communication_channels
    rules:
{% set _comm_channels = lookup('active_alertmanager_channels') %}
{% if _comm_channels %}
      - alert: CommunicationChannelDown
        expr: >
          probe_success{job="blackbox-healthz",
            app=~"{{ _comm_channels | join('|') }}"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Communication channel {{ '{{' }} $labels.app {{ '}}' }} is down"
          description: >
            {{ '{{' }} $labels.instance {{ '}}' }} ({{ '{{' }} $labels.app {{ '}}' }})
            has been unreachable for more than 2 minutes.
            This is a communication-critical application.
{% endif %}

  - name: prometheus_health
    rules:
      - alert: PrometheusTargetMissing
        expr: up == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Prometheus target missing: {{ '{{' }} $labels.instance {{ '}}' }}"
          description: >
            Prometheus target {{ '{{' }} $labels.instance {{ '}}' }} (job: {{ '{{' }} $labels.job {{ '}}' }})
            has been unreachable for more than 5 minutes.

{% for alert_rules_role_id in lookup('alert_rule_roles') %}
{% include 'roles/' + alert_rules_role_id + '/templates/prometheus/alert_rules.yml.j2' %}

{% endfor %}