"groups":
- "name": "logging_fluentd.alerts"
  "rules":
  - "alert": "FluentdNodeDown"
    "annotations":
      "message": "Prometheus could not scrape fluentd {{ $labels.instance }} for more than 10m."
      "summary": "Fluentd cannot be scraped"
    "expr": |
      absent(up{job="collector"} == 1)
    "for": "10m"
    "labels":
      "service": "fluentd"
      "severity": "critical"
  - "alert": "FluentdQueueLengthIncreasing"
    "annotations":
      "message": "For the last hour, fluentd {{ $labels.instance }} average buffer queue length has increased continuously."
      "summary": "Fluentd unable to keep up with traffic over time."
    "expr": |
      deriv(fluentd_output_status_buffer_queue_length[10m]) > 0 and delta(fluentd_output_status_buffer_queue_length[1h]) > 1
    "for": "1h"
    "labels":
      "service": "fluentd"
      "severity": "error"
  - alert: FluentDHighErrorRate
    annotations:
      message: |-
        {{ $value }}% of records have resulted in an error by fluentd {{ $labels.instance }}.
      summary: FluentD output errors are high
    expr: |
      100 * (
        sum by(instance)(rate(fluentd_output_status_num_errors[2m]))
      /
        sum by(instance)(rate(fluentd_output_status_emit_records[2m]))
      ) > 10
    for: 15m
    labels:
      severity: warning

  - alert: FluentDVeryHighErrorRate
    annotations:
      message: |-
        {{ $value }}% of records have resulted in an error by fluentd {{ $labels.instance }}.
      summary: FluentD output errors are very high
    expr: |
      100 * (
        sum by(instance)(rate(fluentd_output_status_num_errors[2m]))
      /
        sum by(instance)(rate(fluentd_output_status_emit_records[2m]))
      ) > 25
    for: 15m
    labels:
      severity: critical

