apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    app.kubernetes.io/part-of: openshift-monitoring
    prometheus: k8s
    role: alert-rules
  name: etcd-prometheus-rules
  namespace: openshift-monitoring
spec:
  groups:
  - name: etcd
    rules:
    - alert: etcdMembersDown
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": members are down ({{ $value
          }}).'
        summary: etcd cluster members are down.
      expr: |
        max without (endpoint) (
          sum without (instance) (up{job=~".*etcd.*"} == bool 0)
        or
          count without (To) (
            sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01
          )
        )
        > 0
      for: 10m
      labels:
        severity: critical
    - alert: etcdNoLeader
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": member {{ $labels.instance
          }} has no leader.'
        summary: etcd cluster has no leader.
      expr: |
        etcd_server_has_leader{job=~".*etcd.*"} == 0
      for: 1m
      labels:
        severity: critical
    - alert: etcdHighNumberOfLeaderChanges
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }} leader changes
          within the last 15 minutes. Frequent elections may be a sign of insufficient
          resources, high network latency, or disruptions by other components and
          should be investigated.'
        summary: etcd cluster has high number of leader changes.
      expr: |
        increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4
      for: 5m
      labels:
        severity: warning
    - alert: etcdGRPCRequestsSlow
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": gRPC requests to {{ $labels.grpc_method
          }} are taking {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd grpc requests are slow
      expr: |
        histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) without(grpc_type))
        > 0.15
      for: 10m
      labels:
        severity: critical
    - alert: etcdMemberCommunicationSlow
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": member communication with
          {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance
          }}.'
        summary: etcd cluster member communication is slow.
      expr: |
        histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.15
      for: 10m
      labels:
        severity: warning
    - alert: etcdHighNumberOfFailedProposals
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures
          within the last 30 minutes on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster has high number of proposal failures.
      expr: |
        rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
      for: 15m
      labels:
        severity: warning
    - alert: etcdHighFsyncDurations
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations
          are {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster 99th percentile fsync durations are too high.
      expr: |
        histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.5
      for: 10m
      labels:
        severity: warning
    - alert: etcdHighFsyncDurations
      annotations:
        message: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations
          are {{ $value }}s on etcd instance {{ $labels.instance }}.'
      expr: |
        histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 1
      for: 10m
      labels:
        severity: critical
    - alert: etcdHighCommitDurations
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations
          {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster 99th percentile commit durations are too high.
      expr: |
        histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.25
      for: 10m
      labels:
        severity: warning
    - alert: etcdBackendQuotaLowSpace
      annotations:
        message: 'etcd cluster "{{ $labels.job }}": database size exceeds the defined
          quota on etcd instance {{ $labels.instance }}, please defrag or increase
          the quota as the writes to etcd will be disabled when it is full.'
      expr: |
        (etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100 > 95
      for: 10m
      labels:
        severity: critical
    - alert: etcdExcessiveDatabaseGrowth
      annotations:
        message: 'etcd cluster "{{ $labels.job }}": Observed surge in etcd writes
          leading to 50% increase in database size over the past four hours on etcd
          instance {{ $labels.instance }}, please check as it might be disruptive.'
      expr: |
        increase(((etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100)[240m:1m]) > 50
      for: 10m
      labels:
        severity: warning
