apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    app.kubernetes.io/component: query-layer
    app.kubernetes.io/instance: thanos-querier
    app.kubernetes.io/name: thanos-query
    app.kubernetes.io/version: 0.20.2
  name: thanos-querier
  namespace: openshift-monitoring
spec:
  groups:
  - name: thanos-query
    rules:
    - alert: ThanosQueryHttpRequestQueryErrorRateHigh
      annotations:
        description: Thanos Query {{$labels.job}} is failing to handle {{ $value |
          humanize }}% of "query" requests.
        summary: Thanos Query is failing to handle requests.
      expr: |
        (
          sum(rate(http_requests_total{code=~"5..", job="thanos-querier", handler="query"}[5m]))
        /
          sum(rate(http_requests_total{job="thanos-querier", handler="query"}[5m]))
        ) * 100 > 5
      for: 1h
      labels:
        severity: warning
    - alert: ThanosQueryHttpRequestQueryRangeErrorRateHigh
      annotations:
        description: Thanos Query {{$labels.job}} is failing to handle {{ $value |
          humanize }}% of "query_range" requests.
        summary: Thanos Query is failing to handle requests.
      expr: |
        (
          sum(rate(http_requests_total{code=~"5..", job="thanos-querier", handler="query_range"}[5m]))
        /
          sum(rate(http_requests_total{job="thanos-querier", handler="query_range"}[5m]))
        ) * 100 > 5
      for: 1h
      labels:
        severity: warning
    - alert: ThanosQueryGrpcServerErrorRate
      annotations:
        description: Thanos Query {{$labels.job}} is failing to handle {{ $value |
          humanize }}% of requests.
        summary: Thanos Query is failing to handle requests.
      expr: |
        (
          sum by (job) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job="thanos-querier"}[5m]))
        /
          sum by (job) (rate(grpc_server_started_total{job="thanos-querier"}[5m]))
        * 100 > 5
        )
      for: 1h
      labels:
        severity: warning
    - alert: ThanosQueryGrpcClientErrorRate
      annotations:
        description: Thanos Query {{$labels.job}} is failing to send {{ $value | humanize
          }}% of requests.
        summary: Thanos Query is failing to send requests.
      expr: |
        (
          sum by (job) (rate(grpc_client_handled_total{grpc_code!="OK", job="thanos-querier"}[5m]))
        /
          sum by (job) (rate(grpc_client_started_total{job="thanos-querier"}[5m]))
        ) * 100 > 5
      for: 1h
      labels:
        severity: warning
    - alert: ThanosQueryHighDNSFailures
      annotations:
        description: Thanos Query {{$labels.job}} have {{ $value | humanize }}% of
          failing DNS queries for store endpoints.
        summary: Thanos Query is having high number of DNS failures.
      expr: |
        (
          sum by (job) (rate(thanos_query_store_apis_dns_failures_total{job="thanos-querier"}[5m]))
        /
          sum by (job) (rate(thanos_query_store_apis_dns_lookups_total{job="thanos-querier"}[5m]))
        ) * 100 > 1
      for: 1h
      labels:
        severity: warning
