apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: machine-config-daemon
  namespace: openshift-machine-config-operator
  labels:
    k8s-app: machine-config-daemon
spec:
  groups:
    - name: mcd-reboot-error
      rules:
        - alert: MCDRebootError
          expr: |
             mcd_reboot_err > 0
          labels:
            severity: critical
          annotations:
            message: "Reboot failed on {{ $labels.node }} , update may be blocked"
    - name: mcd-drain-error
      rules:
        - alert: MCDDrainError
          expr: |
            mcd_drain_err > 0
          labels:
            severity: warning
          for: 15m
          annotations:
            message: "Drain failed on {{ $labels.node }} , updates may be blocked. For more details:  oc logs -f -n openshift-machine-config-operator machine-config-daemon-<hash> -c machine-config-daemon"
    - name: mcd-pivot-error
      rules:
        - alert: MCDPivotError
          expr: |
            mcd_pivot_err > 0
          labels:
            severity: warning
          annotations:
            message: "Error detected in pivot logs on {{ $labels.node }} "
    - name: mcd-kubelet-health-state-error
      rules:
        - alert: KubeletHealthState
          expr: |
            mcd_kubelet_state > 2
          labels:
            severity: warning
          annotations:
            message: "Kubelet health failure threshold reached"
    - name: system-memory-exceeds-reservation
      rules:
        - alert: SystemMemoryExceedsReservation
          expr: |
            sum by (node) (container_memory_rss{id="/system.slice"}) > ((sum by (node) (kube_node_status_capacity{resource="memory"} - kube_node_status_allocatable{resource="memory"})) * 0.9)
          labels:
            severity: warning
          annotations:
            message: "System memory usage of {{ $value | humanize }} on {{ $labels.node }} exceeds 90% of the reservation. Reserved memory ensures system processes can function even when the node is fully allocated and protects against workload out of memory events impacting the proper functioning of the node. The reservation may be increased (https://docs.openshift.com/container-platform/latest/nodes/nodes/nodes-nodes-managing.html) when running nodes with high numbers of pods."
