promethues 쿼리 몇 가지

kubernetes에 prometheus & grafana로 구성하고, grafana 에서 사용한 prometheus 쿼리 몇가지 정리 합니다.

버전

• Prometheus 2.26.0
• Grafana 8.3.0

쿼리

Stat

• namespace 개수: count(kube_namespace_created)
• pvc 개수: count(kube_presistentvolumeclain_info)
• pod 개수: count(count by(pod)(container_spec_memory_reservation_limit_bytes{pod!=""}))
• container 개수: count(kube_pod_container_status_running{namespace!="/"})
• Replica Rate: sum(kube_replicaset_status_ready_replicas)/sum(kube_replicaset_status_replicas)
• Master 정상 가동률: avg(avg_over_time((sum without()(kube_pod_container_Status_ready{namespace="kube-system}) / count without()(kube_pod_container_status_ready{namespace='kube-system'}))[5m:5m]))

Time series

• 초당 네트워크 트래픽:
  • in: sum(rate(node_network_receive_bytes_total[5m]))
  • out: sum(rate(node_network_transmit_bytes_total[5m]))
• API Server 요청 개수: sum by (verb) (rate(apiserver_request_total[5m]))
• node별 CPU 사용률: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
• node별 memory 사용률: (node_memory_MemTotal_bytes{}-node_memory_MemAvailable_bytes{})/node_memory_MemTotal_bytes{}
• Ready 상태의 pod 개수: sum(kube_pod_status_ready{condition="true"})
• Running 상태의 container 개수: count(kube_pod_container_status_running{namespace!="/"})
• pod별 CPU 사용률: rate(container_cpu_usage_seconds_total[5m])
• pod별 메모리 사용률: avg(container_memory_working_set_bytes / on (container, pod) kube_pod_container_resource_limits{resource="memory"}) by (pod)

Table

• Node 현황

{
  {
    "datasource": {
      "type": "prometheus""uid": "iu5kt9I4K"
    },
    "fieldConfig": {
      "defaults": {
        "custom": {
          "align": null
        },
        "mappings": [],
        "thresholds": {
          "mode": "absolute",
          "steps": [
            {
              "color": "green",
              "value": null
            },
            {
              "color": "red",
              "value": 80
            }
          ]
        }
      },
      "overrides": [
        {
          "matcher": {
            "id": "byName",
            "options": "Docker version"
          },
          "properties": [
            {
              "id": "custom.width",
              "value": 161
            }
          ]
        },
        {
          "matcher": {
            "id": "byName",
            "options": "Node name"
          },
          "properties": [
            {
              "id": "custom.width",
              "value": 180
            }
          ]
        },
        {
          "matcher": {
            "id": "byName",
            "options": "K8s version"
          },
          "properties": [
            {
              "id": "custom.width",
              "value": 114
            }
          ]
        },
        {
          "matcher": {
            "id": "byName",
            "options": "OS version"
          },
          "properties": [
            {
              "id": "custom.width",
              "value": 249
            }
          ]
        },
        {
          "matcher": {
            "id": "byName",
            "options": "pod_cidr"
          },
          "properties": [
            {
              "id": "custom.width",
              "value": 113
            }
          ]
        },
        {
          "matcher": {
            "id": "byName",
            "options": "Kernel version"
          },
          "properties": [
            {
              "id": "custom.width",
              "value": 297
            }
          ]
        },
        {
          "matcher": {
            "id": "byName",
            "options": "role"
          },
          "properties": [
            {
              "id": "custom.width",
              "value": 113
            }
          ]
        },
        {
          "matcher": {
            "id": "byName",
            "options": "Node status"
          },
          "properties": [
            {
              "id": "custom.width",
              "value": 124
            },
            {
              "id": "mappings",
              "value": [
                {
                  "from": "",
                  "id": 0,
                  "operator": "",
                  "text": "Good",
                  "to": "",
                  "type": 1,
                  "value": "1"
                },
                {
                  "from": "",
                  "id": 1,
                  "operator": "",
                  "text": "Bad",
                  "to": "",
                  "type": 1,
                  "value": "0"
                }
              ]
            },
            {
              "id": "custom.displayMode",
              "value": "color-background"
            },
            {
              "id": "custom.align",
              "value": "center"
            }
          ]
        },
        {
          "matcher": {
            "id": "byName",
            "options": "Uptime"
          },
          "properties": [
            {
              "id": "unit",
              "value": "s"
            },
            {
              "id": "custom.width",
              "value": 110
            },
            {
              "id": "custom.align",
              "value": "left"
            }
          ]
        },
        {
          "matcher": {
            "id": "byName",
            "options": "POD CIDR"
          },
          "properties": [
            {
              "id": "custom.width",
              "value": 120
            }
          ]
        },
        {
          "matcher": {
            "id": "byName",
            "options": "Pod CIDR"
          },
          "properties": [
            {
              "id": "custom.width",
              "value": 188
            }
          ]
        }
      ]
    },
    "gridPos": {
      "h": 6,
      "w": 15,
      "x": 9,
      "y": 1
    },
    "id": 64,
    "options": {
      "angular": {
        "decimals": 0,
        "defaultColor": "#37872D",
        "format": "none",
        "linkIndex": "0",
        "linkVars": [],
        "mathColorValue": "data[end]",
        "mathDisplayValue": "data[end]",
        "mathScratchPad": "data = size(data)[1] == 0 ? [NaN] : data",
        "options": {},
        "radius": "25px",
        "thresholds": [
          {
            "color": "#37872D",
            "value": "1"
          },
          {
            "color": "#C4162A",
            "value": "0"
          }
        ]
      },
      "frameIndex": 0,
      "showHeader": true,
      "sortBy": [
        {
          "desc": false,
          "displayName": "Node name"
        }
      ]
    },
    "pluginVersion": "7.0.3",
    "targets": [
      {
        "expr": "kube_node_info",
        "format": "table",
        "hide": false,
        "instant": true,
        "interval": "",
        "legendFormat": "{{node}}",
        "refId": "A"
      },
      {
        "expr": " (count by (node)(container_spec_memory_reservation_limit_bytes{pod!=\"\"}))",
        "format": "table",
        "hide": false,
        "instant": true,
        "interval": "",
        "legendFormat": "",
        "refId": "B"
      },
      {
        "expr": "max\n by(node) (max by(instance) (kubelet_running_pod_count{job=\"kubelet\",metrics_path=\"/metrics\"})\n * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\",metrics_path=\"/metrics\"})\n",
        "format": "table",
        "instant": true,
        "interval": "",
        "legendFormat": "",
        "refId": "D"
      },
      {
        "expr": "sum by (node)(label_replace(instance:node_cpu:ratio + on(instance) group_left(nodename) \nnode_uname_info,\"node\",\"$1\",\"nodename\", \"(.+)\") ) / sum by (node)(kube_node_status_capacity{resource=\"cpu\"})",
        "format": "table",
        "instant": true,
        "interval": "",
        "legendFormat": "",
        "refId": "E"
      },
      {
        "expr": "kube_node_role",
        "format": "table",
        "interval": "",
        "legendFormat": "",
        "refId": "F"
      },
      {
        "expr": "label_replace(up{job=\"kubernetes-nodes\"}, \"node\", \"$1\", \"instance\", \"(.+)\")",
        "format": "table",
        "interval": "",
        "legendFormat": "",
        "refId": "G"
      },
      {
        "expr": "label_replace(avg by (kubernetes_node) (time() - node_boot_time_seconds), \"node\", \"$1\", \"kubernetes_node\", \"(.+)\")",
        "format": "table",
        "instant": true,
        "interval": "",
        "legendFormat": "",
        "refId": "H"
      }
    ],
    "timeFrom": null,
    "timeShift": null,
    "title": "노드 현황",
    "transformations": [
      {
        "id": "filterFieldsByName",
        "options": {
          "include": {
            "names": [
              "container_runtime_version",
              "kernel_version",
              "kubelet_version",
              "node",
              "os_image",
              "pod_cidr",
              "role",
              "Value #G",
              "Value #H"
            ]
          }
        }
      },
      {
        "id": "seriesToColumns",
        "options": {
          "byField": "node"
        }
      },
      {
        "id": "organize",
        "options": {
          "excludeByName": {},
          "indexByName": {
            "Value #G": 8,
            "Value #H": 9,
            "container_runtime_version": 7,
            "instance": 2,
            "kernel_version": 5,
            "kubelet_version": 6,
            "node": 0,
            "os_image": 4,
            "pod_cidr": 3,
            "role": 1
          },
          "renameByName": {
            "Value #C": "IP",
            "Value #D": "Pod count",
            "Value #E": "CPU usage",
            "Value #G": "Node status",
            "Value #H": "Uptime",
            "container_runtime_version": "Docker version",
            "instance": "IP",
            "kernel_version": "Kernel version",
            "kubelet_version": "K8s version",
            "node": "Node name",
            "os_image": "OS version",
            "pod_cidr": "Pod CIDR",
            "provider_id": ""
          }
        }
      }
    ],
    "type": "table"
  },