{
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": ">=9.0"
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    },
    {
      "type": "panel",
      "id": "stat",
      "name": "Stat",
      "version": ""
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "graphTooltip": 0,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "type": "row",
      "title": "Overview",
      "id": 1,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      }
    },
    {
      "type": "stat",
      "title": "Version",
      "id": 53,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "expr": "sum by (git_version) (  kubernetes_build_info{})",
          "legendFormat": "{{git_version}}",
          "instant": true
        }
      ],
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 0,
        "y": 1
      },
      "fieldConfig": {
        "defaults": {
          "unit": "none"
        },
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "orientation": "auto",
        "textMode": "name",
        "graphMode": "none",
        "colorMode": "none",
        "noValue": "unknown"
      }
    },
    {
      "type": "stat",
      "title": "Running replicas",
      "id": 2,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "expr": "count(count by (instance) (apiserver_request_total))"
        }
      ],
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 4,
        "y": 1
      },
      "fieldConfig": {
        "defaults": {
          "unit": "none"
        },
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "type": "stat",
      "title": "Uptime",
      "id": 3,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "expr": "(time() - min(process_start_time_seconds{})) / 3600"
        }
      ],
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 8,
        "y": 1
      },
      "fieldConfig": {
        "defaults": {
          "unit": "h",
          "decimals": 1,
          "color": {
            "mode": "fixed",
            "fixedColor": "green"
          }
        },
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "orientation": "auto",
        "textMode": "value",
        "graphMode": "none",
        "colorMode": "value"
      }
    },
    {
      "type": "stat",
      "title": "Current in-flight requests (total)",
      "id": 5,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "expr": "sum(apiserver_current_inflight_requests{})"
        }
      ],
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 12,
        "y": 1
      },
      "fieldConfig": {
        "defaults": {
          "unit": "none"
        },
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "type": "stat",
      "title": "5xx error % (last 30m)",
      "id": 6,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "expr": "100 * ( sum(rate(apiserver_request_total{code=~\"5..\"}[30m])) ) / clamp_min(sum(rate(apiserver_request_total{}[30m])), 1e-9)"
        }
      ],
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 16,
        "y": 1
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "decimals": 2
        },
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "type": "stat",
      "title": "4xx error % (last 30m)",
      "id": 50,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "expr": "100 * ( sum(rate(apiserver_request_total{code=~\"4..\"}[30m])) ) / clamp_min(sum(rate(apiserver_request_total{}[30m])), 1e-9)"
        }
      ],
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 20,
        "y": 1
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "decimals": 2
        },
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "type": "timeseries",
      "title": "QPS \u2014 total & by verb",
      "id": 8,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "total",
          "expr": "sum(rate(apiserver_request_total{}[5m]))"
        },
        {
          "refId": "B",
          "legendFormat": "{{verb}}",
          "expr": "topk(8, sum by (verb) (rate(apiserver_request_total{}[5m])))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 0,
        "y": 5
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "decimals": 2
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "HTTP status rate by class",
      "id": 9,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "2xx",
          "expr": "sum(rate(apiserver_request_total{code=~\"2..\"}[5m]))"
        },
        {
          "refId": "B",
          "legendFormat": "4xx",
          "expr": "sum(rate(apiserver_request_total{code=~\"4..\"}[5m]))"
        },
        {
          "refId": "C",
          "legendFormat": "5xx",
          "expr": "sum(rate(apiserver_request_total{code=~\"5..\"}[5m]))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 12,
        "y": 5
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "row",
      "title": "Request latency \u2014 deep dive",
      "id": 15,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 11
      }
    },
    {
      "type": "timeseries",
      "title": "Overall request duration \u2014 avg & p50/p90/p99",
      "id": 16,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "avg",
          "legendFormat": "avg",
          "expr": "(sum(rate(apiserver_request_duration_seconds_sum{verb!~\"WATCH|WATCHLIST\"}[5m])) / clamp_min(sum(rate(apiserver_request_duration_seconds_count{verb!~\"WATCH|WATCHLIST\"}[5m])), 1e-9))"
        },
        {
          "refId": "p50",
          "legendFormat": "p50",
          "expr": "histogram_quantile(0.5,  sum by (le) (rate(apiserver_request_duration_seconds_bucket{verb!~\"WATCH|WATCHLIST\"}[5m])))"
        },
        {
          "refId": "p90",
          "legendFormat": "p90",
          "expr": "histogram_quantile(0.9,  sum by (le) (rate(apiserver_request_duration_seconds_bucket{verb!~\"WATCH|WATCHLIST\"}[5m])))"
        },
        {
          "refId": "p99",
          "legendFormat": "p99",
          "expr": "histogram_quantile(0.99, sum by (le) (rate(apiserver_request_duration_seconds_bucket{verb!~\"WATCH|WATCHLIST\"}[5m])))"
        }
      ],
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 12
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "decimals": 3
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Request duration p99 by verb",
      "id": 51,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "{{verb}}",
          "expr": "histogram_quantile(0.99, sum by (le, verb) (rate(apiserver_request_duration_seconds_bucket{verb!~\"WATCH|WATCHLIST\"}[5m])))"
        }
      ],
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 12
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "decimals": 3
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "p99 by verb (mutating vs read-only)",
      "id": 18,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "mutating",
          "expr": "histogram_quantile(0.99, sum by (le) (rate(apiserver_request_duration_seconds_bucket{verb=~\"POST|PUT|PATCH|DELETE\"}[5m])))"
        },
        {
          "refId": "B",
          "legendFormat": "read-only",
          "expr": "histogram_quantile(0.99, sum by (le) (rate(apiserver_request_duration_seconds_bucket{verb=~\"GET|LIST\"}[5m])))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 0,
        "y": 20
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "decimals": 3
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Request duration p90 by verb",
      "id": 52,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "{{verb}}",
          "expr": "histogram_quantile(0.9, sum by (le, verb) (rate(apiserver_request_duration_seconds_bucket{verb!~\"WATCH|WATCHLIST\"}[5m])))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 12,
        "y": 20
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "decimals": 3
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "row",
      "title": "Request mix & hotspots",
      "id": 10,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 26
      }
    },
    {
      "type": "timeseries",
      "title": "Top resources by QPS (top 15)",
      "id": 11,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "{{resource}}",
          "expr": "topk(15, sum by (resource) (rate(apiserver_request_total{}[5m])))"
        }
      ],
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 27
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Top API groups by QPS (top 10)",
      "id": 12,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "{{group}}",
          "expr": "topk(10, sum by (group) (rate(apiserver_request_total{}[5m])))"
        }
      ],
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 27
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Mutating vs read-only QPS",
      "id": 13,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "mutating",
          "expr": "sum(rate(apiserver_request_total{verb=~\"POST|PUT|PATCH|DELETE\"}[5m]))"
        },
        {
          "refId": "B",
          "legendFormat": "read-only",
          "expr": "sum(rate(apiserver_request_total{verb=~\"GET|LIST|WATCH|WATCHLIST\"}[5m]))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 0,
        "y": 35
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Top (resource, verb) by QPS (top 15)",
      "id": 14,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "{{resource}} {{verb}}",
          "expr": "topk(15, sum by (resource, verb) (rate(apiserver_request_total{}[5m])))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 12,
        "y": 35
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "row",
      "title": "Payload sizes",
      "id": 25,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 60
      }
    },
    {
      "type": "timeseries",
      "title": "Response size \u2014 p50/p90/p99",
      "id": 26,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "p50",
          "legendFormat": "p50",
          "expr": "histogram_quantile(0.5,  sum by (le) (rate(apiserver_response_sizes_bucket{}[5m])))"
        },
        {
          "refId": "p90",
          "legendFormat": "p90",
          "expr": "histogram_quantile(0.9,  sum by (le) (rate(apiserver_response_sizes_bucket{}[5m])))"
        },
        {
          "refId": "p99",
          "legendFormat": "p99",
          "expr": "histogram_quantile(0.99, sum by (le) (rate(apiserver_response_sizes_bucket{}[5m])))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 0,
        "y": 61
      },
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Largest responses \u2014 p99 by resource (top 10)",
      "id": 27,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "{{resource}}",
          "expr": "topk(10, histogram_quantile(0.99, sum by (le, resource) (rate(apiserver_response_sizes_bucket{}[5m]))))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 12,
        "y": 61
      },
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "row",
      "title": "Admission, Audit, Watches",
      "id": 33,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 80
      }
    },
    {
      "type": "timeseries",
      "title": "Admission controller duration p90 by controller (top 10)",
      "id": 34,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "{{name}}",
          "expr": "topk(10, histogram_quantile(0.9, sum by (le, name) (rate(apiserver_admission_controller_admission_duration_seconds_bucket{}[5m]))))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 0,
        "y": 81
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "decimals": 3
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Admission webhook p90 by webhook (top 10)",
      "id": 35,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "{{name}} ({{type}})",
          "expr": "topk(10, histogram_quantile(0.9, sum by (le, name, type) (rate(apiserver_admission_webhook_admission_duration_seconds_bucket{}[5m]))))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 12,
        "y": 81
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "decimals": 3
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Audit events (/s)",
      "id": 36,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "events",
          "expr": "sum(rate(apiserver_audit_event_total{}[5m]))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 8,
        "x": 0,
        "y": 87
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "hidden",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Audit rejections (/s)",
      "id": 37,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "rejections",
          "expr": "sum(rate(apiserver_audit_requests_rejected_total{}[5m]))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 8,
        "x": 8,
        "y": 87
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "hidden",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Watch events (/s)",
      "id": 38,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "events",
          "expr": "sum(rate(apiserver_watch_events_total{}[5m]))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 8,
        "x": 16,
        "y": 87
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "hidden",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "row",
      "title": "etcd \u2014 rates & duration",
      "id": 54,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 92
      }
    },
    {
      "type": "timeseries",
      "title": "etcd request rate by operation",
      "id": 39,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "{{operation}}",
          "expr": "sum by (operation) (rate(etcd_request_duration_seconds_count{}[5m]))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 0,
        "y": 94
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "etcd request duration p90 by operation",
      "id": 40,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "{{operation}}",
          "expr": "histogram_quantile(0.9, sum by (le, operation) (rate(etcd_request_duration_seconds_bucket{}[5m])))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 12,
        "y": 94
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "decimals": 3
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "row",
      "title": "Runtime \u2014 Go & process",
      "id": 41,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 99
      }
    },
    {
      "type": "timeseries",
      "title": "CPU (cores)",
      "id": 42,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "cpu",
          "expr": "sum(rate(process_cpu_seconds_total{}[5m]))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 0,
        "y": 100
      },
      "fieldConfig": {
        "defaults": {
          "unit": "none",
          "decimals": 2
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "hidden",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Memory RSS",
      "id": 43,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "rss",
          "expr": "sum(process_resident_memory_bytes{})"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 6,
        "y": 100
      },
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "hidden",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Go heap alloc",
      "id": 44,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "heap_alloc",
          "expr": "sum(go_memstats_heap_alloc_bytes{})"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 12,
        "y": 100
      },
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "hidden",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Go GC duration \u2014 p99",
      "id": 45,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "gc p99",
          "expr": "histogram_quantile(0.99, sum by (le) (rate(go_gc_duration_seconds_bucket{}[5m])))"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 18,
        "y": 100
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "decimals": 3
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "hidden",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Goroutines",
      "id": 46,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "goroutines",
          "expr": "sum(go_goroutines{})"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 0,
        "y": 106
      },
      "fieldConfig": {
        "defaults": {
          "unit": "none"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "hidden",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Threads",
      "id": 47,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "threads",
          "expr": "sum(go_threads{})"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 6,
        "y": 106
      },
      "fieldConfig": {
        "defaults": {
          "unit": "none"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "hidden",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "Open FDs",
      "id": 48,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "fds",
          "expr": "sum(process_open_fds{})"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 12,
        "y": 106
      },
      "fieldConfig": {
        "defaults": {
          "unit": "none"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "hidden",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "single"
        }
      }
    },
    {
      "type": "timeseries",
      "title": "FD usage %",
      "id": 49,
      "datasource": "$DS_PROMETHEUS",
      "targets": [
        {
          "refId": "A",
          "legendFormat": "used %",
          "expr": "100 * sum(process_open_fds{}) / clamp_min(sum(process_max_fds{}), 1)"
        }
      ],
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 18,
        "y": 106
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "decimals": 1,
          "min": 0,
          "max": 100
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "hidden",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "single"
        }
      }
    }
  ],
  "refresh": "30s",
  "schemaVersion": 39,
  "style": "dark",
  "tags": [
    "karmada",
    "karmada-apiserver"
  ],
  "templating": {
    "list": [
      {
        "type": "datasource",
        "name": "DS_PROMETHEUS",
        "label": "Prometheus",
        "query": "prometheus",
        "hide": 0
      }
    ]
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h"
    ],
    "time_options": [
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "browser",
  "title": "Karmada API Server Insights",
  "uid": "karmada-apiserver-insights",
  "version": 6,
  "weekStart": ""
}