{
   "__inputs": [ ],
   "__requires": [ ],
   "annotations": {
      "list": [ ]
   },
   "editable": false,
   "gnetId": null,
   "graphTooltip": 0,
   "hideControls": false,
   "id": null,
   "links": [ ],
   "panels": [
      {
         "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.",
         "datasource": null,
         "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.",
         "gridPos": {
            "h": 2,
            "w": 24,
            "x": 0,
            "y": 0
         },
         "id": 2,
         "mode": "markdown",
         "span": 12,
         "title": "Notice",
         "type": "text"
      }
   ],
   "refresh": "10s",
   "rows": [
      {
         "collapse": false,
         "collapsed": false,
         "panels": [
            {
               "cacheTimeout": null,
               "colorBackground": false,
               "colorValue": false,
               "colors": [
                  "#299c46",
                  "rgba(237, 129, 40, 0.89)",
                  "#d44a3a"
               ],
               "datasource": "$datasource",
               "decimals": 3,
               "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?",
               "format": "percentunit",
               "gauge": {
                  "maxValue": 100,
                  "minValue": 0,
                  "show": false,
                  "thresholdLabels": false,
                  "thresholdMarkers": true
               },
               "gridPos": { },
               "id": 3,
               "interval": null,
               "links": [ ],
               "mappingType": 1,
               "mappingTypes": [
                  {
                     "name": "value to text",
                     "value": 1
                  },
                  {
                     "name": "range to text",
                     "value": 2
                  }
               ],
               "maxDataPoints": 100,
               "nullPointMode": "connected",
               "nullText": null,
               "postfix": "",
               "postfixFontSize": "50%",
               "prefix": "",
               "prefixFontSize": "50%",
               "rangeMaps": [
                  {
                     "from": "null",
                     "text": "N/A",
                     "to": "null"
                  }
               ],
               "span": 4,
               "sparkline": {
                  "fillColor": "rgba(31, 118, 189, 0.18)",
                  "full": false,
                  "lineColor": "rgb(31, 120, 193)",
                  "show": false
               },
               "tableColumn": "",
               "targets": [
                  {
                     "expr": "apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "",
                     "refId": "A"
                  }
               ],
               "thresholds": "",
               "title": "Availability (30d) > 99.000%",
               "tooltip": {
                  "shared": false
               },
               "type": "singlestat",
               "valueFontSize": "80%",
               "valueMaps": [
                  {
                     "op": "=",
                     "text": "N/A",
                     "value": "null"
                  }
               ],
               "valueName": "avg"
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "decimals": 3,
               "description": "How much error budget is left looking at our 0.990% availability guarantees?",
               "fill": 10,
               "fillGradient": 0,
               "gridPos": { },
               "id": 4,
               "legend": {
                  "alignAsTable": false,
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "rightSide": false,
                  "show": true,
                  "sideWidth": null,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 8,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "errorbudget",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "ErrorBudget (30d) > 99.000%",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "decimals": 3,
                     "format": "percentunit",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  },
                  {
                     "decimals": 3,
                     "format": "percentunit",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": false,
         "title": "Dashboard Row",
         "titleSize": "h6",
         "type": "row"
      },
      {
         "collapse": false,
         "collapsed": false,
         "panels": [
            {
               "cacheTimeout": null,
               "colorBackground": false,
               "colorValue": false,
               "colors": [
                  "#299c46",
                  "rgba(237, 129, 40, 0.89)",
                  "#d44a3a"
               ],
               "datasource": "$datasource",
               "decimals": 3,
               "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?",
               "format": "percentunit",
               "gauge": {
                  "maxValue": 100,
                  "minValue": 0,
                  "show": false,
                  "thresholdLabels": false,
                  "thresholdMarkers": true
               },
               "gridPos": { },
               "id": 5,
               "interval": null,
               "links": [ ],
               "mappingType": 1,
               "mappingTypes": [
                  {
                     "name": "value to text",
                     "value": 1
                  },
                  {
                     "name": "range to text",
                     "value": 2
                  }
               ],
               "maxDataPoints": 100,
               "nullPointMode": "connected",
               "nullText": null,
               "postfix": "",
               "postfixFontSize": "50%",
               "prefix": "",
               "prefixFontSize": "50%",
               "rangeMaps": [
                  {
                     "from": "null",
                     "text": "N/A",
                     "to": "null"
                  }
               ],
               "span": 3,
               "sparkline": {
                  "fillColor": "rgba(31, 118, 189, 0.18)",
                  "full": false,
                  "lineColor": "rgb(31, 120, 193)",
                  "show": false
               },
               "tableColumn": "",
               "targets": [
                  {
                     "expr": "apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "",
                     "refId": "A"
                  }
               ],
               "thresholds": "",
               "title": "Read Availability (30d)",
               "tooltip": {
                  "shared": false
               },
               "type": "singlestat",
               "valueFontSize": "80%",
               "valueMaps": [
                  {
                     "op": "=",
                     "text": "N/A",
                     "value": "null"
                  }
               ],
               "valueName": "avg"
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "How many read requests (LIST,GET) per second do the apiservers get by code?",
               "fill": 10,
               "fillGradient": 0,
               "gridPos": { },
               "id": 6,
               "legend": {
                  "alignAsTable": false,
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "rightSide": false,
                  "show": true,
                  "sideWidth": null,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [
                  {
                     "alias": "/2../i",
                     "color": "#56A64B"
                  },
                  {
                     "alias": "/3../i",
                     "color": "#F2CC0C"
                  },
                  {
                     "alias": "/4../i",
                     "color": "#3274D9"
                  },
                  {
                     "alias": "/5../i",
                     "color": "#E02F44"
                  }
               ],
               "spaceLength": 10,
               "span": 3,
               "stack": true,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{ code }}",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Read SLI - Requests",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "reqps",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  },
                  {
                     "format": "reqps",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?",
               "fill": 1,
               "fillGradient": 0,
               "gridPos": { },
               "id": 7,
               "legend": {
                  "alignAsTable": false,
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "rightSide": false,
                  "show": true,
                  "sideWidth": null,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 3,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{ resource }}",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Read SLI - Errors",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "percentunit",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "percentunit",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?",
               "fill": 1,
               "fillGradient": 0,
               "gridPos": { },
               "id": 8,
               "legend": {
                  "alignAsTable": false,
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "rightSide": false,
                  "show": true,
                  "sideWidth": null,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 3,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{ resource }}",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Read SLI - Duration",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "s",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  },
                  {
                     "format": "s",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": false,
         "title": "Dashboard Row",
         "titleSize": "h6",
         "type": "row"
      },
      {
         "collapse": false,
         "collapsed": false,
         "panels": [
            {
               "cacheTimeout": null,
               "colorBackground": false,
               "colorValue": false,
               "colors": [
                  "#299c46",
                  "rgba(237, 129, 40, 0.89)",
                  "#d44a3a"
               ],
               "datasource": "$datasource",
               "decimals": 3,
               "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?",
               "format": "percentunit",
               "gauge": {
                  "maxValue": 100,
                  "minValue": 0,
                  "show": false,
                  "thresholdLabels": false,
                  "thresholdMarkers": true
               },
               "gridPos": { },
               "id": 9,
               "interval": null,
               "links": [ ],
               "mappingType": 1,
               "mappingTypes": [
                  {
                     "name": "value to text",
                     "value": 1
                  },
                  {
                     "name": "range to text",
                     "value": 2
                  }
               ],
               "maxDataPoints": 100,
               "nullPointMode": "connected",
               "nullText": null,
               "postfix": "",
               "postfixFontSize": "50%",
               "prefix": "",
               "prefixFontSize": "50%",
               "rangeMaps": [
                  {
                     "from": "null",
                     "text": "N/A",
                     "to": "null"
                  }
               ],
               "span": 3,
               "sparkline": {
                  "fillColor": "rgba(31, 118, 189, 0.18)",
                  "full": false,
                  "lineColor": "rgb(31, 120, 193)",
                  "show": false
               },
               "tableColumn": "",
               "targets": [
                  {
                     "expr": "apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "",
                     "refId": "A"
                  }
               ],
               "thresholds": "",
               "title": "Write Availability (30d)",
               "tooltip": {
                  "shared": false
               },
               "type": "singlestat",
               "valueFontSize": "80%",
               "valueMaps": [
                  {
                     "op": "=",
                     "text": "N/A",
                     "value": "null"
                  }
               ],
               "valueName": "avg"
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?",
               "fill": 10,
               "fillGradient": 0,
               "gridPos": { },
               "id": 10,
               "legend": {
                  "alignAsTable": false,
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "rightSide": false,
                  "show": true,
                  "sideWidth": null,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [
                  {
                     "alias": "/2../i",
                     "color": "#56A64B"
                  },
                  {
                     "alias": "/3../i",
                     "color": "#F2CC0C"
                  },
                  {
                     "alias": "/4../i",
                     "color": "#3274D9"
                  },
                  {
                     "alias": "/5../i",
                     "color": "#E02F44"
                  }
               ],
               "spaceLength": 10,
               "span": 3,
               "stack": true,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{ code }}",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Write SLI - Requests",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "reqps",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  },
                  {
                     "format": "reqps",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?",
               "fill": 1,
               "fillGradient": 0,
               "gridPos": { },
               "id": 11,
               "legend": {
                  "alignAsTable": false,
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "rightSide": false,
                  "show": true,
                  "sideWidth": null,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 3,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{ resource }}",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Write SLI - Errors",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "percentunit",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "percentunit",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?",
               "fill": 1,
               "fillGradient": 0,
               "gridPos": { },
               "id": 12,
               "legend": {
                  "alignAsTable": false,
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "rightSide": false,
                  "show": true,
                  "sideWidth": null,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 3,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{ resource }}",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Write SLI - Duration",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "s",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  },
                  {
                     "format": "s",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": false,
         "title": "Dashboard Row",
         "titleSize": "h6",
         "type": "row"
      },
      {
         "collapse": false,
         "collapsed": false,
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "fillGradient": 0,
               "gridPos": { },
               "id": 13,
               "legend": {
                  "alignAsTable": false,
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "rightSide": false,
                  "show": false,
                  "sideWidth": null,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 6,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{instance}} {{name}}",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Work Queue Add Rate",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "ops",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "ops",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "fillGradient": 0,
               "gridPos": { },
               "id": 14,
               "legend": {
                  "alignAsTable": false,
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "rightSide": false,
                  "show": false,
                  "sideWidth": null,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 6,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{instance}} {{name}}",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Work Queue Depth",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "fillGradient": 0,
               "gridPos": { },
               "id": 15,
               "legend": {
                  "alignAsTable": true,
                  "avg": false,
                  "current": true,
                  "max": false,
                  "min": false,
                  "rightSide": true,
                  "show": true,
                  "sideWidth": null,
                  "total": false,
                  "values": true
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 12,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{instance}} {{name}}",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Work Queue Latency",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "s",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  },
                  {
                     "format": "s",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": false,
         "title": "Dashboard Row",
         "titleSize": "h6",
         "type": "row"
      },
      {
         "collapse": false,
         "collapsed": false,
         "panels": [
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "fillGradient": 0,
               "gridPos": { },
               "id": 16,
               "legend": {
                  "alignAsTable": false,
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "rightSide": false,
                  "show": true,
                  "sideWidth": null,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{instance}}",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Memory",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "bytes",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  },
                  {
                     "format": "bytes",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "fillGradient": 0,
               "gridPos": { },
               "id": 17,
               "legend": {
                  "alignAsTable": false,
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "rightSide": false,
                  "show": true,
                  "sideWidth": null,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{instance}}",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "CPU usage",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": 0,
                     "show": true
                  }
               ]
            },
            {
               "aliasColors": { },
               "bars": false,
               "dashLength": 10,
               "dashes": false,
               "datasource": "$datasource",
               "fill": 1,
               "fillGradient": 0,
               "gridPos": { },
               "id": 18,
               "legend": {
                  "alignAsTable": false,
                  "avg": false,
                  "current": false,
                  "max": false,
                  "min": false,
                  "rightSide": false,
                  "show": true,
                  "sideWidth": null,
                  "total": false,
                  "values": false
               },
               "lines": true,
               "linewidth": 1,
               "links": [ ],
               "nullPointMode": "null",
               "percentage": false,
               "pointradius": 5,
               "points": false,
               "renderer": "flot",
               "repeat": null,
               "seriesOverrides": [ ],
               "spaceLength": 10,
               "span": 4,
               "stack": false,
               "steppedLine": false,
               "targets": [
                  {
                     "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}",
                     "format": "time_series",
                     "intervalFactor": 2,
                     "legendFormat": "{{instance}}",
                     "refId": "A"
                  }
               ],
               "thresholds": [ ],
               "timeFrom": null,
               "timeShift": null,
               "title": "Goroutines",
               "tooltip": {
                  "shared": false,
                  "sort": 0,
                  "value_type": "individual"
               },
               "type": "graph",
               "xaxis": {
                  "buckets": null,
                  "mode": "time",
                  "name": null,
                  "show": true,
                  "values": [ ]
               },
               "yaxes": [
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  },
                  {
                     "format": "short",
                     "label": null,
                     "logBase": 1,
                     "max": null,
                     "min": null,
                     "show": true
                  }
               ]
            }
         ],
         "repeat": null,
         "repeatIteration": null,
         "repeatRowId": null,
         "showTitle": false,
         "title": "Dashboard Row",
         "titleSize": "h6",
         "type": "row"
      }
   ],
   "schemaVersion": 14,
   "style": "dark",
   "tags": [
      "kubernetes"
   ],
   "templating": {
      "list": [
         {
            "current": {
               "text": "default",
               "value": "default"
            },
            "hide": 0,
            "label": null,
            "name": "datasource",
            "options": [ ],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "type": "datasource"
         },
         {
            "allValue": null,
            "current": { },
            "datasource": "$datasource",
            "hide": 2,
            "includeAll": false,
            "label": "cluster",
            "multi": false,
            "name": "cluster",
            "options": [ ],
            "query": "label_values(apiserver_request_total, cluster)",
            "refresh": 2,
            "regex": "",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [ ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
         },
         {
            "allValue": null,
            "current": { },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": true,
            "label": null,
            "multi": false,
            "name": "instance",
            "options": [ ],
            "query": "label_values(apiserver_request_total{job=\"apiserver\", cluster=\"$cluster\"}, instance)",
            "refresh": 2,
            "regex": "",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [ ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
         }
      ]
   },
   "time": {
      "from": "now-1h",
      "to": "now"
   },
   "timepicker": {
      "refresh_intervals": [
         "5s",
         "10s",
         "30s",
         "1m",
         "5m",
         "15m",
         "30m",
         "1h",
         "2h",
         "1d"
      ],
      "time_options": [
         "5m",
         "15m",
         "1h",
         "6h",
         "12h",
         "24h",
         "2d",
         "7d",
         "30d"
      ]
   },
   "timezone": "UTC",
   "title": "API server",
   "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b",
   "version": 0
}