diff --git a/charts/kubezero-ci/Chart.yaml b/charts/kubezero-ci/Chart.yaml index 04fac169..09a69854 100644 --- a/charts/kubezero-ci/Chart.yaml +++ b/charts/kubezero-ci/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-ci description: KubeZero umbrella chart for all things CI type: application -version: 0.4.25 +version: 0.4.26 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-ci/values.yaml b/charts/kubezero-ci/values.yaml index 67d9e986..4f44e6ab 100644 --- a/charts/kubezero-ci/values.yaml +++ b/charts/kubezero-ci/values.yaml @@ -17,7 +17,7 @@ gitea: enabled: false image: - tag: 1.15.10 + tag: 1.16.1 rootless: true securityContext: @@ -117,10 +117,10 @@ jenkins: - kubernetes:1.31.3 - workflow-aggregator:2.6 - git:4.10.3 - - configuration-as-code:1.55.1 + - configuration-as-code:1346.ve8cfa_3473c94 - antisamy-markup-formatter:2.7 - prometheus:2.0.10 - - htmlpublisher:1.28 + - htmlpublisher:1.29 - build-discarder:60.v1747b0eb632a serviceAccountAgent: diff --git a/charts/kubezero-storage/.gitignore b/charts/kubezero-storage/.gitignore new file mode 100644 index 00000000..02ace455 --- /dev/null +++ b/charts/kubezero-storage/.gitignore @@ -0,0 +1,2 @@ +jsonnet/vendor/ +jsonnet/tmp/ diff --git a/charts/kubezero-storage/.helmignore b/charts/kubezero-storage/.helmignore new file mode 100644 index 00000000..fc451861 --- /dev/null +++ b/charts/kubezero-storage/.helmignore @@ -0,0 +1 @@ +jsonnet diff --git a/charts/kubezero-storage/Chart.yaml b/charts/kubezero-storage/Chart.yaml index 3f9c4aa5..8ad1ba51 100644 --- a/charts/kubezero-storage/Chart.yaml +++ b/charts/kubezero-storage/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-storage description: KubeZero umbrella chart for all things storage incl. AWS EBS/EFS, openEBS-lvm, gemini type: application -version: 0.5.4 +version: 0.5.7 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -28,7 +28,7 @@ dependencies: condition: gemini.enabled # repository: https://charts.fairwinds.com/stable - name: aws-ebs-csi-driver - version: 2.6.2 + version: 2.6.3 condition: aws-ebs-csi-driver.enabled # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: aws-efs-csi-driver diff --git a/charts/kubezero-storage/README.md b/charts/kubezero-storage/README.md index bc5b5e27..f0290940 100644 --- a/charts/kubezero-storage/README.md +++ b/charts/kubezero-storage/README.md @@ -1,6 +1,6 @@ # kubezero-storage -![Version: 0.5.3](https://img.shields.io/badge/Version-0.5.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.5.7](https://img.shields.io/badge/Version-0.5.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero umbrella chart for all things storage incl. AWS EBS/EFS, openEBS-lvm, gemini @@ -18,7 +18,7 @@ Kubernetes: `>= 1.20.0` | Repository | Name | Version | |------------|------|---------| -| | aws-ebs-csi-driver | 2.6.2 | +| | aws-ebs-csi-driver | 2.6.3 | | | aws-efs-csi-driver | 2.2.3 | | | gemini | 0.0.8 | | | lvm-localpv | 0.8.6 | @@ -73,7 +73,6 @@ Kubernetes: `>= 1.20.0` | aws-efs-csi-driver.node.tolerations[0].key | string | `"kubezero-workergroup"` | | | aws-efs-csi-driver.node.tolerations[0].operator | string | `"Exists"` | | | aws-efs-csi-driver.replicaCount | int | `1` | | -| aws-efs-csi-driver.storageClasses[0].name | string | `"efs-sc"` | | | gemini.enabled | bool | `false` | | | gemini.resources.limits.cpu | string | `"400m"` | | | gemini.resources.limits.memory | string | `"128Mi"` | | @@ -90,6 +89,7 @@ Kubernetes: `>= 1.20.0` | lvm-localpv.lvmNode.tolerations[0].effect | string | `"NoSchedule"` | | | lvm-localpv.lvmNode.tolerations[0].key | string | `"kubezero-workergroup"` | | | lvm-localpv.lvmNode.tolerations[0].operator | string | `"Exists"` | | +| lvm-localpv.prometheus.enabled | bool | `false` | | | lvm-localpv.storageClass.default | bool | `false` | | | lvm-localpv.storageClass.vgpattern | string | `""` | | | snapshotController.enabled | bool | `true` | | @@ -107,3 +107,4 @@ Kubernetes: `>= 1.20.0` - https://kubernetes-csi.github.io/docs/snapshot-controller.html#deployment ## Resources +- https://github.com/openebs/monitoring/blob/develop/docs/openebs-mixin-user-guide.md#install-openebs-mixin-in-existing-prometheus-stack diff --git a/charts/kubezero-storage/README.md.gotmpl b/charts/kubezero-storage/README.md.gotmpl index f62a1f79..1b593cbd 100644 --- a/charts/kubezero-storage/README.md.gotmpl +++ b/charts/kubezero-storage/README.md.gotmpl @@ -19,3 +19,4 @@ - https://kubernetes-csi.github.io/docs/snapshot-controller.html#deployment ## Resources +- https://github.com/openebs/monitoring/blob/develop/docs/openebs-mixin-user-guide.md#install-openebs-mixin-in-existing-prometheus-stack diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md index 6127fb83..1d510925 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md @@ -1,5 +1,9 @@ # Helm chart +## v2.6.3 + +* Bump app/driver to version `v1.5.1` + ## v2.6.2 * Update csi-resizer version to v1.1.0 diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml index 9bdd84b2..c55b3977 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml @@ -3,7 +3,7 @@ annotations: - kind: added description: Custom controller.updateStrategy to set controller deployment strategy. apiVersion: v2 -appVersion: 1.5.0 +appVersion: 1.5.1 description: A Helm chart for AWS EBS CSI Driver home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver keywords: @@ -19,4 +19,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 2.6.2 +version: 2.6.3 diff --git a/charts/kubezero-storage/charts/gemini/Chart.yaml b/charts/kubezero-storage/charts/gemini/Chart.yaml index 535b27c0..d5fce1fa 100644 --- a/charts/kubezero-storage/charts/gemini/Chart.yaml +++ b/charts/kubezero-storage/charts/gemini/Chart.yaml @@ -1,9 +1,9 @@ apiVersion: v1 -appVersion: 0.1.0 +appVersion: "1.0" description: Automated backup and restore of PersistentVolumes using the VolumeSnapshot API maintainers: - email: robertb@fairwinds.com name: rbren name: gemini -version: 0.0.8 +version: 1.0.0 diff --git a/charts/kubezero-storage/charts/gemini/README.md b/charts/kubezero-storage/charts/gemini/README.md index f8067076..f262fcfb 100644 --- a/charts/kubezero-storage/charts/gemini/README.md +++ b/charts/kubezero-storage/charts/gemini/README.md @@ -28,7 +28,7 @@ Your cluster must support the [VolumeSnapshot API](https://kubernetes.io/docs/co |-----|------|---------|-------------| | image.pullPolicy | string | `"Always"` | imagePullPolicy - Highly recommended to leave this as `Always` | | image.repository | string | `"quay.io/fairwinds/gemini"` | Repository for the gemini image | -| image.tag | string | `"0.1"` | The gemini image tag to use | +| image.tag | string | `nil` | The gemini image tag to use. Defaults to .Chart.appVersion | | rbac.create | bool | `true` | If true, create a new ServiceAccount and attach permissions | | rbac.serviceAccountName | string | `nil` | | | verbosity | int | `5` | How verbose the controller logs should be | diff --git a/charts/kubezero-storage/charts/gemini/templates/deployment.yaml b/charts/kubezero-storage/charts/gemini/templates/deployment.yaml index 52beafe8..d435c6d4 100644 --- a/charts/kubezero-storage/charts/gemini/templates/deployment.yaml +++ b/charts/kubezero-storage/charts/gemini/templates/deployment.yaml @@ -31,7 +31,7 @@ spec: - -v - {{ . | quote }} {{- end }} - image: '{{.Values.image.repository}}:{{.Values.image.tag}}' + image: "{{.Values.image.repository}}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: '{{.Values.image.pullPolicy}}' name: gemini-controller resources: diff --git a/charts/kubezero-storage/charts/gemini/values.yaml b/charts/kubezero-storage/charts/gemini/values.yaml index 07ec88a5..32098ad6 100644 --- a/charts/kubezero-storage/charts/gemini/values.yaml +++ b/charts/kubezero-storage/charts/gemini/values.yaml @@ -3,8 +3,8 @@ image: pullPolicy: Always # image.repository -- Repository for the gemini image repository: quay.io/fairwinds/gemini - # image.tag -- The gemini image tag to use - tag: "0.1" + # image.tag -- The gemini image tag to use. Defaults to .Chart.appVersion + tag: rbac: # rbac.create -- If true, create a new ServiceAccount and attach permissions diff --git a/charts/kubezero-storage/jsonnet/Makefile b/charts/kubezero-storage/jsonnet/Makefile new file mode 100644 index 00000000..eb4d2df0 --- /dev/null +++ b/charts/kubezero-storage/jsonnet/Makefile @@ -0,0 +1,35 @@ +JSONNET_ARGS := -n 2 --max-blank-lines 2 --string-style s --comment-style s +ifneq (,$(shell which jsonnetfmt)) + JSONNET_FMT_CMD := jsonnetfmt +else + JSONNET_FMT_CMD := jsonnet + JSONNET_FMT_ARGS := fmt $(JSONNET_ARGS) +endif +JSONNET_FMT := $(JSONNET_FMT_CMD) $(JSONNET_FMT_ARGS) + +dashboardsDirPath=dashboards +rulessDirPath=rules + +all: fmt generate lint + +fmt: + find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \ + xargs -n 1 -- $(JSONNET_FMT) -i + +generate: clean + jsonnet -J vendor -m rules -e '(import "prometheusRules.libsonnet")' + jsonnet -J vendor -m dashboards -e '(import "dashboards.libsonnet")' + +render: generate + ../../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml ../templates/lvm/grafana-dashboards.yaml + ../../kubezero-metrics/sync_prometheus_rules.py rules.yaml ../templates/lvm/ + +lint: + find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \ + while read f; do \ + $(JSONNET_FMT) "$$f" | diff -u "$$f" -; \ + done + +clean: + rm -rf $(dashboardsDirPath) $(rulessDirPath) + mkdir -p $(dashboardsDirPath) $(rulessDirPath) diff --git a/charts/kubezero-storage/jsonnet/config.libsonnet b/charts/kubezero-storage/jsonnet/config.libsonnet new file mode 100644 index 00000000..20e03ce3 --- /dev/null +++ b/charts/kubezero-storage/jsonnet/config.libsonnet @@ -0,0 +1,24 @@ +{ + _config+:: { + dashboards+: { + cStor: false, + jiva: false, + localPV: false, + lvmLocalPV: true, + deviceLocalPV: false, + zfsLocalPV: false, + ndm: false, + npd: false, + }, + alertRules+: { + cStor: false, + jiva: false, + localPV: false, + lvmLocalPV: true, + deviceLocalPV: false, + zfsLocalPV: false, + ndm: false, + npd: false, + }, + }, +} diff --git a/charts/kubezero-storage/jsonnet/dashboards.libsonnet b/charts/kubezero-storage/jsonnet/dashboards.libsonnet new file mode 100644 index 00000000..13a8b292 --- /dev/null +++ b/charts/kubezero-storage/jsonnet/dashboards.libsonnet @@ -0,0 +1,10 @@ +local addMixin = (import 'kube-prometheus/lib/mixin.libsonnet'); + +local openebsMixin = addMixin({ + name: 'openebs', + mixin: (import 'github.com/openebs/monitoring/jsonnet/openebs-mixin/mixin.libsonnet') + + (import 'config.libsonnet'), +}); + + +{ ['openebs-' + name]: openebsMixin.grafanaDashboards[name] for name in std.objectFields(openebsMixin.grafanaDashboards) } diff --git a/charts/kubezero-storage/jsonnet/dashboards.yaml b/charts/kubezero-storage/jsonnet/dashboards.yaml new file mode 100644 index 00000000..e4afcb55 --- /dev/null +++ b/charts/kubezero-storage/jsonnet/dashboards.yaml @@ -0,0 +1,10 @@ +configmap: openebs-lvmlocalpv-grafana-dashboard +gzip: true +folder: KubeZero +condition: 'index .Values "lvm-localpv" "prometheus" "enabled"' +dashboards: +- name: openebs-localpv-pool + url: file://dashboards/openebs-lvmlocalpv-pool.json + tags: + - storage + - openEBS diff --git a/charts/kubezero-storage/jsonnet/dashboards/openebs-lvmlocalpv-pool.json b/charts/kubezero-storage/jsonnet/dashboards/openebs-lvmlocalpv-pool.json new file mode 100644 index 00000000..1e12929e --- /dev/null +++ b/charts/kubezero-storage/jsonnet/dashboards/openebs-lvmlocalpv-pool.json @@ -0,0 +1,3087 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 30, + "iteration": 1624996506300, + "links": [ ], + "panels": [ + { + "collapsed": false, + "datasource": "$datasource", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 22, + "panels": [ ], + "title": "Volume group Stats", + "type": "row" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [ ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total size" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "continuous-greens" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free size" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "continuous-RdYlGr" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used size" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { } + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_vg_total_size_bytes{name=~\"$volume_group\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "Total size", + "refId": "A" + }, + { + "exemplar": true, + "expr": "lvm_vg_free_size_bytes{name=~\"$volume_group\", instance=~\"$node.*\"}", + "hide": false, + "interval": "", + "legendFormat": "Free size", + "refId": "B" + }, + { + "exemplar": true, + "expr": "lvm_vg_total_size_bytes{name=~\"$volume_group\", instance=~\"$node.*\"}-lvm_vg_free_size_bytes{name=~\"$volume_group\", instance=~\"$node.*\"}", + "hide": false, + "interval": "", + "legendFormat": "Used size", + "refId": "C" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Group Capacity", + "type": "bargauge" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [ ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Metadata total size" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "continuous-greens" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Metadata free size" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "continuous-RdYlGr" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Metadata used size" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { } + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_vg_mda_total_size_bytes{name=~\"$volume_group\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "Metadata total size", + "refId": "A" + }, + { + "exemplar": true, + "expr": "lvm_vg_mda_free_size_bytes{name=~\"$volume_group\", instance=~\"$node.*\"}", + "hide": false, + "interval": "", + "legendFormat": "Metadata free size", + "refId": "B" + }, + { + "exemplar": true, + "expr": "lvm_vg_mda_total_size_bytes{name=~\"$volume_group\", instance=~\"$node.*\"}-lvm_vg_mda_free_size_bytes{name=~\"$volume_group\", instance=~\"$node.*\"}", + "hide": false, + "interval": "", + "legendFormat": "Metadata used size", + "refId": "C" + } + ], + "title": "Volume Group Metadata Capacity", + "type": "bargauge" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [ + { + "from": "", + "id": 1, + "text": "undefined", + "to": "", + "type": 1, + "value": "-1" + }, + { + "from": "", + "id": 2, + "text": "writable", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 3, + "text": "read-only", + "to": "", + "type": 1, + "value": "1" + } + ], + "max": 1, + "min": -1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 6, + "options": { + "graph": { }, + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_vg_permission{name=~\"$volume_group\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "Permission", + "refId": "A" + } + ], + "title": "Volume Group Permission", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [ + { + "from": "", + "id": 1, + "text": "undefined", + "to": "", + "type": 1, + "value": "-1" + }, + { + "from": "", + "id": 2, + "text": "normal", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 3, + "text": "contiguous", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 4, + "text": "cling", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 5, + "text": "anywhere", + "to": "", + "type": 1, + "value": "3" + }, + { + "from": "", + "id": 6, + "text": "inherited", + "to": "", + "type": 1, + "value": "4" + } + ], + "max": 4, + "min": -1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 8, + "options": { + "graph": { }, + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_vg_allocation_policy{name=~\"$volume_group\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "allocation policy", + "refId": "A" + } + ], + "title": "Volume Group Allocation Policy", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [ ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max lv count" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "unlimited", + "to": "", + "type": 1, + "value": "0" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 12, + "options": { + "graph": { }, + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_vg_lv_count{name=~\"$volume_group\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "current lv count", + "refId": "A" + }, + { + "exemplar": true, + "expr": "lvm_vg_max_lv_count{name=~\"$volume_group\", instance=~\"$node.*\"}", + "hide": false, + "interval": "", + "legendFormat": "max lv count", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Group volumes count", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [ ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "current pv count" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "graph": true, + "legend": false, + "tooltip": false + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max pv count" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "unlimited", + "to": "", + "type": 1, + "value": "0" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 16, + "options": { + "graph": { }, + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_vg_pv_count{name=~\"$volume_group\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "current pv count", + "refId": "A" + }, + { + "exemplar": true, + "expr": "lvm_vg_missing_pv_count{name=~\"$volume_group\", instance=~\"$node.*\"}", + "hide": false, + "interval": "", + "legendFormat": "missing pv count", + "refId": "B" + }, + { + "exemplar": true, + "expr": "lvm_vg_max_pv_count{name=~\"$volume_group\", instance=~\"$node.*\"}", + "hide": false, + "interval": "", + "legendFormat": "max pv count", + "refId": "C" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Group pv count", + "type": "timeseries" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "lvm_vg_mda_count{name=~\"$volume_group\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "current metadata count", + "refId": "A" + }, + { + "exemplar": true, + "expr": "lvm_vg_mda_used_count{name=~\"$volume_group\", instance=~\"$node.*\"}", + "hide": false, + "interval": "", + "legendFormat": "used metadata count", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Volume Group metadata count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "$$hashKey": "object:1180", + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1181", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "lvm_vg_snap_count{name=~\"$volume_group\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "snapshots count", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Volume Group snapshots count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "$$hashKey": "object:1431", + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1432", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "filterable": false + }, + "mappings": [ ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "used %" + }, + { + "id": "custom.displayMode", + "value": "lcd-gauge" + }, + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "active_status" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "color-background" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Total size" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "segtype" + }, + "properties": [ + { + "id": "displayName", + "value": "Type" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 34, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "" + } + ] + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_lv_used_percent{vg=~\"$volume_group\",segtype!=\"$thinPool\", instance=~\"$node.*\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "exemplar": true, + "expr": "lvm_lv_total_size_bytes{vg=~\"$volume_group\",segtype!=\"$thinPool\", instance=~\"$node.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Volume Group Volumes", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "active_status", + "device", + "dm_path", + "host", + "name", + "path", + "segtype", + "Value #A", + "Value #B" + ] + } + } + }, + { + "id": "merge", + "options": { } + }, + { + "id": "organize", + "options": { + "excludeByName": { }, + "indexByName": { + "Value #A": 8, + "Value #B": 7, + "active_status": 1, + "device": 2, + "dm_path": 3, + "host": 5, + "name": 0, + "path": 4, + "segtype": 6 + }, + "renameByName": { } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "datasource": "$datasource", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 52, + "panels": [ ], + "title": "Volume Group Performance Stats", + "type": "row" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(node_disk_reads_completed_total{instance=~\"$node.*\"}[5m]) + on(device) group_left(name) (lvm_lv_when_full{vg=~\"$volume_group\", instance=~\"$node.*\"} * 0))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "reads/sec", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Volume Group IO Reads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "$$hashKey": "object:310", + "format": "iops", + "label": "IO read", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:311", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "hiddenSeries": false, + "id": 58, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(node_disk_writes_completed_total{instance=~\"$node.*\"}[5m]) + on(device) group_left(name) (lvm_lv_when_full{vg=~\"$volume_group\", instance=~\"$node.*\"} * 0))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "writes/sec", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Volume Group IO Writes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "$$hashKey": "object:557", + "format": "iops", + "label": "IO write", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:558", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "hiddenSeries": false, + "id": 54, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$node.*\"}[5m]) + on(device) group_left(name) (lvm_lv_when_full{vg=~\"$volume_group\", instance=~\"$node.*\"} * 0))", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "Read bytes", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$node.*\"}[5m]) + on(device) group_left(name) (lvm_lv_when_full{vg=~\"$volume_group\", instance=~\"$node.*\"} * 0))", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "Written bytes", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Volume Group R/W Data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "$$hashKey": "object:76", + "format": "Bps", + "label": "bytes read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:77", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "nullPointMode": "connected", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(node_disk_io_time_seconds_total{instance=~\"$node.*\"}[5m]) + on(device) group_left(name) (lvm_lv_when_full{vg=~\"$volume_group\", instance=~\"$node.*\"} * 0))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "utilization", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Volume Group I/O Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "$$hashKey": "object:804", + "format": "percentunit", + "label": "%util", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:805", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "$datasource", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 24, + "panels": [ ], + "title": "Thin Pool Stats", + "type": "row" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "decimals": 0, + "mappings": [ + { + "from": "", + "id": 1, + "text": "undefined", + "to": "", + "type": 1, + "value": "-1" + }, + { + "from": "", + "id": 2, + "text": "healthy", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 3, + "text": "partial", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 4, + "text": "refresh needed", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 5, + "text": "mismatches exist", + "to": "", + "type": 1, + "value": "3" + } + ], + "max": 3, + "min": -1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 26, + "options": { + "graph": { }, + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_lv_health_status{name=~\"$thin_pool\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "Health Status", + "refId": "A" + } + ], + "title": "Health Status", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [ + { + "from": "", + "id": 1, + "text": "undefined", + "to": "", + "type": 1, + "value": "-1" + }, + { + "from": "", + "id": 2, + "text": "error", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 3, + "text": "queue", + "to": "", + "type": 1, + "value": "1" + } + ], + "max": 1, + "min": -1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 50, + "options": { + "graph": { }, + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_lv_when_full{name=~\"$thin_pool\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "behaviour", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Behaviour when full", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Used %" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total size" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "continuous-greens" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 65 + }, + "id": 30, + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { } + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_lv_total_size_bytes{name=~\"$thin_pool\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "Total size", + "refId": "A" + }, + { + "exemplar": true, + "expr": "lvm_lv_used_percent{name=~\"$thin_pool\", instance=~\"$node.*\"}", + "hide": false, + "interval": "", + "legendFormat": "Used %", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Pool Capacity", + "type": "bargauge" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Metadata size" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "continuous-greens" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Metadata used %" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 65 + }, + "id": 38, + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { } + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_lv_mda_total_size_bytes{name=~\"$thin_pool\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "Metadata size", + "refId": "A" + }, + { + "exemplar": true, + "expr": "lvm_lv_mda_used_percent{name=~\"$thin_pool\", instance=~\"$node.*\"}", + "hide": false, + "interval": "", + "legendFormat": "Metadata used %", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Pool Metadata Capacity", + "type": "bargauge" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 74 + }, + "hiddenSeries": false, + "id": 40, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "lvm_lv_snap_percent{name=~\"$thin_pool\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "snapshot full %", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Pool Snapshot Full %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "$$hashKey": "object:4099", + "decimals": 2, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:4100", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [ + { + "from": "", + "id": 1, + "text": "undefined", + "to": "", + "type": 1, + "value": "-1" + }, + { + "from": "", + "id": 2, + "text": "unknown", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 3, + "text": "writeable", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 4, + "text": "read-only", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 5, + "text": "read-only-override", + "to": "", + "type": 1, + "value": "3" + } + ], + "max": 3, + "min": -1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 48, + "options": { + "graph": { }, + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_lv_permission{name=~\"$thin_pool\", instance=~\"$node.*\"}", + "interval": "", + "legendFormat": "permission", + "refId": "A" + } + ], + "title": "Pool Permission", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "filterable": false + }, + "mappings": [ ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "displayName", + "value": "Used %" + }, + { + "id": "custom.displayMode", + "value": "lcd-gauge" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Total Size" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "active_status" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "color-background" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 82 + }, + "id": 42, + "options": { + "showHeader": true + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_lv_used_percent{pool=~\"$thin_pool\",segtype=~\"$thinVolume\", instance=~\"$node.*\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "exemplar": true, + "expr": "lvm_lv_total_size_bytes{pool=~\"$thin_pool\",segtype=~\"$thinVolume\", instance=~\"$node.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B" + } + ], + "title": "Thin Volumes", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "active_status", + "device", + "dm_path", + "host", + "Value #A", + "Value #B", + "path", + "name" + ] + } + } + }, + { + "id": "merge", + "options": { } + }, + { + "id": "organize", + "options": { + "excludeByName": { }, + "indexByName": { + "Value #A": 7, + "Value #B": 6, + "active_status": 1, + "device": 2, + "dm_path": 3, + "host": 5, + "name": 0, + "path": 4 + }, + "renameByName": { } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "datasource": "$datasource", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 87 + }, + "id": 62, + "panels": [ ], + "title": "Thin Pool Performance Stats", + "type": "row" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 88 + }, + "hiddenSeries": false, + "id": 64, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(node_disk_reads_completed_total{instance=~\"$node.*\"}[5m]) + on(device) group_left(name) (lvm_lv_when_full{vg=~\"$volume_group\",segtype=~\"$thinVolume\", instance=~\"$node.*\"} * 0))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "reads/sec", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Thin Pool IO Reads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "$$hashKey": "object:1004", + "format": "iops", + "label": "IO read", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1005", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 88 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(node_disk_writes_completed_total{instance=~\"$node.*\"}[5m]) + on(device) group_left(name) (lvm_lv_when_full{vg=~\"$volume_group\",segtype=~\"$thinVolume\", instance=~\"$node.*\"} * 0))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "writes/sec", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Thin Pool IO Writes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "$$hashKey": "object:1745", + "format": "iops", + "label": "IO write", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1746", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 96 + }, + "hiddenSeries": false, + "id": 68, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$node.*\"}[5m]) + on(device) group_left(name) (lvm_lv_when_full{vg=~\"$volume_group\",segtype=~\"$thinVolume\", instance=~\"$node.*\"} * 0))", + "interval": "", + "intervalFactor": 4, + "legendFormat": "Read bytes", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$node.*\"}[5m]) + on(device) group_left(name) (lvm_lv_when_full{vg=~\"$volume_group\",segtype=~\"$thinVolume\", instance=~\"$node.*\"} * 0))", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "Written bytes", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Thin Pool R/W Data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "$$hashKey": "object:1834", + "format": "Bps", + "label": "bytes read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1835", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 96 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(node_disk_io_time_seconds_total{instance=~\"$node.*\"}[5m]) + on(device) group_left(name) (lvm_lv_when_full{vg=~\"$volume_group\",segtype=~\"$thinVolume\", instance=~\"$node.*\"} * 0))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "utililization", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeRegions": [ ], + "timeShift": null, + "title": "Thin Pool Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "$$hashKey": "object:1923", + "format": "percentunit", + "label": "%util", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1924", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "$datasource", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 104 + }, + "id": 44, + "panels": [ ], + "title": "Volume Group PV Stats", + "type": "row" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "filterable": false + }, + "mappings": [ ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Total Size" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Used Size" + }, + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "custom.displayMode", + "value": "lcd-gauge" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "displayName", + "value": "Free Size" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "displayName", + "value": "Device Size" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "in_use" + }, + "properties": [ + { + "id": "displayName", + "value": "In Use" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 105 + }, + "id": 46, + "options": { + "showHeader": true + }, + "pluginVersion": "7.5.5", + "targets": [ + { + "exemplar": true, + "expr": "lvm_pv_total_size_bytes{vg=~\"$volume_group\", instance=~\"$node.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "exemplar": true, + "expr": "lvm_pv_used_size_bytes{vg=~\"$volume_group\", instance=~\"$node.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "exemplar": true, + "expr": "lvm_pv_free_size_bytes{vg=~\"$volume_group\", instance=~\"$node.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "exemplar": true, + "expr": "lvm_pv_device_size_bytes{vg=~\"$volume_group\", instance=~\"$node.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + } + ], + "title": "Volume Group PV", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "allocatable", + "in_use", + "name", + "Value #B", + "Value #A", + "Value #C", + "Value #D" + ] + } + } + }, + { + "id": "merge", + "options": { } + }, + { + "id": "organize", + "options": { + "excludeByName": { }, + "indexByName": { + "Value #A": 4, + "Value #B": 5, + "Value #C": 6, + "Value #D": 3, + "allocatable": 1, + "in_use": 2, + "name": 0 + }, + "renameByName": { } + } + } + ], + "type": "table" + } + ], + "refresh": "5m", + "schemaVersion": 27, + "style": "dark", + "tags": [ + "OpenEBS", + "LVM" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "datasource", + "options": [ ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "definition": "label_values(node_uname_info, instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "node", + "multi": false, + "name": "node", + "options": [ ], + "query": "label_values(node_uname_info, instance)", + "refresh": 1, + "regex": "(.*):.*", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "definition": "label_values(lvm_vg_total_size_bytes{instance=~\"$node.*\"}, name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "volume_group", + "multi": false, + "name": "volume_group", + "options": [ ], + "query": "label_values(lvm_vg_total_size_bytes{instance=~\"$node.*\"}, name)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "description": null, + "error": null, + "hide": 2, + "label": "thinPool", + "name": "thinPool", + "query": "thin-pool", + "skipUrlSync": false, + "type": "constant" + }, + { + "description": null, + "error": null, + "hide": 2, + "label": "thinVolume", + "name": "thinVolume", + "query": "thin", + "skipUrlSync": false, + "type": "constant" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "definition": "label_values(lvm_lv_total_size_bytes{vg=~\"$volume_group\", segtype=~\"$thinPool\", instance=~\"$node.*\"}, name) ", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "thin_pool", + "multi": false, + "name": "thin_pool", + "options": [ ], + "query": "label_values(lvm_lv_total_size_bytes{vg=~\"$volume_group\", segtype=~\"$thinPool\", instance=~\"$node.*\"}, name) ", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { }, + "timezone": "", + "title": "LVM Pool", + "uid": "F0b6Kmknk", + "version": 8 +} diff --git a/charts/kubezero-storage/jsonnet/jsonnetfile.json b/charts/kubezero-storage/jsonnet/jsonnetfile.json new file mode 100644 index 00000000..724a52a9 --- /dev/null +++ b/charts/kubezero-storage/jsonnet/jsonnetfile.json @@ -0,0 +1,24 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/prometheus-operator/kube-prometheus.git", + "subdir": "jsonnet/kube-prometheus" + } + }, + "version": "release-0.9" + }, + { + "source": { + "git": { + "remote": "https://github.com/openebs/monitoring.git", + "subdir": "" + } + }, + "version": "develop" + } + ], + "legacyImports": true +} diff --git a/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json b/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json new file mode 100644 index 00000000..4b07926b --- /dev/null +++ b/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json @@ -0,0 +1,170 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/brancz/kubernetes-grafana.git", + "subdir": "grafana" + } + }, + "version": "90f38916f1f8a310a715d18e36f787f84df4ddf5", + "sum": "0kZ1pnuIirDtbg6F9at5+NQOwKNONIGEPq0eECzvRkI=" + }, + { + "source": { + "git": { + "remote": "https://github.com/etcd-io/etcd.git", + "subdir": "contrib/mixin" + } + }, + "version": "fa191c64bd3c3eaa6cc167e837d7a73e3b7d7c72", + "sum": "W/Azptf1PoqjyMwJON96UY69MFugDA4IAYiKURscryc=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet-lib.git", + "subdir": "grafonnet" + } + }, + "version": "6db00c292d3a1c71661fc875f90e0ec7caa538c2", + "sum": "gF8foHByYcB25jcUOBqP6jxk0OPifQMjPvKY0HaCk6w=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "grafana-builder" + } + }, + "version": "03d32a72a2a0bf0ee00ffc853be5f07ad3bafcbe", + "sum": "0KkygBQd/AFzUvVzezE4qF/uDYgrwUXVpZfINBti0oc=" + }, + { + "source": { + "git": { + "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin.git", + "subdir": "" + } + }, + "version": "fb9d8ed4bc4a3d6efac525f72e8a0d2c583a0fe2", + "sum": "xjKkdp+5fkekCNBUIgZCHTRmVdUEmQNFKslrL2Ho8gs=" + }, + { + "source": { + "git": { + "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin.git", + "subdir": "lib/promgrafonnet" + } + }, + "version": "177bc8ec789fa049a9585713d232035b159f8c92", + "sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps=" + }, + { + "source": { + "git": { + "remote": "https://github.com/kubernetes/kube-state-metrics.git", + "subdir": "jsonnet/kube-state-metrics" + } + }, + "version": "f170cc73f11c1580d7f38af746be0f2fa79c6a1e", + "sum": "S5qI+PJUdNeYOv76jH5nxwYS9N6U7CRxvyuB1wI4cTE=" + }, + { + "source": { + "git": { + "remote": "https://github.com/kubernetes/kube-state-metrics.git", + "subdir": "jsonnet/kube-state-metrics-mixin" + } + }, + "version": "f170cc73f11c1580d7f38af746be0f2fa79c6a1e", + "sum": "u8gaydJoxEjzizQ8jY8xSjYgWooPmxw+wIWdDxifMAk=" + }, + { + "source": { + "git": { + "remote": "https://github.com/openebs/monitoring.git", + "subdir": "" + } + }, + "version": "87b902c85429eedce78033049de9280f39e81372", + "sum": "TyL/PPFwpRRzAOwYVlvgBsSHS6/+Z6TgRGFe8beJcpE=" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus-operator/kube-prometheus.git", + "subdir": "jsonnet/kube-prometheus" + } + }, + "version": "c8db3c85db6e13b94726b7c5d630a6ebe7038cf9", + "sum": "7DfVEg2kDzIQPgxTThnjQ4S2ZksrF0hkNLysXP6Wv24=" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus-operator/prometheus-operator.git", + "subdir": "jsonnet/mixin" + } + }, + "version": "83fe36566f4e0894eb5ffcd2638a0f039a17bdeb", + "sum": "6reUygVmQrLEWQzTKcH8ceDbvM+2ztK3z2VBR2K2l+U=", + "name": "prometheus-operator-mixin" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus-operator/prometheus-operator.git", + "subdir": "jsonnet/prometheus-operator" + } + }, + "version": "83fe36566f4e0894eb5ffcd2638a0f039a17bdeb", + "sum": "J1G++A8hrtr3+OZQMmcNeb1w/C30bXqqwpwHL/Xhsd4=" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus/alertmanager.git", + "subdir": "doc/alertmanager-mixin" + } + }, + "version": "b408b522bc653d014e53035e59fa394cc1edd762", + "sum": "pep+dHzfIjh2SU5pEkwilMCAT/NoL6YYflV4x8cr7vU=", + "name": "alertmanager" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus/node_exporter.git", + "subdir": "docs/node-mixin" + } + }, + "version": "832909dd257eb368cf83363ffcae3ab84cb4bcb1", + "sum": "MmxGhE2PJ1a52mk2x7vDpMT2at4Jglbud/rK74CB5i0=" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus/prometheus.git", + "subdir": "documentation/prometheus-mixin" + } + }, + "version": "751ca03faddc9c64089c41d0da370a3a0b477742", + "sum": "AS8WYFi/z10BZSF6DFkKBscjB32XDMM7iIso7CO/FyI=", + "name": "prometheus" + }, + { + "source": { + "git": { + "remote": "https://github.com/thanos-io/thanos.git", + "subdir": "mixin" + } + }, + "version": "ff363498fc95cfe17de894d7237bcf38bdd0bc36", + "sum": "cajthvLKDjYgYHCKQU2g/pTMRkxcbuJEvTnCyJOihl8=", + "name": "thanos-mixin" + } + ], + "legacyImports": false +} diff --git a/charts/kubezero-storage/jsonnet/prometheusRules.libsonnet b/charts/kubezero-storage/jsonnet/prometheusRules.libsonnet new file mode 100644 index 00000000..7552203b --- /dev/null +++ b/charts/kubezero-storage/jsonnet/prometheusRules.libsonnet @@ -0,0 +1,9 @@ +local addMixin = (import 'kube-prometheus/lib/mixin.libsonnet'); + +local openebsMixin = addMixin({ + name: 'openebs', + mixin: (import 'github.com/openebs/monitoring/jsonnet/openebs-mixin/mixin.libsonnet') + + (import 'config.libsonnet'), +}); + +{ 'openebs-mixin-prometheusRules': openebsMixin.prometheusRules } diff --git a/charts/kubezero-storage/jsonnet/rules.yaml b/charts/kubezero-storage/jsonnet/rules.yaml new file mode 100644 index 00000000..5816638a --- /dev/null +++ b/charts/kubezero-storage/jsonnet/rules.yaml @@ -0,0 +1,4 @@ +rules: +- name: prometheus-rules + url: file://rules/openebs-mixin-prometheusRules + condition: 'index .Values "lvm-localpv" "prometheus" "enabled"' diff --git a/charts/kubezero-storage/jsonnet/rules/openebs-mixin-prometheusRules b/charts/kubezero-storage/jsonnet/rules/openebs-mixin-prometheusRules new file mode 100644 index 00000000..febe0936 --- /dev/null +++ b/charts/kubezero-storage/jsonnet/rules/openebs-mixin-prometheusRules @@ -0,0 +1,156 @@ +{ + "apiVersion": "monitoring.coreos.com/v1", + "kind": "PrometheusRule", + "metadata": { + "labels": { + "prometheus": "k8s" + }, + "name": "openebs", + "namespace": "monitoring" + }, + "spec": { + "groups": [ + { + "name": "lvm-pool", + "rules": [ + { + "alert": "LVMVolumeGroupMissingPhysicalVolume", + "annotations": { + "componentType": "volume group", + "description": "LVM volume group '{{ $labels.name }}' on node '{{ $labels.instance }}' is missing {{ $value }} underlying physical volume(s).", + "summary": "LVM volume group '{{ $labels.name }}' is missing the underlying physical volume.", + "vgName": "{{ $labels.name }}" + }, + "expr": "lvm_vg_missing_pv_count > 0", + "for": "5m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "LVMVolumeGroupCapacityLow", + "annotations": { + "componentType": "volume group", + "description": "LVM volume group '{{ $labels.name }}' on node '{{ $labels.instance }}' has {{ with printf \"lvm_vg_free_size_bytes{instance='%s',name='%s'}\" $labels.instance $labels.name | query }} {{ . | first | value }} {{ end }}bytes of space remaining", + "summary": "LVM volume group '{{ $labels.name }}' is running low on capacity. Already {{ $value }}% of total capacity is consumed.", + "vgName": "{{ $labels.name }}" + }, + "expr": "((lvm_vg_total_size_bytes - lvm_vg_free_size_bytes)/lvm_vg_total_size_bytes)*100 > 90", + "for": "5m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "LVMThinPoolCapacityLow", + "annotations": { + "componentType": "logical volume", + "description": "LVM thin pool '{{ $labels.name }}' on node '{{ $labels.instance }}' has {{ with printf \"lvm_lv_total_size_bytes{instance='%s',name='%s',segtype='%s'}-((lvm_lv_used_percent{instance='%s',name='%s',segtype='%s'}*lvm_lv_total_size_bytes{instance='%s',name='%s',segtype='%s'})/100)\" $labels.instance $labels.name $labels.segtype $labels.instance $labels.name $labels.segtype $labels.instance $labels.name $labels.segtype | query }} {{ . | first | value }} {{ end }}bytes of space remaining", + "lvName": "{{ $labels.name }}", + "summary": "LVM thin pool '{{ $labels.name }}' is running low on capacity. Already {{ $value }}% of total capacity is consumed." + }, + "expr": "lvm_lv_used_percent{segtype=\"thin-pool\"} > 90", + "for": "5m", + "labels": { + "severity": "critical" + } + } + ] + }, + { + "name": "persistent-volume-claim", + "rules": [ + { + "alert": "StalePersistentVolumeClaim", + "annotations": { + "description": "Persistent Volume Claim '{{ $labels.persistentvolumeclaim }}' has no consumer", + "summary": "Persistent Volume Claim '{{ $labels.persistentvolumeclaim }}' in namespace '{{ $labels.namespace }}' is not consumed by any pod in any namespace" + }, + "expr": "kube_persistentvolumeclaim_info unless (kube_persistentvolumeclaim_info * on(persistentvolumeclaim) group_left kube_pod_spec_volumes_persistentvolumeclaims_info) == 1", + "for": "5m", + "labels": { + "severity": "info" + } + }, + { + "alert": "PendingPersistentVolumeClaim", + "annotations": { + "description": "Persistent Volume Claim '{{ $labels.persistentvolumeclaim }}' has been in pending state for more than 5 minutes", + "summary": "Persistent Volume Claim '{{ $labels.persistentvolumeclaim }}' pending in namespace '{{ $labels.namespace }}'" + }, + "expr": "kube_persistentvolumeclaim_status_phase{phase=\"Pending\"} == 1", + "for": "5m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "LostPersistentVolumeClaim", + "annotations": { + "description": "Persistent Volume Claim '{{ $labels.persistentvolumeclaim }}' has been in lost state for more than 5 minutes", + "summary": "Persistent Volume Claim '{{ $labels.persistentvolumeclaim }}' in namespace '{{ $labels.namespace }}' lost it's corresponding persistent volume" + }, + "expr": "kube_persistentvolumeclaim_status_phase{phase=\"Lost\"} == 1", + "for": "5m", + "labels": { + "severity": "warning" + } + } + ] + }, + { + "name": "volume-node", + "rules": [ + { + "alert": "VolumeNodeFileSystemIsReadOnly", + "annotations": { + "description": "Persistent Volume's filesystem on node '{{ $labels.node }}' for persistent volume claim '{{ $labels.persistentvolumeclaim }}' has become read-only", + "summary": "Volume mount failed for persistent volume claim '{{ $labels.persistentvolumeclaim }}' on node '{{ $labels.node }}' due to read-only file-system" + }, + "expr": "kubelet_volume_stats_inodes * on(node) group_left(reason) problem_counter{reason=\"FilesystemIsReadOnly\"} > 0", + "for": "5m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "VolumeNodeExt4Error", + "annotations": { + "description": "Persistent Volume's on node '{{ $labels.node }}' persistent volume claim '{{ $labels.persistentvolumeclaim }}' encountering ext4 filesystem error", + "summary": "Node '{{ $labels.node }}' has encountered errors on ext4 file-system on volume having claim '{{ $labels.persistentvolumeclaim }}'" + }, + "expr": "kubelet_volume_stats_inodes * on(node) group_left(reason) problem_counter{reason=\"Ext4Error\"} > 0", + "for": "5m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "VolumeNodeIOError", + "annotations": { + "description": "Persistent Volume on node '{{ $labels.node }}' for persistent volume claim '{{ $labels.persistentvolumeclaim }}' encountering errors w.r.t buffer I/O ", + "summary": "IO errors encountered on volume having persistent volume claim '{{ $labels.persistentvolumeclaim }}' on node '{{ $labels.node }}'" + }, + "expr": "kubelet_volume_stats_inodes * on(node) group_left(reason) problem_counter{reason=\"IOError\"} > 0", + "for": "5m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "VolumeNodeExt4Warning", + "annotations": { + "description": "Persistent Volume on node '{{ $labels.node }}' receiving ext4 filesystem warning for persistent volume claim '{{ $labels.persistentvolumeclaim }}'", + "summary": "Node '{{ $labels.node }}' has encountered warning on ext4 file-system on volume having claim '{{ $labels.persistentvolumeclaim }}'" + }, + "expr": "kubelet_volume_stats_inodes * on(node) group_left(reason) problem_counter{reason=\"Ext4Warning\"} > 0", + "for": "5m", + "labels": { + "severity": "critical" + } + } + ] + } + ] + } +} diff --git a/charts/kubezero-storage/templates/lvm/grafana-dashboards.yaml b/charts/kubezero-storage/templates/lvm/grafana-dashboards.yaml new file mode 100644 index 00000000..5a90fa85 --- /dev/null +++ b/charts/kubezero-storage/templates/lvm/grafana-dashboards.yaml @@ -0,0 +1,15 @@ +{{- if index .Values "lvm-localpv" "prometheus" "enabled" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" (include "kubezero-lib.fullname" $) "openebs-lvmlocalpv-grafana-dashboard" | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} + labels: + grafana_dashboard: "1" + {{- include "kubezero-lib.labels" . | nindent 4 }} + annotations: + k8s-sidecar-target-directory: KubeZero +binaryData: + openebs-localpv-pool.json.gz: + H4sIAAAAAAAC/+1dbXPbNhL+7l/B47U3dmunkizJTmb6IXYubaZJ44uTdHppRgORkIQzBfIIULbi8f32wwvfCeqVkigb/dBWAE0Ci8Xus4vF7v2BYZgAY5cCilxMzBfGPWtijQ4ilP36In4ZYavo6QfIoW8w62weJ602oIC4gW9B1mFe+e4Y0hEMiJl6BmLQd3g/9QOYah8hW9GKLBdfuo7r8xf6wz44bBwbrWaT/avTOTaaR+lXYzAWH36ZzMX4h/HSgT7NDIFOPfGcDcio7wLfNsO+B/Hfr+zfD/xxE9qI5kZrDjGkb2zWggPHkS0+8EYfXdehyGPtDdGI+COn4f9T6IvhcHJ1W+3nz7udRve0IXsdhG84zb98FT89gKFDYqpHNDct13GARyB/7wA4JKbSfKqzESL7yk0WVhI8t3i37HernWq4iyYT/p7y3xGl4neLibZa8e9k/F/jNoqoIKL52XWCMTSGvht4xjVbo9Qgo1Xx3Vu5HuE37hef5wBBx7508QANs3O14QAEDs1SQBJVsFa6kTWPXcGLrBtThAM3ICe/+H86H2JGyZBA/gnwPISH2YnLeY18SEauY5PyD3mQzQpTMITmcfYJQqFHUnsw+uc+9zs1GbbcEOLci8QTE+AEMGTdXO/D8RIf8BkXznj9eSP/8szvr6VEDDCiYmNCqz+lkCTkTj1nuhPoM4aGebLkaQuoNYLF1Y141uxPf+cCIz8R0/UiMWh+ZHLEMQj6Bs2DGdQyPd9lS0gRXHCp5AAkPWcQsviXauYUC07MwtMPq63DBgj5mo2w/nT8YP/p/OLXmY6fmAKoKR1Tcm4dAh7kWx/mq7HznBprtuaosWaJGot/JiRPaxFEPAdM34UTdqy0FDRdH3ERHup5c8R+f2OsBZz0Q0xyBhZ8r3g7l6/AsYoLZjqA0N9d+jsT2mnKZrWMUH2CRTILKJeORJhBKU/Zot1+YjrTcQS4yCEwCu+4TL5/KJDMc4Ihwp+hT8Ipnz3rPOukFDrwh5BmJ5SZL7yDY0ZQv/BR0ecJVeNMxr3JsEe5IO5xxu8JzXDPsd7P//vL/G4iIEVPQIq/zGMDYUIBtmQnZkv17Ie/zIcsURBmiIxRpkguBw4htl+7PtusOQWQec6HAwECzZdKHbXqNAdMTFY2yxBRZ8DiktNPxLZ69heVzn7dRT6pKRUToa2m4mWKige5zc2w8xi+Zlg3ZXHE7dcjNKDFjizY/kWA7UvgAQvRaRFv99k2BcEQatCtQXctQPc7SAFnO4Nq9F0xRQcahldL0EDj8VXweKZhBiBva0BeU0A+tsHOQLlKQWwDnfM51w1bKkT7NqB6Fct/sg9kDdbD7iooHr97fzC5BxxIKTyxmOgjyCpH5FZAqLBVci8Cd4i8BX1YJH3YeeUAC46ZvOYPgIC6+YcYSV46aIjDZxq5btsHt9d0KsntIFzA9Fycvg/pXfzzoQ9sri0iVeLmVibFWaExVgAP4hhIxXYpBivtpvHxUVYtqMAEn90bzsme68TajTeCPFgQrX8gm+bOeqS2cNl2uOac/cLo5A0gpvngK0Soj/pB+InChCNmDT89c8xcpV3xDwqNiCGzJgpGlwcwV6kkFD8LWX0Hs4CUOZBLZRaxnEAXzeIySN3KbCK2Udi0ihYXWyn1G0NqFN8ZgTXzpDmTRsuOvlU6+ls/PLascPCNSsd+Wjp2HwL7xMXOtMrB5wlfCtrG4C7/JnOMOPefNJf2L4A+YWKf7s67sKQDgM3HpwtY/5t0zD9X2wHd2XZAJHrv02OOJa4a0mcXPm9J8MCHzANeWj/1Xcq1nJJUoSQvsSgi3iBMgDmwiB12A+aZbTpGhH9oWxD+Kv7iAsh9QVyleGeso7i3lEAfRe4tDak0pNKQSkOqRSEV5pLL2UdAJfyqQ+5X3SSiWm/07fLRs50wrHLgrUoH3ikdOMDT2xH0K8Xgp5WOvVs6doTZyBGtdqu2l8PgbY3Bt4XBC874EhB+rkH4JkE4cBzXEuCjx1AIsqbbwuLJhw354eog+cvk1VfZV2tkvj1k3pwHzbFbfIXG5TXD5V8VGrGhFeJWQlIYJjGciWG5AaabOOmP13kGMb8oDvtVAQCzkN9MU61gsDlorESBs7HgHEQ4xw5SxydkOaUWEcRnapDUbGmUtEmU5Ex6YhduCxxZge8zUia7fyuBBuCuoolWcBSekX3zYws2FcAqZ09y49AwUsNIDSM1jNw8jOz1yJTNbvwBDkQ0IfuLayFvBHcfr4Q5ySzQqSBZRH14ZzmBDVU0w+K1arQYKzNPCWULrrgY0sIBuhNaznEM9m3o0Reqb/Oz+/f86L7AjsUlqyhCVojOZ7GQWTZWNpJAeYW8oBCaK4iWjLU93rwV42kr5glbMcW46zIzRkdcbDbiYkdmjLdVM4ZHY+CqJluFKSMHNI8KF5Ubc16djLk5s9/GbcTCEBa244CDABEZYUhG/HBbhyhSo5DRW4iHEs43Mu2QrJBJZSEjcYlTO25hpfWLaPgltKvSyLxC11WrUxgG4ywbYolnC1SRCiE5/VOIeRNMhkXOjKSOokce8hZahTWSb+WWkOIaDL99oHhaeb8mmSk3vUjubaLxNm/0mZyPhQX2LrmihqGVxhJq5Qh4xqGPkRWVRcMpdZNcP81TfJ4mEoYoZ5KAZLIWeJG9mH0dWwMb+gI8mgPHTe06udPeZ3g06WSWP1TtHiaprJvCV7jp50H7Lfcj5Puqvnm0E905ju5tbFGDsrnyGyg10R7iMsxCZJjtEUw7GL4splw+wGG40XJ/sITWKRl5yoK7T0sd4Kuu3xHuSMgKVSFyepEGQ9hGE2QHIL4c+FDQchI+x8133JuX/Xw/sG7kjklPLH29E2WsvSj5We5pteyMZaRCI03BHZyxUb/7bsR0529wKkI6+/9h8vBFs3neyPKBDS00BsIxlfUtDWJekh6ZLLuFzs7sHHiHO7wABBZD84UWKTwu1YiZG1OKFHO3aMksm+auJlNoDmeTu0Ra2GrTImcB7hFWKE7R/hZO4lFnkuJpHFa98b0SEGs1NBDTQKwOQIxg4G0XifEvsqWkZGEIVhvsUTZ0DT7WBR/t0+YTAB/t05YGH7s7zFcnU1nwHD+cc56MEmXw3MAyxXD++Ejnuip7/TK5rnYdOviZj9n4+8tNHLiFZyjKMaRPp4Td/r25wopkjlnTRzazPudY9kkqfcUqXxTn9rO+ETK0WZvDVWBRNIE9BndoQMzNnXUvughiL530GUDkYAzb9aFUuCMudrkjyhI7V8qjxYx6uyY9gUOBAndJ+cL3Kz8G7y6b1f70VH0Kfjon/xjHQb9CYENfDdUvprP1C1NAljL6JENNbhYtSJKtHWE7E+mXDsXw/WSoMv9Cdvvbz6yLjhC+YrbOolZhgm+LuUxM+QKqGuXi1uTmPfiMSoUkWVum1Ayv/+ZIeLH8TTH5I10OwgeYyKlFxrd6MaT4kUj6tUi6d1GUQqVxdWyyMpZOIVbLY+ly2l4h72w4QZZSEtrjngfoSNU1cgktC+pTtZe9JxLziq4Yjpb3XeTF89fSm7sPc7dKGBkG/WH5giz6FtcfAlxIxVa6smGU5EUkRe/zig5hG94l3QdlpMr4dvO0emGcFTqz3KG6Fh2yhyrlQMQfqkv9gkFU185DB0yj0BG+qnjDPuIRpi5nogEf8neniKhe/uI2jwI6hExSmvH1KGlzWnLdubNcTZs4B42QWUxkL13fRh+prB3a0m6scKLS6c4+UeHum5fko7pslTxvyTUmxy25DulmyzcKJ1sePq521qI84NBHLXt71EKC8aEPKDzk2K/HrIKbHo+yJz3LZX8L2bpIhHmvBopfOuOvR8aPhosPpcY5krW3eg4c0EMu2I+MwxCp3o4g7g3Y+pdA1BIsavxgNI6OFj3RiXpeMxUpXH2tmfhSzPUnAq19Oup58974wMetz3iqOuM5bTbKTB7kekR99mGydeD8Y27uAGTl45zTpg4l0binwlCS1YDP+VMHPi0NfPYH+PC0xvCpIB852T2EPn+IgWvsUxX26XTOVsM+goPqCH46nXMNfh43+Glt1elzvgr2aW8N+/AjmH9D3y2Nvl0EFvloOKLXysMc7SvSkGmGr0ieO9YRLK1wI6mAotozURT3wxgyBmOjh70laJUyUj1h6v8hSTBzAWp5UezDT38Yr5ju1Ci2KhR71i2DfBelGFawjXDhGYcnR8ZPEtEahz8e1RHVnp1pUKs9ehV69FZBtd2G9uhVD0/VF9g0Pl0fJCG3x5VUj0C2ODZ51B69gCIHfZPpB/fJpffTe+OTYugaEq0Jic4b7TLEEIqWQoB5go2+59xURxh03uhoHFSvALdOVx3glvzh7AC3jyPEyze4ztIRbTrprq6q9oTT7pZeRNbl1uaUWxtB4NDRdB/rrXnAp6jaUnHbKrbGgCjHmAaG0K6WcbZVdW2MiLwmRwx4hxSXKSqsvjangNmpLmBm7KiIcKckqW1LJ7Xd6I1AKbXDay9JThh+n63nLXGhbfH7Z7+KLwpcGpC1ypap36TLS2iku0Gk29L1g58IoGWaz/X3Ec7+N4AB3CSYnQOjmhpGGbuqA1uGozoNjaM2iaOS04ttYKg+HIEJYthlNfy0aj74i+izBp+uwae7D7BrkURgOlXXaqm6lJKwmERnR4m8PimSaFWTOmf/kl2VpU+qKNOV4K1lazmNk2N+inDgBuREMD8x1ynJtEr+oefLOiy6nZL8Q3MUbV5NWumtaLo+N59ic4QhCvSN0Sbjm+TbN7BgiXqMtXPOhAGE/u5SbhmYpfBNCF5SVD/KtMDZTc0tk0+YW4eqo+QQmd7vVkcX8vpsQ1Wntt02chdlMjytPr8KCg+Esner1UfFieclyHkdYmjSZ8yU5DjUwEQDk10Bk3dRWYunoI+Pt0TLYJ/Q3kZASsEdUIpSzjVKqSlK4fWDdoJUsjJpG2AlLpVUC8CSlyPbRy7xCFaFMDqIf+2D6LP2CjH8bV3gRd8k3W2kfihSRYGX9cXp8vVdhDe4VGrWJUxehsRGQ36dHbIOj18zPL7deP68tK5La3bg/C6CzFeOmG83Gw0dMq8DaXQgjQ4Z14E08wJpAnyD3Vu8j6E04vp6IUP/3sSGA/vExc50H8PC48GfREaiDgx/khFNCoNc8v25jmjapB+S4fIxIvxDW7Ejk8+tFRMuDLyr4rv2PDxJ1ynUdQqLJ1+brFO48pHXwrXYFq0+9qkulRB19T3j+qlV33tEJSJXOXHuLJu44LxVgtZaS5blq0+dPI53CugnLMUUt8v8K0+9UN62SVVtpbxCcqDm5grpiQQduoDejAJ6C1W9W6LAnjgX0VXyjGyVvLNZVfK6+1Al72lXwjsvue/VbS2bKEiXwdtp3Mn5Srkj2zp3pA5IqWdASo3K4C2LQHV9vIxm0MXxqg6UaTbKE0nuZ3U8NiOdRFLjoQrP/VYDRF0NiDQg2htAtLPyeLVCRHtTNy8DiXTRvIox0Vm788iq5rEpdTUo0mXzKnMSPe+ugomefMlgfWtJ17+rERrS9e8qp74ujKfGqboqXuUw9fy0/djr4rE5ameeduZV6MxbCbie6cp4GrjuD3DdVWG8WvnyRMW8PamZl8AkXSxvE0jpeev0EVbLY9Nqa3BUryi4ZqPskuCC9fIy5TOvPu9B1Tx9a0zfGqvfrTF9m6j2F7nEdbpdUf7pXdq73OVav2Zi8cnusle7pPwrYdU9PdojzC/P7ZLyb7DxicAN34tsL49QSzLxtrt7czHSU9z4U7sk6nazb/u3I73wEuljJdVFpaQaMD31aEl1WSmppLvw0RLr1UK3Z3Pm+t5coHUc1wJUmU/MiHVn2d3VJe/BLnB/9nJG3yt9VzZLRlWmtdRd2U5Z56XyIm1MZdVl2DSfqO7YhoyiumMbXZfd2q3Yg/CBqBK5yK7WkI4vkzA8OQYJNGmdyeYoMaYN/Bv5JAWpBIrsAddPPENscSH+58W1GX+LCnFJEU58aTI9V7Lt7lWnfhl2ItCRB2gqGReloVO48rJwM/WA0rFjQ2L5yAuTVub88LLEaKE5lLxpDBkKnJeO6pRR7WM2x4FDkeLxkEfSTsu0JyreSWmvHi8q6gs3uKeiiOz+HBElWwci4otmpnUI7/KPkhvkffKd6ym2FMOOuC817DwWzri3nc8p35/yEDhzbD7XhSt9tYjZZFGOU072njw1keeAASdtD+GBm+jfI3NH3GDirNdkLj/kn5/HCSvMfw4zHD774ejFsx8W5wnFsRuTJGLhyb+icZrZ3sKEeJv64ZDj5IxTHUz6fpQvUubR3yEbcpQ4GRbtNDUaPDbEUfPOODSNWZfh1LK/W4pj1yZVwsyt9SRb7bl4DaZoqdadhynwg3fFSbWyL15A3nni5XoXUxuWG5otW56ntFXKZlrszcx1M9PchVByFnce5QNarmYmHD025M40dibF4mRXy4gw5R8tLb+2S9VHKvAOIvP2QRoXKGUJxgmssXt70oyCc8LM01iGAyR/5iHrRjrQk7ZvLk4wceLCePv5nRGLOTOQtu3rRr/72/gGh0bRJDaczg8e/g9vPLns8x8BAA== +{{- end }} diff --git a/charts/kubezero-storage/templates/lvm/prometheus-rules.yaml b/charts/kubezero-storage/templates/lvm/prometheus-rules.yaml new file mode 100644 index 00000000..905c6b37 --- /dev/null +++ b/charts/kubezero-storage/templates/lvm/prometheus-rules.yaml @@ -0,0 +1,104 @@ +{{- if index .Values "lvm-localpv" "prometheus" "enabled" }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kubezero-lib.fullname" $) "prometheus-rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubezero-lib.labels" . | nindent 4 }} +spec: + groups: + - name: lvm-pool + rules: + - alert: LVMVolumeGroupMissingPhysicalVolume + annotations: + componentType: volume group + description: LVM volume group '{{`{{`}} $labels.name {{`}}`}}' on node '{{`{{`}} $labels.instance {{`}}`}}' is missing {{`{{`}} $value {{`}}`}} underlying physical volume(s). + summary: LVM volume group '{{`{{`}} $labels.name {{`}}`}}' is missing the underlying physical volume. + vgName: '{{`{{`}} $labels.name {{`}}`}}' + expr: lvm_vg_missing_pv_count > 0 + for: 5m + labels: + severity: critical + - alert: LVMVolumeGroupCapacityLow + annotations: + componentType: volume group + description: LVM volume group '{{`{{`}} $labels.name {{`}}`}}' on node '{{`{{`}} $labels.instance {{`}}`}}' has {{`{{`}} with printf "lvm_vg_free_size_bytes{instance='%s',name='%s'}" $labels.instance $labels.name | query {{`}}`}} {{`{{`}} . | first | value {{`}}`}} {{`{{`}} end {{`}}`}}bytes of space remaining + summary: LVM volume group '{{`{{`}} $labels.name {{`}}`}}' is running low on capacity. Already {{`{{`}} $value {{`}}`}}% of total capacity is consumed. + vgName: '{{`{{`}} $labels.name {{`}}`}}' + expr: ((lvm_vg_total_size_bytes - lvm_vg_free_size_bytes)/lvm_vg_total_size_bytes)*100 > 90 + for: 5m + labels: + severity: critical + - alert: LVMThinPoolCapacityLow + annotations: + componentType: logical volume + description: LVM thin pool '{{`{{`}} $labels.name {{`}}`}}' on node '{{`{{`}} $labels.instance {{`}}`}}' has {{`{{`}} with printf "lvm_lv_total_size_bytes{instance='%s',name='%s',segtype='%s'}-((lvm_lv_used_percent{instance='%s',name='%s',segtype='%s'}*lvm_lv_total_size_bytes{instance='%s',name='%s',segtype='%s'})/100)" $labels.instance $labels.name $labels.segtype $labels.instance $labels.name $labels.segtype $labels.instance $labels.name $labels.segtype | query {{`}}`}} {{`{{`}} . | first | value {{`}}`}} {{`{{`}} end {{`}}`}}bytes of space remaining + lvName: '{{`{{`}} $labels.name {{`}}`}}' + summary: LVM thin pool '{{`{{`}} $labels.name {{`}}`}}' is running low on capacity. Already {{`{{`}} $value {{`}}`}}% of total capacity is consumed. + expr: lvm_lv_used_percent{segtype="thin-pool"} > 90 + for: 5m + labels: + severity: critical + - name: persistent-volume-claim + rules: + - alert: StalePersistentVolumeClaim + annotations: + description: Persistent Volume Claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' has no consumer + summary: Persistent Volume Claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' in namespace '{{`{{`}} $labels.namespace {{`}}`}}' is not consumed by any pod in any namespace + expr: kube_persistentvolumeclaim_info unless (kube_persistentvolumeclaim_info * on(persistentvolumeclaim) group_left kube_pod_spec_volumes_persistentvolumeclaims_info) == 1 + for: 5m + labels: + severity: info + - alert: PendingPersistentVolumeClaim + annotations: + description: Persistent Volume Claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' has been in pending state for more than 5 minutes + summary: Persistent Volume Claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' pending in namespace '{{`{{`}} $labels.namespace {{`}}`}}' + expr: kube_persistentvolumeclaim_status_phase{phase="Pending"} == 1 + for: 5m + labels: + severity: warning + - alert: LostPersistentVolumeClaim + annotations: + description: Persistent Volume Claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' has been in lost state for more than 5 minutes + summary: Persistent Volume Claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' in namespace '{{`{{`}} $labels.namespace {{`}}`}}' lost it's corresponding persistent volume + expr: kube_persistentvolumeclaim_status_phase{phase="Lost"} == 1 + for: 5m + labels: + severity: warning + - name: volume-node + rules: + - alert: VolumeNodeFileSystemIsReadOnly + annotations: + description: Persistent Volume's filesystem on node '{{`{{`}} $labels.node {{`}}`}}' for persistent volume claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' has become read-only + summary: Volume mount failed for persistent volume claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' on node '{{`{{`}} $labels.node {{`}}`}}' due to read-only file-system + expr: kubelet_volume_stats_inodes * on(node) group_left(reason) problem_counter{reason="FilesystemIsReadOnly"} > 0 + for: 5m + labels: + severity: critical + - alert: VolumeNodeExt4Error + annotations: + description: Persistent Volume's on node '{{`{{`}} $labels.node {{`}}`}}' persistent volume claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' encountering ext4 filesystem error + summary: Node '{{`{{`}} $labels.node {{`}}`}}' has encountered errors on ext4 file-system on volume having claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' + expr: kubelet_volume_stats_inodes * on(node) group_left(reason) problem_counter{reason="Ext4Error"} > 0 + for: 5m + labels: + severity: critical + - alert: VolumeNodeIOError + annotations: + description: 'Persistent Volume on node ''{{`{{`}} $labels.node {{`}}`}}'' for persistent volume claim ''{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}'' encountering errors w.r.t buffer I/O ' + summary: IO errors encountered on volume having persistent volume claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' on node '{{`{{`}} $labels.node {{`}}`}}' + expr: kubelet_volume_stats_inodes * on(node) group_left(reason) problem_counter{reason="IOError"} > 0 + for: 5m + labels: + severity: critical + - alert: VolumeNodeExt4Warning + annotations: + description: Persistent Volume on node '{{`{{`}} $labels.node {{`}}`}}' receiving ext4 filesystem warning for persistent volume claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' + summary: Node '{{`{{`}} $labels.node {{`}}`}}' has encountered warning on ext4 file-system on volume having claim '{{`{{`}} $labels.persistentvolumeclaim {{`}}`}}' + expr: kubelet_volume_stats_inodes * on(node) group_left(reason) problem_counter{reason="Ext4Warning"} > 0 + for: 5m + labels: + severity: critical + +{{- end }} diff --git a/charts/kubezero-storage/templates/lvm/servicemonitor.yaml b/charts/kubezero-storage/templates/lvm/servicemonitor.yaml new file mode 100644 index 00000000..f3ab7dd9 --- /dev/null +++ b/charts/kubezero-storage/templates/lvm/servicemonitor.yaml @@ -0,0 +1,18 @@ +{{- if index .Values "lvm-localpv" "prometheus" "enabled" }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ printf "%s-%s" (include "kubezero-lib.fullname" $) "openebs-monitoring-lvmlocalpv" | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubezero-lib.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + name: openebs-lvm-node + namespaceSelector: + any: true + endpoints: + - port: metrics + path: /metrics +{{- end }} diff --git a/charts/kubezero-storage/update.sh b/charts/kubezero-storage/update.sh index 04f21e5f..3d2d5e1b 100755 --- a/charts/kubezero-storage/update.sh +++ b/charts/kubezero-storage/update.sh @@ -26,3 +26,7 @@ VERSION=$(yq eval '.dependencies[] | select(.name=="aws-efs-csi-driver") | .vers rm -rf charts/aws-efs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-efs-csi-driver/releases/download/helm-chart-aws-efs-csi-driver-${VERSION}/aws-efs-csi-driver-${VERSION}.tgz | tar xfz - -C charts patch -i efs.patch -p0 --no-backup-if-mismatch + +# Metrics +cd jsonnet +make render diff --git a/charts/kubezero-storage/values.yaml b/charts/kubezero-storage/values.yaml index 24a951e7..459a35fc 100644 --- a/charts/kubezero-storage/values.yaml +++ b/charts/kubezero-storage/values.yaml @@ -44,6 +44,8 @@ lvm-localpv: analytics: enabled: false + prometheus: + enabled: false gemini: enabled: false @@ -170,8 +172,12 @@ aws-efs-csi-driver: effect: NoSchedule operator: Exists - storageClasses: - - name: efs-sc + #storageClasses: + #- name: efs-sc + # parameters: + # provisioningMode: efs-ap + # directoryPerms: "700" + # fileSystemId: #PersistentVolumes: # - name: example-pv