diff --git a/Dockerfile b/Dockerfile index e8e5cdda..12c69db0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG ALPINE_VERSION=3.21 FROM docker.io/alpine:${ALPINE_VERSION} ARG ALPINE_VERSION -ARG KUBE_VERSION=1.31 +ARG KUBE_VERSION=1.32 ARG SOPS_VERSION="3.10.1" ARG VALS_VERSION="0.40.1" diff --git a/admin/hooks-1.32.sh b/admin/hooks-1.32.sh new file mode 100644 index 00000000..87449dee --- /dev/null +++ b/admin/hooks-1.32.sh @@ -0,0 +1,28 @@ +### v1.32 + +# All things BEFORE the first controller / control plane upgrade +pre_control_plane_upgrade_cluster() { + echo +} + + +# All things after the first controller / control plane upgrade +post_control_plane_upgrade_cluster() { + echo +} + + +# All things AFTER all contollers are on the new version +pre_cluster_upgrade_final() { + set +e + + echo + + set -e +} + + +# Last call +post_cluster_upgrade_final() { + echo +} diff --git a/admin/kubezero.sh b/admin/kubezero.sh index 7dc2260a..5891334c 100755 --- a/admin/kubezero.sh +++ b/admin/kubezero.sh @@ -57,6 +57,7 @@ render_kubeadm() { local phase=$1 helm template $CHARTS/kubeadm --output-dir ${WORKDIR} \ + --kube-version $KUBE_VERSION \ -f ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml \ --set patches=/etc/kubernetes/patches @@ -113,6 +114,8 @@ post_kubeadm() { # Migrate KubeZero Config to current version upgrade_kubezero_config() { + ARGOCD=$(argo_used) + # get current values, argo app over cm get_kubezero_values $ARGOCD @@ -166,9 +169,7 @@ kubeadm_upgrade() { else pre_cluster_upgrade_final - # Finally upgrade addons last, with 1.32 we can ONLY call addon phase - #_kubeadm upgrade apply phase addon all $KUBE_VERSION - _kubeadm upgrade apply $KUBE_VERSION + _kubeadm upgrade apply phase addon all $KUBE_VERSION post_cluster_upgrade_final @@ -204,10 +205,6 @@ control_plane_node() { # Put PKI in place cp -r ${WORKDIR}/pki ${HOSTFS}/etc/kubernetes - ### 1.31 only to clean up previous aws-iam-auth certs - rm -f ${HOSTFS}/etc/kubernetes/pki/aws-iam-authenticator.key ${HOSTFS}/etc/kubernetes/pki/aws-iam-authenticator.crt - ### - # Always use kubeadm kubectl config to never run into chicken egg with custom auth hooks cp ${WORKDIR}/super-admin.conf ${HOSTFS}/root/.kube/config @@ -341,9 +338,7 @@ apply_module() { [ -f $CHARTS/kubezero/hooks.d/pre-install.sh ] && . $CHARTS/kubezero/hooks.d/pre-install.sh kubectl replace -f $WORKDIR/kubezero/templates $(field_manager $ARGOCD) else - #_helm apply $t - # During 1.31 we change the ArgoCD tracking so replace - _helm replace $t + _helm apply $t fi done @@ -357,7 +352,9 @@ delete_module() { get_kubezero_values $ARGOCD # Always use embedded kubezero chart - helm template $CHARTS/kubezero -f $WORKDIR/kubezero-values.yaml --version ~$KUBE_VERSION --devel --output-dir $WORKDIR + helm template $CHARTS/kubezero -f $WORKDIR/kubezero-values.yaml \ + --kube-version $KUBE_VERSION \ + --version ~$KUBE_VERSION --devel --output-dir $WORKDIR for t in $MODULES; do _helm delete $t diff --git a/admin/libhelm.sh b/admin/libhelm.sh index 57a90ce4..0ec8355f 100644 --- a/admin/libhelm.sh +++ b/admin/libhelm.sh @@ -356,7 +356,7 @@ EOF function admin_job() { TASKS="$1" - [ -z "$KUBE_VERSION" ] && KUBE_VERSION="latest" + ADMIN_TAG=${ADMIN_TAG:-$KUBE_VERSION} echo "Deploy cluster admin task: $TASKS" cat <= 1.31.0-0" +kubeVersion: ">= 1.32.0-0" diff --git a/charts/kubeadm/README.md b/charts/kubeadm/README.md index c7c68b88..44cff2ff 100644 --- a/charts/kubeadm/README.md +++ b/charts/kubeadm/README.md @@ -1,6 +1,6 @@ # kubeadm -![Version: 1.25.8](https://img.shields.io/badge/Version-1.25.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 1.32.3](https://img.shields.io/badge/Version-1.32.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero Kubeadm cluster config @@ -14,19 +14,18 @@ KubeZero Kubeadm cluster config ## Requirements -Kubernetes: `>= 1.25.0` +Kubernetes: `>= 1.32.0-0` ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| | api.apiAudiences | string | `"istio-ca"` | | -| api.awsIamAuth.enabled | bool | `false` | | -| api.awsIamAuth.kubeAdminRole | string | `"arn:aws:iam::000000000000:role/KubernetesNode"` | | -| api.awsIamAuth.workerNodeRole | string | `"arn:aws:iam::000000000000:role/KubernetesNode"` | | +| api.awsIamAuth | bool | `false` | | | api.endpoint | string | `"kube-api.changeme.org:6443"` | | | api.etcdServers | string | `"https://etcd:2379"` | | | api.extraArgs | object | `{}` | | +| api.falco.enabled | bool | `false` | | | api.listenPort | int | `6443` | | | api.oidcEndpoint | string | `""` | s3://${CFN[ConfigBucket]}/k8s/$CLUSTERNAME | | api.serviceAccountIssuer | string | `""` | https://s3.${REGION}.amazonaws.com/${CFN[ConfigBucket]}/k8s/$CLUSTERNAME | diff --git a/charts/kubeadm/templates/ClusterConfiguration.yaml b/charts/kubeadm/templates/ClusterConfiguration.yaml index 2adfef1f..c247f9e1 100644 --- a/charts/kubeadm/templates/ClusterConfiguration.yaml +++ b/charts/kubeadm/templates/ClusterConfiguration.yaml @@ -4,6 +4,7 @@ kubernetesVersion: {{ .Chart.Version }} clusterName: {{ .Values.global.clusterName }} featureGates: ControlPlaneKubeletLocalMode: true + NodeLocalCRISocket: true controlPlaneEndpoint: {{ .Values.api.endpoint }} networking: podSubnet: 10.244.0.0/16 @@ -119,6 +120,8 @@ apiServer: value: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }} - name: authorization-config value: /etc/kubernetes/apiserver/authz-config.yaml + - name: authentication-config + value: /etc/kubernetes/apiserver/authn-config.yaml - name: enable-admission-plugins value: DenyServiceExternalIPs,NodeRestriction,EventRateLimit,ExtendedResourceToleration {{- if .Values.global.highAvailable }} @@ -127,6 +130,11 @@ apiServer: {{- end }} - name: logging-format value: json + # Required for MutatingAdmissionPolicy + # Required for VolumeAttributesClass + # Required for CoordinatedLeaderElection - coordination.k8s.io/v1alpha1=true + - name: runtime-config + value: admissionregistration.k8s.io/v1alpha1=true,storage.k8s.io/v1beta1=true {{- with .Values.api.extraArgs }} {{- toYaml . | nindent 4 }} {{- end }} diff --git a/charts/kubeadm/templates/_helpers.tpl b/charts/kubeadm/templates/_helpers.tpl index aa22b4b5..fcbbee5d 100644 --- a/charts/kubeadm/templates/_helpers.tpl +++ b/charts/kubeadm/templates/_helpers.tpl @@ -1,9 +1,9 @@ {{- /* Feature gates for all control plane components */ -}} {{- /* Issues: MemoryQoS */ -}} {{- /* v1.28: PodAndContainerStatsFromCRI still not working */ -}} -{{- /* v1.28: UnknownVersionInteroperabilityProxy requires StorageVersionAPI which is still alpha in 1.30 */ -}} +{{- /* v1.32: not required? working ? "DisableNodeKubeProxyVersion" "CoordinatedLeaderElection" */ -}} {{- define "kubeadm.featuregates" }} -{{- $gates := list "CustomCPUCFSQuotaPeriod" "AuthorizeWithSelectors" "AuthorizeNodeWithSelectors" "ConsistentListFromCache" "VolumeAttributesClass" "WatchList" }} +{{- $gates := list "CustomCPUCFSQuotaPeriod" "VolumeAttributesClass" "MutatingAdmissionPolicy" }} {{- if eq .return "csv" }} {{- range $key := $gates }} {{- $key }}=true, diff --git a/charts/kubeadm/templates/apiserver/admission-configuration.yaml b/charts/kubeadm/templates/apiserver/admission-configuration.yaml index 76d79d07..e725bd16 100644 --- a/charts/kubeadm/templates/apiserver/admission-configuration.yaml +++ b/charts/kubeadm/templates/apiserver/admission-configuration.yaml @@ -1,7 +1,5 @@ apiVersion: apiserver.config.k8s.io/v1 kind: AdmissionConfiguration -metadata: - name: kubezero-admissionconfiguration plugins: - name: EventRateLimit path: /etc/kubernetes/apiserver/event-config.yaml diff --git a/charts/kubeadm/templates/apiserver/authn-config.yaml b/charts/kubeadm/templates/apiserver/authn-config.yaml new file mode 100644 index 00000000..77b98f55 --- /dev/null +++ b/charts/kubeadm/templates/apiserver/authn-config.yaml @@ -0,0 +1,10 @@ +apiVersion: apiserver.config.k8s.io/v1beta1 +kind: AuthenticationConfiguration +anonymous: + enabled: true + conditions: + - path: /livez + - path: /readyz + - path: /healthz + - path: /.well-known/openid-configuration + - path: /openid/v1/jwks diff --git a/charts/kubeadm/templates/apiserver/authz-config.yaml b/charts/kubeadm/templates/apiserver/authz-config.yaml index 7b625227..c1d657aa 100644 --- a/charts/kubeadm/templates/apiserver/authz-config.yaml +++ b/charts/kubeadm/templates/apiserver/authz-config.yaml @@ -1,4 +1,4 @@ -apiVersion: apiserver.config.k8s.io/v1beta1 +apiVersion: apiserver.config.k8s.io/v1 kind: AuthorizationConfiguration authorizers: - type: Node diff --git a/charts/kubeadm/templates/patches/kube-scheduler1+json.yaml b/charts/kubeadm/templates/patches/kube-scheduler1+json.yaml index 41b6183c..503ac480 100644 --- a/charts/kubeadm/templates/patches/kube-scheduler1+json.yaml +++ b/charts/kubeadm/templates/patches/kube-scheduler1+json.yaml @@ -8,3 +8,6 @@ json: - op: replace path: /spec/containers/0/startupProbe/httpGet/host value: {{ .Values.listenAddress }} + - op: replace + path: /spec/containers/0/readinessProbe/httpGet/host + value: {{ .Values.listenAddress }} diff --git a/charts/kubeadm/update.sh b/charts/kubeadm/update.sh new file mode 100755 index 00000000..3f7c5736 --- /dev/null +++ b/charts/kubeadm/update.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -ex + +. ../../scripts/lib-update.sh + +login_ecr_public +update_helm + +update_docs diff --git a/charts/kubezero-addons/README.md b/charts/kubezero-addons/README.md index 666a004f..a98d6129 100644 --- a/charts/kubezero-addons/README.md +++ b/charts/kubezero-addons/README.md @@ -1,6 +1,6 @@ # kubezero-addons -![Version: 0.8.13](https://img.shields.io/badge/Version-0.8.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.30](https://img.shields.io/badge/AppVersion-v1.30-informational?style=flat-square) +![Version: 0.8.14](https://img.shields.io/badge/Version-0.8.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.31](https://img.shields.io/badge/AppVersion-v1.31-informational?style=flat-square) KubeZero umbrella chart for various optional cluster addons @@ -18,13 +18,13 @@ Kubernetes: `>= 1.30.0-0` | Repository | Name | Version | |------------|------|---------| -| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.17.1 | -| https://caas-team.github.io/helm-charts/ | py-kube-downscaler | 0.2.12 | -| https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.15.1 | -| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.46.0 | -| https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.17.0 | +| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.17.2 | +| https://caas-team.github.io/helm-charts/ | py-kube-downscaler | 0.3.2 | +| https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.16.1 | +| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.46.6 | +| https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.17.1 | | https://twin.github.io/helm-charts | aws-eks-asg-rolling-update-handler | 1.5.0 | -| oci://public.ecr.aws/aws-ec2/helm | aws-node-termination-handler | 0.26.0 | +| oci://public.ecr.aws/aws-ec2/helm | aws-node-termination-handler | 0.27.0 | | oci://public.ecr.aws/neuron | neuron-helm-chart | 1.1.1 | # MetalLB @@ -109,7 +109,7 @@ Device plugin for [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/) | cluster-autoscaler.extraArgs.scan-interval | string | `"30s"` | | | cluster-autoscaler.extraArgs.skip-nodes-with-local-storage | bool | `false` | | | cluster-autoscaler.image.repository | string | `"registry.k8s.io/autoscaling/cluster-autoscaler"` | | -| cluster-autoscaler.image.tag | string | `"v1.31.1"` | | +| cluster-autoscaler.image.tag | string | `"v1.32.1"` | | | cluster-autoscaler.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | | cluster-autoscaler.podDisruptionBudget | bool | `false` | | | cluster-autoscaler.prometheusRule.enabled | bool | `false` | | diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml index c14d15f5..99a179e7 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.24.0 +appVersion: 1.25.0 description: A Helm chart for the AWS Node Termination Handler. home: https://github.com/aws/aws-node-termination-handler/ icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png @@ -21,4 +21,4 @@ name: aws-node-termination-handler sources: - https://github.com/aws/aws-node-termination-handler/ type: application -version: 0.26.0 +version: 0.27.0 diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/README.md b/charts/kubezero-addons/charts/aws-node-termination-handler/README.md index 06ceac1a..081ff487 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/README.md +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/README.md @@ -95,6 +95,7 @@ The configuration in this table applies to all AWS Node Termination Handler mode | `webhookTemplateConfigMapName` | Pass the webhook template file as a configmap. | "``" | | `webhookTemplateConfigMapKey` | Name of the Configmap key storing the template file. | `""` | | `enableSqsTerminationDraining` | If `true`, this turns on queue-processor mode which drains nodes when an SQS termination event is received. | `false` | +| `enableOutOfServiceTaint` | If `true`, this will add out-of-service taint to node after cordon/drain process which would forcefully evict pods without matching tolerations and detach persistent volumes. | `false` | ### Queue-Processor Mode Configuration @@ -120,6 +121,9 @@ The configuration in this table applies to AWS Node Termination Handler in queue | `managedAsgTag` | [DEPRECATED](Use `managedTag` instead) The node tag to check if `checkASGTagBeforeDraining` is `true`. | `useProviderId` | If `true`, fetch node name through Kubernetes node spec ProviderID instead of AWS event PrivateDnsHostname. | `false` | | `topologySpreadConstraints` | [Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/) for pod scheduling. Useful with a highly available deployment to reduce the risk of running multiple replicas on the same Node | `[]` | +| `heartbeatInterval` | The time period in seconds between consecutive heartbeat signals. Valid range: 30-3600 seconds (30 seconds to 1 hour). | `-1` | +| `heartbeatUntil` | The duration in seconds over which heartbeat signals are sent. Valid range: 60-172800 seconds (1 minute to 48 hours). | `-1` | + ### IMDS Mode Configuration The configuration in this table applies to AWS Node Termination Handler in IMDS mode. diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.linux.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.linux.yaml index be6385de..ee5298be 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.linux.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.linux.yaml @@ -99,6 +99,8 @@ spec: value: {{ .Values.cordonOnly | quote }} - name: TAINT_NODE value: {{ .Values.taintNode | quote }} + - name: ENABLE_OUT_OF_SERVICE_TAINT + value: {{ .Values.enableOutOfServiceTaint | quote }} - name: EXCLUDE_FROM_LOAD_BALANCERS value: {{ .Values.excludeFromLoadBalancers | quote }} - name: DELETE_LOCAL_DATA diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.windows.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.windows.yaml index 95af69d1..8c296373 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.windows.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.windows.yaml @@ -99,6 +99,8 @@ spec: value: {{ .Values.cordonOnly | quote }} - name: TAINT_NODE value: {{ .Values.taintNode | quote }} + - name: ENABLE_OUT_OF_SERVICE_TAINT + value: {{ .Values.enableOutOfServiceTaint | quote }} - name: EXCLUDE_FROM_LOAD_BALANCERS value: {{ .Values.excludeFromLoadBalancers | quote }} - name: DELETE_LOCAL_DATA diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/deployment.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/deployment.yaml index c0e2d285..f74eb3da 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/deployment.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/deployment.yaml @@ -102,6 +102,8 @@ spec: value: {{ .Values.cordonOnly | quote }} - name: TAINT_NODE value: {{ .Values.taintNode | quote }} + - name: ENABLE_OUT_OF_SERVICE_TAINT + value: {{ .Values.enableOutOfServiceTaint | quote }} - name: EXCLUDE_FROM_LOAD_BALANCERS value: {{ .Values.excludeFromLoadBalancers | quote }} - name: DELETE_LOCAL_DATA diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/values.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/values.yaml index f6c7bf42..1b145bd7 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/values.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/values.yaml @@ -86,6 +86,9 @@ cordonOnly: false # Taint node upon spot interruption termination notice. taintNode: false +# Add out-of-service taint to node after cordon/drain process which would forcefully evict pods without matching tolerations and detach persistent volumes. +enableOutOfServiceTaint: false + # Exclude node from load balancer before cordoning via the ServiceNodeExclusion feature gate. excludeFromLoadBalancers: false @@ -285,6 +288,12 @@ enableRebalanceDraining: false # deleteSqsMsgIfNodeNotFound If true, delete the SQS Message from the SQS Queue if the targeted node(s) are not found. Only used in Queue Processor mode. deleteSqsMsgIfNodeNotFound: false +# The time period in seconds between consecutive heartbeat signals. Valid range: 30-3600 seconds (30 seconds to 1 hour). +heartbeatInterval: -1 + +# The duration in seconds over which heartbeat signals are sent. Valid range: 60-172800 seconds (1 minute to 48 hours). +heartbeatUntil: -1 + # --------------------------------------------------------------------------------------------------------------------- # Testing # --------------------------------------------------------------------------------------------------------------------- diff --git a/charts/kubezero-addons/values.yaml b/charts/kubezero-addons/values.yaml index ef476b26..e4cef89a 100644 --- a/charts/kubezero-addons/values.yaml +++ b/charts/kubezero-addons/values.yaml @@ -219,7 +219,7 @@ cluster-autoscaler: image: repository: registry.k8s.io/autoscaling/cluster-autoscaler - tag: v1.31.1 + tag: v1.32.1 autoDiscovery: clusterName: "" diff --git a/charts/kubezero-network/Chart.yaml b/charts/kubezero-network/Chart.yaml index e429ae36..53856862 100644 --- a/charts/kubezero-network/Chart.yaml +++ b/charts/kubezero-network/Chart.yaml @@ -30,4 +30,4 @@ dependencies: version: 1.24.0 repository: https://haproxytech.github.io/helm-charts condition: haproxy.enabled -kubeVersion: ">= 1.29.0-0" +kubeVersion: ">= 1.30.0-0" diff --git a/charts/kubezero-network/README.md b/charts/kubezero-network/README.md index 852f2e7d..776bcb10 100644 --- a/charts/kubezero-network/README.md +++ b/charts/kubezero-network/README.md @@ -1,6 +1,6 @@ # kubezero-network -![Version: 0.5.7](https://img.shields.io/badge/Version-0.5.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.5.8](https://img.shields.io/badge/Version-0.5.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero umbrella chart for all things network @@ -14,13 +14,13 @@ KubeZero umbrella chart for all things network ## Requirements -Kubernetes: `>= 1.29.0-0` +Kubernetes: `>= 1.30.0-0` | Repository | Name | Version | |------------|------|---------| -| https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 | -| https://haproxytech.github.io/helm-charts | haproxy | 1.23.0 | -| https://helm.cilium.io/ | cilium | 1.16.6 | +| https://cdn.zero-downtime.net/charts/ | kubezero-lib | 0.2.1 | +| https://haproxytech.github.io/helm-charts | haproxy | 1.24.0 | +| https://helm.cilium.io/ | cilium | 1.17.3 | | https://metallb.github.io/metallb | metallb | 0.14.9 | ## Values @@ -116,5 +116,5 @@ Kubernetes: `>= 1.29.0-0` | multus.defaultNetworks | list | `[]` | | | multus.enabled | bool | `false` | | | multus.image.repository | string | `"ghcr.io/k8snetworkplumbingwg/multus-cni"` | | -| multus.image.tag | string | `"v3.9.3"` | | +| multus.image.tag | string | `"v4.2.0"` | | | multus.readinessindicatorfile | string | `"/etc/cni/net.d/05-cilium.conflist"` | | diff --git a/charts/kubezero-network/templates/cilium-grafana-dashboards.yaml b/charts/kubezero-network/templates/cilium-grafana-dashboards.yaml index 358e8c67..9dae4b7d 100644 --- a/charts/kubezero-network/templates/cilium-grafana-dashboards.yaml +++ b/charts/kubezero-network/templates/cilium-grafana-dashboards.yaml @@ -11,7 +11,7 @@ metadata: k8s-sidecar-target-directory: KubeZero binaryData: cilium-agents.json.gz: - H4sIAAAAAAAC/+2da5PaSJaGv/evyKFnN1y92EYU14mYD3bZ3eOYdk1F2+3Z3W4HkUgJpbCQaF3qMjW1v31PpgQIJFFQICHBOx+mXSB0OZnKfJ9zTp58+I6xGrdtx+e+6dhe7S/sgT6iDy3T8+mv39RfLPpUfTMMTMv/YNOXWn3xqcF97jmBqwv6onblOhPhX4vAq8WOETYfWvJ73w1E7PNr00j51NQd+8KxHFee0B0P+YtGnTU1jf6v3a4z7Sx+aptP1IXfLJ6F/Sd7YwnXX7oF/36qjjO4dz10uGvUou8e1X+/0v8/ysNrhvB015zKM8nj382OZyPHZRemZQYT9uLa96feX16/1tXfr0zn9Rmj53ZNPbxoTRimv/LMtZHp6dz6H8HdTz53/Y+O7V/T9w315djl0+vPjmP55nRm4ZppzP9pmfY32Uq/fVV/TrktLG/eTrNWero16C6EZVw49sgczxs9/KkY8cDyvaVP6XM9aor4h/TxxFFtR3diCd8XL3WLe56p12JHPdaXzhN4vjNJnojfmd5bxzWE++nauaXvR9zyRD150IWwfeEK43+F66w7TN7vx+jufHHn11IO+pkPhSUPSPvyyuK6mNDV5AE88J3Vg4bcfWOZYzs6ppH8+p+m4V//yHVfma7xqrNyiOHy20/+veogsm3F6iVGpmX9Y8p107+XfWD1EtRdDJOuPntO20meQr5dP7opJpfdSYyFbaRbUb4t846Y/v2N+a/Zd0tfPa7cgml79CJeBpbqq2ktJp/9g2zXqWPx2UsnP+RuLeVQZdflEUh9N3VM2/9k/ksao73ynXzrxDsa1lxzGESXSBhkNj5El177VB710yt5QU+ZXtyIxL169IKue2zP5/o30x6n3crYdYKpGtNqCcNP4u299ib9a1fQjVqGN+tnD1knc0ajlXNlvsTLw9DiTHw6padJ+WZxG9lDCB96jhX4iQ7s+WLqxeai2f9WHyQ2SJHxhLATdpNdlluBvJpNrbLy7WN9iwvQALTu9L3G6smX/v6aadnANtV440wni8aIHVJzqJu59EqvWmTVqNzXr4Wb1t5qOqkN7y/lnJnoWs50pgRqdCHniXdg6jpTerVNsWEDhdcOrbjGfMlfqrHwThhzRfB9r98YNZopZ4l1KfWTWuKIx+c1Tg4mvuWuLd//shpZ19q61s3byN+tfjq3AL3IpnHlLI8atevl4b0m5YLWjH1wtzwd1+Tk2UicW9mpO5+TY+0Su9Z8ilxWQ9zSU4Y5w/SmFr+fTcdKRC8dMI1riqHjSym0dICcVH6eXVJqxtRBYDExL93VrEUmJODMlfM6rp+cMR4TNplawdi0vwjXiyZhTXt1/kpbCEfSrGPhL/fEpZvYgAQiZbzQZ7r87/L3d1PVA71g8sLlvngRauyBGpO8QfTeeAOf5L718K3nDWju+evvtfCw32t1NnWMv/7f77U/039/rz3+pk2+np2x4T17QR/UmUXTtXXGfmCdxvKFSd/Ti61EozkRAxIu8r1bOsSUQoXeormw05YnR9V8P87P8/CgLvb4uHwWl9vjJPbIL8TogxFO/ImG+rpoB9MPZeN7ZRGinX9GNom1VaRm5JNED/JdrLmBC8eNC+dt4AJwAbhwArhA+lCXw8qhkYHflFjLjkac+my1gWHC73IxsJpmX8n5462wnNvPzjpr1yamXXvG67B0rcXAvOZKjfIYfvWR92r4fRsjF6Ja+mANUvWBVAdBqgiZqKe+MOPMRD1TF5430KcB8Yzu2MZzuUkCk9bInZjku/Y0EtX3YSyasCpuLDnlphvr7b6NRZNP1XsWTZ/pxrrYCLYvrn5lv3p8LBiN9cyOuy02pm0SMxafeiIBUtlDs7YyNDdbTzi72ukjs9bRFsPSPHaY8pw/CZueQE8+nEuwDR8CfAjwIcCHAB/CUfkQhvf+bMo6oAfhzZefBjem6wfcGkzExHHvBys3VjK3QrvR67SqHod8Qw0sVc2X0PLso7J8aW0+6hv9Ub/iNv/45r+r1dNFm+sNUXWr87uq9PJ2r9XqdmHv/fsvP5r22ps6fnfmUxY4eu9m7wmC7qQTdBO+zUP7Nld8T2lT6Abup7O8XU0pb1iBPs2KGClDdxboz6xKb0pOo7v4MZdPxd4q3Ym8Ifj8sMwAPj/4/ODzK9TnJz0h1HB0MdtfmnkHeaW7VMcdkgfr9DZknRZY54CskyLjU1+SXOVpQrVsrfC//JR+2xvo17xEfgXNuH6ITDflu72bMknfxZtyU4N9uMwyWGZ20fuN2OmX6KwsPCsjdeUHoCfQE+gJ9AR6OiF6ImO5JVhzYVkqE7HUgeOeaFaRlLTOZqjU7wGVDohKcpXwijClzm4PRkYptOjiDS0oApQi1MtkD7q913ZiyXd+cZ4Uki5V97gZrzNHEURcqt7B79aZ491GkPYPeiBGwlywWSUvx82F0VYKhb29+nHGhYFaJ2DajH7BaLA2XcG8exJKE/pQtwKDVBjTncmUBlPSpzRCEFJym35kyFURYZGxV6BB0CBoEDQIGkQsDbG04gmxs22dKU3LWHvXBSOWJJw2nI7obZgmsr3oDdk4gMH+i8XORq/OeKfTba+qo6q5iTlqv6E7Zar4KJK93DhPTkGLbRwlTLTYYQKEx95ipAPV2Jpna364TGnNHaOXnxSBvrylJ2OrsFoAH/9IdMSiakUSj51ReBfUU+qMPongl/5mdlzqgH5Bv6Bf0G9R9CsnzdXxGUT8vKp06s+nuXifUJYoVZVJZX0kOR6SypbVImk84XmBKzZQc5sm2q2qqtouxXYjrcJm91nO6j9aN6Ozt9ublf95c/UBpX8g3iDeIN4QujiFRLa8wxZf1J2uiVukPL2j+uPYTwsLSIsEunCj7JoPnhqfn11dYK/VDuZD4LYxESuu+1JDIYshMvOQrFFyy1BJvdzNLYc5NPf+mruYyFgvS5Y2d2Kwle4p+FKl5rWEFu7Ol41orjm+9hE3i9d85WMZWeZTc548FrnPw/KvXjDZtPjr6y1OqzuB7T9jnxf5aPK/U+5fn+W/00t4ucdH9vAgr5jY82U72iQMY7pMHiWVKmydnolH9TJkftwZlhyB1EBqIDWQGkgNpAZSA6mB1PYVLstEtfN9otpyRipIbfd8OZBaSUiNGgOUBkoDpYHSQGmnRmnOFJwG4Q5OA6flG1FrtrIwrYOI2rFE1KoKVGwnovqezZjKQ9gLQAWgAlABqABUACoobAAVgCqvwFc2UXUR+DqSwBeACtEpwBRgCjAFmAJMAaagrgFTgKn9R6fOG1ks1Ud0CtGpBUzVmSv8wLXpZHEkyY2sYlcjqnqRhlpnO6cDhhdh8iKIYAG6AF2ALkAXoAvQBRUO6AJ05RbByqauHqir1DuYgbr2TV1kVTbffO2snMUazzO2FOw2N6vVGO4TtUW5xo1IcR9VWrc3RVbdytYTRVr18IVfev8sbo8DWdT8L2q0kd31Lm2QITC4DCZDetNTq7nLYz6atvmRTxNzQNxI9F760XAWu8hiKOLuN8O5nY+Z24w0seKjKV1YPhV8BsfiM6Djvad8BnAawGmw7DSgSdaC2wBuA7gN4DaA2wBug6V9zDeL1WYVfGntfSVhHbmweTgSDNNfaF09sX9yVnhX7rzg3UtZZA2MwFWya2c3g3xl1ZkKcCvMr5Uo4UIAKFLez+0TZ6Ndm7EYEVgGLAOWlR7LDKGbdGlvteOD18Br4DXwGnitcryWDPNmAlsbwLYO2OSGtGVevHjEQLYbd2HNItJnkT4L5EL6LPY9AFWBqkBVoKr1VLX1isVWJwuqGoCqKi9iXAtV22yEwIBqKagWA7X4XnVj8Bp4DbwGXgOvgdeSvJbrMqhsKdeDlKtycT9IueKlHFzvkHKQcpBykHJIaYLzHd7YvXhje084Y9vNLAXf3EnBrxXkFo3Ka/T40PH9laY7OUHuO9NvL9p1luZjpWF3QOPFwHd8bm2imNsr9Qno9zafiLh2LqLUdnRZVYtgX0qaTkpPgQUDUNBQ0FDQUNBQ0FDQUNB5Kujexgq6CwVdAgWd5tqGgk5R0PA/Qz1DPUM9Qz1DPUM9Qz0j+RuwtCssaZ2NaekctFSi0sgbUZIT+Dqdlj4acdN6HjZVnZpk9WP59IH7rOLH5a0T3M5a+95rok4wq327oZHfFagVjKJUKEoFTF7BZBSlAkGDoEHQIGgQ9Hpd3t52+XS7n6XLW1hzU6WaVHPkjlBiMGfTGXKvX/ji6fSDOiMdWwxHq+spiA4vuXPlqcXz7r6JENgKbAW2AluBrcBWYCuwFdgKbPU0WyUKGmTDFXZoqWbBX8AVMjuR2YnMTmAVMjtRJArkBCkNcgI57T0q1WllgRMqwR1F4YkUjkotDLceq0K4ifDqjL1+1mXCSnHbXKgAgJthG5uz3E4AFyvwi7oW4DfwG/gN/AZ+A79B0IPfwG/7j3xlAlwfu7IcRd0TANxhAA7BN8Ab4A3wVsqcRuAbUhcBcAA4AFzVd9XsZGUu9lHIvUybZM74iHQGSY7BH4EIxFZgVHRm4sZc9HYjLnqvnpu5QhfmjTB2DnCVt4BKt53+Rj5VF/Y06qdcqNeB2cK/ddxvzLTDvit/hpIqIGQQMggZ4c0T4OPptOhdTBPzdMYmpj0NwrlEFQlJH9xy1wg18uaV21fUs2GS8ixIOc+vtWMo4cfwyUksX9EoKk2MSAJ0EnQSdBJ00snopOH04Lu9ZwklTYNSKqNSGt77wtuDUmI/sF4V1dJnl49GJD2glqCW5Ph13oZaglqCWjoBtUTGcv288y4u+WRtHL5Gw9uNYMyc3rRqecTIlTW3jYyPzDthXMwa4vsGbze4VkuNf8+6lvpJ7XAZD9sYmtFL/fLzxVVpDT7qG/1R/2gM3imtoTtay+ifH4+hWZl7da/REcPuERm77MPIEfXui8+MRLjEsXIPKENujEatipv8nSDSJDheNnp59clwpA0bjSMxeqkt3es3Ro3mEVm69GP4sVm8g1EkT0v/3GWGsE2ytSv+CITno2PnbO7R3Jdcdot3xbDZMSpucX4zLq2BRYPAveqAY4RjdXmhRhgN0TgOI7OVde2lMnSrRQOGXnFDl9nAnWZftKvuVp2YNgxcxR4cLp6T0c+3wnJuP69dvZls5s2COUvXWoQVn71w8Bh6dj7GKGQBYS9jZ/Z2B8k9h9zIwLRneT3yh1PuXw90x7Z9l+vfBmN9EHnxUjN8PJ/7gUcfcelolZ+M+MS07ukT6SBRSUCu4zs0ItNHny+ukmlBYUZQ+LN6eL7np04nckW2TROS7+3TyUH1fS3cPBrDb2heCWBPL+6s72uDjlMzr5zu0817Uc7eG6FMZQw849t0I78rZx+uqJFZdm9+v1Gi5oermxa7mJlLRbeRqRk7CJmai6+RqYlMTWRqJgcpZGoiU5MhUzMfgyNTE7lsyNQsvbGRqcmQqYlMTeRYIVMTmZqw+EYWR6ZmrpZGpiYyNbMsjkxNZGoiU7NMRkamJjI1kakJAyNTE5maOxojl0zNRCG2rFRNrdlDruZx5mp2kKt5mFzNDnI188zV7CBXM99czQ5yNfPP1eycQK5mB7mayNWMmgm5msjVRK4mcjWRq5nh80KuJnI1d8pqQzZbQYZGriZyNZGriVxN5GpWOZ8NuZqFWRq5mgVbHLmauVoauZrI1cyyOHI1kauJXM0yGRm5msjVRK4mDIxcTeRq7miMYqpq9rJSNftI1TyNspqRvwTpmocorVmM8U+6vGapTHysJTZLZeRjLrNZRkPvu9TmZeTBRwpn7CCkcC6+RgonUjiRwpkcpJDCiRROhhTOfAyOFE4kuSGFs/TGRgonQwonUjiRfIUUTqRwwuIbWRwpnLlaGimcSOHMsjhSOJHCiRTOMhkZKZxI4UQKJwyMFE6kcO5ojILKbWblcJ43kMN5GuU2kcO5avwCS24ih/O0THyspTeRw3nCht53CU7kcCKHM57D2UAKJ1I4kcKZyaFI4dyvo6DUEfu21m/rouI+sHLnEBraqJN7IPM5/p32tkt0+610906vu5N3Z6VDCG4npivu+ZeOmvxWv1oViyzhmVvrPPL50BJrvEeuOb72T9t55AWTGZCY0wE3DJqFvAz+SIMI+vcMQs5+t/OGiIeH8FKPjxv4d7IZ4o1lOTqXNPJm9sAACACEAggNBAGCAEGAIIoiCCOYTAdqHnSDqe+xuTuw3NnAgndancqn4qyzfXmpo93TWg29ktSRiCpnYUcXhYFKwgUpgQr13tBsQxcc+I7Pre14gbuCz6ihzmx6S8/yJwd5mcdH9vAgr67+sReWeBdZJxaTeEfWYb8o6wArgBXACmAFsOJ0sMKZehtARa6u3H5Gpl6jA1F1QFEl02xc7ouZsqKh9cbUhTegAcjeQkr9pk2+nsX1lC7bsgANFV5oR8X0KXxq9uuUrAfPKyQSJBIkEiQSPK/5iqSk5ylDJfXgeSqJ5ymwXcH1axnFH9j0JFuFpXOXQ7G7Y+ruilpzkGGha8Et/3pA96gmivIaK7xRNr/RDRYSZKtJ6XYTJEtvSESwv6kzQ1FCUUJRQlFCUcLpVpzTTWtkRDLPoScPrSfjTjfDdaYD3QlsP/K3GaYrlF+LhNL7n355/+mTFEpbeeFI23iFOODCC+0asiQLTIXB3o9l7iO7ojEVoUqoJpTBh2qCajox1TQ5fBH8Ehd103V+XvmykB/LWwbLHQ9faJ1unWntBv2fpp3B1qm23q5g08fTK9j0sVIFm/Kuk2kYkl/yMIiaOdb1vZXQzqFNEZadgDVCawQq5QLWWLwmciGg9VKGbyphkWJy8jNdWf1zuLJKlD+moo4DblmzBDJX6MK8EcazM8nUeQaST+ssvK8z9gNbWeeTi2dLXfkzXVil5ofXzvRzbRYavCTzsPfKMnBuwbmFkCCcW3BunY5za1iCkGBWIr4GIVW6mODw3hfe3mOCJJ961YwLfnb5aGTqkE6QTogLQjpBOp2OdCrJ9th0KT4W7FIlV5e3EMqwd96vftyq5GbWtbaudWHmfQYJ0+7lJEKFuTdCGXZ4eY7fP4NX+xpw9cB+/4TL3w4mpVrhszxbF7QUKrb3TXktsxhwit8xpcRWmc+FT29ysj7G84zozkYOiH34DLWVQbjZespnqGXEXpu99WOwHo5rS93G4vY4oJdS+UgsLlvoLq3gMc1Wl8FkSANaGiqqY6gTmx/5NEHccSPpDl0gHLZjF1mMuNz9Zji380SdbQbUWXtfOZap36c0uHwweKMQyIM3Ct4oeKOOyhvlij82CuXlvt27IWyzAnu9H0VpXjI3ac5bTrNg+S3eFcNmx6j8vuSyd5fWxsORNsx/LUYhW61oWjtd47eRFnAQP4th+guBrEAqK22A8PiFG1hCbbR4d69SJc/YUjLBVCHKwOpumYWZf5bA4pYfH9lL9vAgnySRNEDMKKL23iWbIHUAzyGXgCZ/3TWnM6Hc7/vXjMYonfq1aQnmjNi7y09ha81uQ/6lC2Ize8xIYgtbv69Lv4eUqMDCaicptICF6tNMxJH2UoMZDVNPzPYlxcu0jga6rBJdbpuOcOhSgZmCTWue73t3vE03t4Oe21nPXZMJHZogJoM/Aq7kwovGq36/ziKh5+mOXAhjJRSe0lHBlAY4QT92xdS6J4WmO7bhDYaBLKfyMNuHm3Sg3LHb9jZKIKVLy2v+iY5Q4vEzySMpENtSIJ6xInbYUzewvSjcbElOUopF+gtZphBw8OvDrw/ldTp+/RIs0Gk2shbotOCJq8oCnQ+Xp7ZC54ONJToQTxBPEE8QTyh4fDCvWJZ6Ou9CPFWl4vF24qld0ZLHF+qBmLSEN1NPBQQj39vGS995SU/EjMBV8xbzHUZGtu6Zaeuu6rfcYvS+2r7cwSKs0OXJo+jULIwly32pfdexGJcnoptg1Odt4b2CXKt2sBIrqiHXINdOQa4dPn+Vl7jOcqc1bOi9iudPKoVVWhP3ud7q8YqbeFLeUtbft7V+WxdVN3BOZZT3MkgIoyEa6MGpBt5u7f/k9AqET1AgfEkLsNymqyeLHi/tOlGCF7IypihmcUIzK9etgZholbPZrIyFCjE/1CB0Pw1mzqpZMtumnsHc3YH9/n/sslqhfrB2aB9VM7Qb/5FPfmC4vJ99mLlDfw03LPg5vyTBFY/t52vB5EmZz78JW65gYba4ZfSfsE0M+tO/ddxvM9es77ChUB5duTRUOncti/HRSOjy4OQex/DUwlMLTy08tfDUwlObSWdhJa3Z9F9aj4zR6Z1X3uU1Uu9GaZ1eWsvon1ff6VX+znw0lqZpdjwWbnkjEZ3ueaNZfUd5+bt0u9dqdbtHYOmyd+mRLriO4NqeQhPpzX0SIYpce3oljZKbPKvq3ozZ7vkm3PPH6J6fTK3QL6xcwYagZoFfvkC//D7sT7dRcZf8G5Wz/DNW65+QX5yO955acIYVZ3CMwzF+Eo7xUmwKRVLzGnVg87SwCFdHlba6cZvrjaoHHUy75EZudNvNqnfkp6cilDPewCfReSph8DxrGfA+a+ORivcvHaXY9lUizzXH134OTgtZV9YS1VkrvMy7s5Ql+gdRamitAalCP/A23j8o9tPtXQ+SDlK3oSHN7nPVdAmmncGystiuK45jd5+57Hgzdn4/S/+aZYvFTs1CmwKlkWKGFDOQNEgaJI0FwXN13DWGfIiAel4GdsdD/qLXr5N4b9ZZt1On+avdOsPq1dx6dHPU7rYFevR+UkSwevXYd6te9Tp0NvU6xEJSyIM4gEchtiVzWCef1JSqN+aVavth+UYVtEl1bPvuMltE6sHiN6gudR+h6W6Xzamv1E4Hv8yerZiCcwqT54XjaFQLrKjwnFzWKHe58vlYvGLSE+OpInNyy6yXiy2zUFUOjiQ4ko7KkUQDmgVX0lG6kuBGWgPdo77RH/UB3bkZuNfoiGEXbiMYGG4juI1K4DbaYAFNpuNIQ9n6Y9ytcZ7Q4kpjiqikUugj8Lnvre7cqM4j/Q3KNRHSb7TS99+zc4WLIi4WdP1vT1hCElzyG3WaxMe/1+opro4wJ4Q+8QJd7lBNHx1kPc9gkEDZvW0MHi0oidmCyW0ukQgD/wX8F/BfIBEGiTDFejDKDNijpt6v/IqSWTDiWioIRu3quN6JJ+YXY3GYGs4NODeO17mRXIqTURyk1y73Uhwkziwnzqjhe8NciGNOlymdHQ6UJFO+/rBRakw9nyVqoZCMNrN80ih15gS+TtegQ0bctApNIUqVvumGe7e594zuil0Jl13GnaUb+8028ortY0tZbWX+araemr9aGb757hNLSfVwuF/qZRaZPODj0HFncdlwd2nLPWmivwwmQxrnU1cdymM+mrb5kU8TbqC4keQendFsFrvIYiLi7jfDuZ1Llm3mmcRywpQ2l88GLykq78BNijQvOEpPMc1ri4m5vzVYtrT0ibm9T7CcCG6DKPdHlJKk0qPmjTrbJUSeprlV2PxP9JGS5dmVEKXyVscWsHe8us6OW8fPU+rjtgmT6V/0G/41W2TOnyGGDXUGdQZ1BnUGdZajOkvmNGbJM02DPquiPutDn+1Fn/Whz6DPoM820GdNJBkiyfAo1dnSHusHSjGUcUekYeVt4delXsnXb4wqv+1dtCyltEZutbpiqB+HkUvdm89Hnd75eRXKVz8d22hnJB20+vtm5/pqZ7krPU5XrZj1k9jsbZo39PjbecP7qoj4RZRB9BwkntWsTqtnvSUuR3eRBzCjDjXQGGhcfTReXZyLyAXYeF3kohQL8Ejq6I5tC92X3RQbxOdoaldw477cVam1Oms2G3XWb6uq1M2qV6W+5abs1y995+VccZV3X6IGbze4VkmwS4ZFs8iurZ3ocqiKbk00BzkZ7RRbbEMU/1kB8czlC+7IaRfq2djsnGq/oONa7dHOWK3YbWG1B7U/XcJ1LEu4WO+BqjjnoHIkFCJgjYB1MUj+IzetVY2PkPX+bVxmh8dRRKx/CezyhqvbWr+tiyOJonYyxHwDUdQqJSWvBlH1BYYM3GCb8Kk2+XpWXKUF9aIXVKIk5pqIm2cUDuhlqtMRzeO7bGK8nkMRJEaQGDgKHAWOHm2M+PDx4aj8OpMTbOAKEFMh1i57sm+70eu0mkeQt44unXfWA7EBBo/CLF32gaMrhs3KF9XG6FxkIkmGc6vTK8C5tfqRiUX4Rfq7ZnpovsiefrG1+6vOwq19zthrtv0VdSew/edfs4CF/OpCuya8zO3A3gXPXpZQ4nyXTkaVjk4D+S6s9veA7tIWvvCYfJpx2AOQ+nLyqS8afI3wNcLXCF/jHnyNX9SdruGqlKd3VH8c+2mBemmRQBdhOSfL+uCp8fnZW4bsdQuT+RC4LclZcfmfCnCLITLzkKxRslCKzr25VdY/mntvzf0cnu9unavSaWaUy+uc57cFsM+Hq0svlmDcNcfXPgrixclYEq9uSalH4GsOSCMJW7+PCt9tT+Ovtz33c7lbPqT875T71wXQd3i5x0f28CCvuCOGkzXormjGZZFN2AtOfxFrMpu6FIrogcvAZeAycBm4DFwGLgOXgcueyWXJOGs2mLUBZofcm5TfAcxKCGbULoAyZOYjMx9UhvptJ7lUHGQGqQ4yA5ltRWa97SNm/Sww65VoefdTJId812S+K3DrSdz6nrqr5bEXZDpG/1LE5QG5gFxALiAXkAvIBeQCcgG5gFy7IFdKMCyTuTr7ZK498hUiZU/hlSKJZ5bQWvCUK/zAtQf6UuSnCKyKXXhvdAW4AlwBrgBXgCvAFeAKahtwBbjaEq46W8ezut0Mtuo29x3PQq2W/OBqXjJiQEqERMnAFbowb4SxBrHEHwG36BNpL/k3vRmmEft7BcHaKyVVdHpB64z0LT1yAfAVXkjBl7r0jtj1RT5snf0qd9sTnsfde/b3nsfeK+uxXyLrgb5AX6Av0BfoC7vBgr/AX+Av8NdO/JUMbmUDWAsAdpoAprru0RPYB/sGDAYGA4OBwcBgYDBEwCDJQWBo7rzTCxMRsN55FoC1AWCnAWBz4NohBBb9uLKRsEswGBgMDAYGA4OBwcBgEOVgMDR3YUu8siHsHBB2ohB2YmEwABgADAAGAAOAAcAWADbZAMBy3SKon7FXp9bo5VeJGtpr+y2CEtorW3Ip3UKfXKhPLoV/67jfrhzL1O+fXnsfCS32A+s0ClRb9LwZOuvtZpvqJp81ElnQVtBW0FbQVtBW0Fb7dW5f8slaZ2dNdwVN4Wpuy8MPqUy5rfdxZN4J42LWCt93G8YwtSVi/Ur9pHY4r/KTdjYETZ/ltrNotoxWs+J2DqZG2fuzaIz6RjdvOxe0TVAmnGl9wFnF4ew0sAxABiADkAHIAGQAMgAZgAxABiCrDJAlg2W9rGAZeKzqPHYlsetIKYyeDewF9gJ7gb3AXmAvsBfYC+wF9qpYMCwLvppNwFfVg2H02McbA6OHA34Bv4BfwC/gF/DrEOs8mo12VioR1nlUXT19ImFj6qUVUDuJp+jZoJ+gn6CfoJ+gn6CfitBPCfdTtoBqNiCgKi6g3tuGGi6PUkHNHs6DhoKGgoaChoKGgoY6iA9Ka2ZJKA0SquoRPE4agqaW4/RCzZ9ucw31XXSi2tQVlsOXZkV5eTl8yR+eN8Kz0YRyLSZ80eot9f5QW48XDR0ZPro8TRJqdV1tfi1fTKZywotNAeELsOgpsddKD1w3fA0e4h3btHUrMMQby0pOMzVrpnFSO1XNllkT9O27T4OrX/7x8f3nv73/9VP8gMX7HX9ta38Ewr1Xyi/1tAt7aUufjsXdit6aN0jsFVh9wWIm4Jb1JRphI9u++iF+tpiJlmYfqQDlXLpSjXQ+XNf+PBjQudNH3A3eTimcTRq5Z9JFWn2gTu7N3tKbsKeoty1eF3G5/ZYLiM7bh36zTatsdf1FYzWfaqxoRNRS2i+89nfxV1NOTo9hPzcnC+VRG4VylMbV25fa9ezl8J3os9rSz6am/i2W+TO728FsYImPqjWtEXvVz+N/aIs5o9aO/VuL/3HeiH9zvfh3M/ZvLcq+WX66f8k5QmYZ+Xot+nCp0hH7KEh56tHYEYRJQzd+8E/fCCb/Cj+9mY8lysC1WyG+faK5Q3Xd2neP/w/nhB+pqyYEAA== + H4sIAAAAAAAC/+2da3PbRpaGv+dX9DCzW1aWtgmK11TNB0d2Mq6JNarY8exu4mI1gSaFMggwuOgSjfa37+kGSIIEQJEiAQLkOx8mFgnicrrR/T7nnD798A1jNW7bjs9907G92vfsgT6iDy3T8+mv39RfLPpUfTMMTMt/b9OXWn3xqcF97jmBqwv6onblOhPhX4vAq8WOETYfWvJ73w1E7PNr00j51NQd+8KxHFee0B0P+YtGnTU1jf6v3a4z7Sx+aptP1IXfLJ6F/Sd7YwnXX7oF/36qjjO4dz10uGvUou8e1X+/0P8/ysNrhvB015zKM8nj386OZyPHZRemZQYT9uLa96fe969f6+rvV6bz+ozRc7umHl60JgzTX3nm2sj0dG79j+DuR5+7/gfH9q/p+4b6cuzy6fUnx7F8czqzcM005v+0TPurbKXfvqg/p9wWljdvp1krPd0adBfCMi4ce2SO540e/lSMeGD53tKn9LkeNUX8Q/p44qi2ozuxhO+Ll7rFPc/Ua7GjHutL5wk835kkT8TvTO8HxzWE+/HauaXvR9zyRD150IWwfeEK43+F66w7TN7vh+jufHHn11IO+pkPhSUPSPvyyuK6mNDV5AE88J3Vg4bcfWOZYzs6ppH8+l+m4V//yHVfma7xqrNyiOHy24/+veogsm3F6iVGpmX9c8p107+XfWD1EtRdDJOuPntO20meQr5dP7opJpfdSYyFbaRbUb4t846Y/v2N+efsu6WvHlduwbQ9ehEvA0v11bQWk8/+Xrbr1LH47KWTH3K3lnKosuvyCKS+mzqm7X80/5TGaK98J9868ZaGNdccBtElEgaZjQ/Rpdc+lUf99Epe0FOmFzcica8evaDrHtvzuf7VtMdptzJ2nWCqxrRawvCTeHuvvUn/2hV0o5bhzfrZQ9bJnNFo5VyZL/HyMLQ4E59O6WlSvlncRvYQwoeeYwV+ogN7vph6sblo9r/VB4kNUmQ8IeyE3WSX5VYgr2ZTq6x8+1jf4gI0AK07fa+xevKlv79kWjawTTXeONPJojFih9Qc6mYuvdKrFlk1Kvf1a+GmtbeaTmrD+0s5Zya6ljOdKYEaXch54h2Yus6UXm1TbNhA4bVDK64xX/KXaiy8E8ZcEXzb6zdGjWbKWWJdSv2kljji8XmNk4OJb7lry/e/rEbWtbaudfM28jern84tQC+yaVw5y6NG7Xp5eK9JuaA1Yx/cLU/HNTl5NhLnVnbqzufkWLvErjWfIpfVELf0lGHOML2pxe9n07ES0UsHTOOaYuj4UgotHSAnlZ9nl5SaMXUQWEzMS3c1a5EJCThz5byO6ydnjMeETaZWMDbtz8L1oklY016dv9IWwpE061j4yz1x6SY2IIFIGS/0mS7/u/z93VT1QC+YvHC5L16EGnugxiRvEL033sAnuW89fO15A5p7/vZ7LTzs91qdTR3jb//3e+2v9N/fa4+/aZMvZ2dseM9e0Ad1ZtF0bZ2x71insXxh0vf0YivRaE7EgISLfO+WDjGlUKG3aC7stOXJUTXfj/PzPDyoiz0+Lp/F5fY4iT3yCzF6b4QTf6KhvizawfRD2fhOWYRo51+RTWJtFakZ+STRg3wTa27gwnHjwnkbuABcAC6cAC6QPtTlsHJoZOA3JdayoxGnPlttYJjwu1wMrKbZV3L++EFYzu0nZ521axPTrj3jdVi61mJgXnOlRnkMv/rIezX8vo2RC1EtfbAGqfpAqoMgVYRM1FNfmHFmop6pC88b6NOAeEZ3bOO53CSBSWvkTkzyXXsaier7MBZNWBU3lpxy0431w76NRZNP1XsWTZ/pxrrYCLYvrn5lv3p8LBiN9cyOuy02pm0SMxafeiIBUtlDs7YyNDdbTzi72ukjs9bRFsPSPHaY8pw/CZueQE8+nEuwDR8CfAjwIcCHAB/CUfkQhvf+bMo6oAfhzeefBjem6wfcGkzExHHvBys3VjK3QrvR67SqHod8Qw0sVc3n0PLsg7J8aW0+6hv9Ub/iNv/w5r+r1dNFm+sNUXWr87uq9PJ2r9XqdmHv/fsvP5j22ps6fnfmUxY4eu9m7wmC7qQTdBO+zUP7Nld8T2lT6Abup7O8XU0pb1iBPs2KGClDdxboz6xKb0pOo7v4MZdPxX5QuhN5Q/D5YZkBfH7w+cHnV6jPT3pCqOHoYra/NPMO8kp3qY47JA/W6W3IOi2wzgFZJ0XGp74kucrThGrZWuF//in9tjfQr3mJ/Aqacf0QmW7Kt3s3ZZK+izflpgZ7f5llsMzsoncbsdMv0VlZeFZG6soPQE+gJ9AT6An0dEL0RMZyS7DmwrJUJmKpA8c90awiKWmdzVCp3wMqHRCV5CrhFWFKnd0ejIxSaNHFG1pQBChFqJfJHnR7r+3Eku/84jwpJF2q7nEzXmeOIoi4VL2D360zx9uNIO2f9ECMhLlgs0pejpsLo60UCvvh6scZFwZqnYBpM/oFo8HadAXz7kkoTehD3QoMUmFMdyZTGkxJn9IIQUjJbfqRIVdFhEXGXoEGQYOgQdAgaBCxNMTSiifEzrZ1pjQtY+1dF4xYknDacDqit2GayPaiN2TjAAb7LxY7G706451Ot72qjqrmJuao/YbulKnio0j2cuM8OQUttnGUMNFihwkQHnuLkQ5UY2uerfn+MqU1d4xeflQE+vKWnoytwmoBfPwj0RGLqhVJPHZG4V1QT6kz+iSCX/qb2XGpA/oF/YJ+Qb9F0a+cNFfHZxDx86rSqT+f5uJ9QlmiVFUmlfWR5HhIKltWi6TxhOcFrthAzW2aaLeqqmq7FNuNtAqb3Wc5q/9o3YzO3m5vVv7nzdV7lP6BeIN4g3hD6OIUEtnyDlt8Vne6Jm6R8vSO6o9jPy0sIC0S6MKNsmvee2p8fnZ1gb1WO5gPgdvGRKy47ksNhSyGyMxDskbJLUMl9XI3txzm0Nz7a+5iImO9LFna3InBVrqn4EuVmtcSWrg7Xzaiueb42kfcLF7zlY9lZJlPzXnyWOQ+D8u/esFk0+Kvr7c4re4Etv+MfV7ko8n/Trl/fZb/Ti/h5R4f2cODvGJiz5ftaJMwjOkyeZRUqrB1eiYe1cuQ+XFnWHIEUgOpgdRAaiA1kBpIDaQGUttXuCwT1c73iWrLGakgtd3z5UBqJSE1agxQGigNlAZKA6WdGqU5U3AahDs4DZyWb0St2crCtA4iascSUasqULGdiOpbNmMqD2EvABWACkAFoAJQAaigsAFUAKq8Al/ZRNVF4OtIAl8AKkSnAFOAKcAUYAowBZiCugZMAab2H506b2SxVB/RKUSnFjBVZ67wA9emk8WRJDeyil2NqOpFGmqd7ZwOGF6EyYsgggXoAnQBugBdgC5AF1Q4oAvQlVsEK5u6eqCuUu9gBuraN3WRVdl887WzchZrPM/YUrDb3KxWY7hP1BblGjcixX1Uad3eFFl1K1tPFGnVwxd+6f2zuD0OZFHz79VoI7vrXdogQ2BwGUyG9KanVnOXx3wwbfMDnybmgLiR6L30o+EsdpHFUMTdr4ZzOx8ztxlpYsVHU7qwfCr4DI7FZ0DHe0/5DOA0gNNg2WlAk6wFtwHcBnAbwG0AtwHcBkv7mG8Wq80q+NLa+0rCOnJh83AkGKa/0Lp6Yv/krPCu3HnBu5eyyBoYgatk185uBvnKqjMV4FaYXytRwoUAUKS8n9snzka7NmMxIrAMWAYsKz2WGUI36dLeascHr4HXwGvgNfBa5XgtGebNBLY2gG0dsMkNacu8ePGIgWw37sKaRaTPIn0WyIX0Wex7AKoCVYGqQFXrqWrrFYutThZUNQBVVV7EuBaqttkIgQHVUlAtBmrxverG4DXwGngNvAZeA68leS3XZVDZUq4HKVfl4n6QcsVLObjeIeUg5SDlIOWQ0gTnO7yxe/HG9p5wxrabWQq+uZOCXyvILRqV1+jxoeP7K013coLcd6ZfX7TrLM3HSsPugMaLge/43NpEMbdX6hPQ720+EXHtXESp7eiyqhbBvpQ0nZSeAgsGoKChoKGgoaChoKGgoaDzVNC9jRV0Fwq6BAo6zbUNBZ2ioOF/hnqGeoZ6hnqGeoZ6hnpG8jdgaVdY0job09I5aKlEpZE3oiQn8HU6LX004qb1PGyqOjXJ6sfy6QP3WcWPy1snuJ219r3XRJ1gVvt6QyO/K1ArGEWpUJQKmLyCyShKBYIGQYOgQdAg6PW6vL3t8ul2P0uXt7Dmpko1qebIHaHEYM6mM+Rev/DF0+kHdUY6thiOVtdTEB1ecufKU4vn3X0TIbAV2ApsBbYCW4GtwFZgK7AV2OpptkoUNMiGK+zQUs2Cv4ArZHYisxOZncAqZHaiSBTICVIa5ARy2ntUqtPKAidUgjuKwhMpHJVaGG49VoVwE+HVGXv9rMuEleK2uVABADfDNjZnuZ0ALlbgF3UtwG/gN/Ab+A38Bn6DoAe/gd/2H/nKBLg+dmU5ironALjDAByCb4A3wBvgrZQ5jcA3pC4C4ABwALiq76rZycpc7KOQe5k2yZzxEekMkhyDPwIRiK3AqOjMxI256IeNuOidem7mCl2YN8LYOcBV3gIq3Xb6G/lUXdjTqJ9yoV4HZgv/1nG/MtMO+678GUqqgJBByCBkhDdPgI+n06J3MU3M0xmbmPY0COcSVSQkfXDLXSPUyJtXbl9Rz4ZJyrMg5Ty/1o6hhB/DJyexfEWjqDQxIgnQSdBJ0EnQSSejk4bTg+/2niWUNA1KqYxKaXjvC28PSol9x3pVVEufXD4akfSAWoJakuPXeRtqCWoJaukE1BIZy/Xzzru45JO1cfgaDW83gjFzetOq5REjV9bcNjI+Mu+EcTFriG8bvN3gWi01/j3rWuontcNlPGxjaEYv9ctPF1elNfiob/RH/aMxeKe0hu5oLaN/fjyGZmXu1b1GRwy7R2Tssg8jR9S7Lz4xEuESx8o9oAy5MRq1Km7yt4JIk+B42ejl1SfDkTZsNI7E6KW2dK/fGDWaR2Tp0o/hx2bxDkaRPC39c5cZwjbJ1q74IxCej46ds7lHc19y2S3eFcNmx6i4xfnNuLQGFg0C96oDjhGO1eWFGmE0ROM4jMxW1rWXytCtFg0YesUNXWYDd5p90a66W3Vi2jBwFXtwuHhORj9/EJZz+2nt6s1kM28WzFm61iKs+OyFg8fQs/MxRiELCHsZO7O3O0juOeRGBqY9y+uRP5xy/3qgO7btu1z/Ohjrg8iLl5rh4/ncDzz6iEtHq/xkxCemdU+fSAeJSgJyHd+hEZk++nRxlUwLCjOCwp/Vw/M9P3U6kSuybZqQfG+fTg6q72vh5tEYfkPzSgB7enFnfV8bdJyaeeV0n27ei3L23ghlKmPgGd+mG/ltOftwRY3Msnvzu40SNd9f3bTYxcxcKrqNTM3YQcjUXHyNTE1kaiJTMzlIIVMTmZoMmZr5GByZmshlQ6Zm6Y2NTE2GTE1kaiLHCpmayNSExTeyODI1c7U0MjWRqZllcWRqIlMTmZplMjIyNZGpiUxNGBiZmsjU3NEYuWRqJgqxZaVqas0ecjWPM1ezg1zNw+RqdpCrmWeuZge5mvnmanaQq5l/rmbnBHI1O8jVRK5m1EzI1USuJnI1kauJXM0MnxdyNZGruVNWG7LZCjI0cjWRq4lcTeRqIlezyvlsyNUszNLI1SzY4sjVzNXSyNVErmaWxZGriVxN5GqWycjI1USuJnI1YWDkaiJXc0djFFNVs5eVqtlHquZplNWM/CVI1zxEac1ijH/S5TVLZeJjLbFZKiMfc5nNMhp636U2LyMPPlI4YwchhXPxNVI4kcKJFM7kIIUUTqRwMqRw5mNwpHAiyQ0pnKU3NlI4GVI4kcKJ5CukcCKFExbfyOJI4czV0kjhRApnlsWRwokUTqRwlsnISOFECidSOGFgpHAihXNHYxRUbjMrh/O8gRzO0yi3iRzOVeMXWHITOZynZeJjLb2JHM4TNvS+S3AihxM5nPEczgZSOJHCiRTOTA5FCud+HQWljti3tX5bFxX3gZU7h9DQRp3cA5nP8e+0t12i22+lu3d63Z28OysdQnA7MV1xz7901OS3+tWqWGQJz9xa55HPh5ZY4z1yzfG1f9rOIy+YzIDEnA64YdAs5GXwRxpE0L9nEHL2u503RDw8hJd6fNzAv5PNEG8sy9G5pJE3swcGQAAgFEBoIAgQBAgCBFEUQRjBZDpQ86AbTH2Pzd2B5c4GFrzT6lQ+FWed7ctLHe2e1mrolaSORFQ5Czu6KAxUEi5ICVSo94ZmG7rgwHd8bm3HC9wVfEYNdWbTW3qWPznIyzw+socHeXX1j72wxNvIOrGYxFuyDvtFWQdYAawAVgArgBWngxXO1NsAKnJ15fYzMvUaHYiqA4oqmWbjcl/MlBUNrTemLrwBDUD2FlLqN23y5Syup3TZlgVoqPBCOyqmj+FTs1+nZD14XiGRIJEgkSCR4HnNVyQlPU8ZKqkHz1NJPE+B7QquX8so/sCmJ9kqLJ27HIrdHVN3V9SagwwLXQtu+dcDukc1UZTXWOGNsvmNbrCQIFtNSrebIFl6QyKC/V2dGYoSihKKEooSihJOt+KcblojI5J5Dj15aD0Zd7oZrjMd6E5g+5G/zTBdofxaJJTe/fTLu48fpVDaygtH2sYrxAEXXmjXkCVZYCoM9m4scx/ZFY2pCFVCNaEMPlQTVNOJqabJ4Yvgl7iom67z88qXhfxQ3jJY7nj4Qut060xrN+j/NO0Mtk619XYFmz6cXsGmD5Uq2JR3nUzDkPySh0HUzLGu762Edg5tirDsBKwRWiNQKRewxuI1kQsBrZcyfFMJixSTk5/pyuqfw5VVovwxFXUccMuaJZC5QhfmjTCenUmmzjOQfFpn4X2dse/YyjqfXDxb6sqf6MIqNT+8dqafa7PQ4CWZh71TloFzC84thATh3IJz63ScW8MShASzEvE1CKnSxQSH977w9h4TJPnUq2Zc8JPLRyNTh3SCdEJcENIJ0ul0pFNJtsemS/GxYJcqubq8hVCGvfN+9eNWJTezrrV1rQsz7zNImHYvJxEqzL0RyrDDy3P8/hm82teAqwf2+ydc/nYwKdUKn+XZuqClULG9b8prmcWAU/yOKSW2ynwufHqTk/UxnmdEdzZyQOzDZ6itDMLN1lM+Qy0j9trsrR+D9XBcW+o2FrfHAb2UykdicdlCd2kFj2m2ugwmQxrQ0lBRHUOd2PzApwnijhtJd+gC4bAdu8hixOXuV8O5nSfqbDOgztr7yrFM/T6lweWDwRuFQB68UfBGwRt1VN4oV/yxUSgv9+3eDWGbFdjr/ShK85K5SXPecpoFy2/xrhg2O0bl9yWXvbu0Nh6OtGH+azEK2WpF09rpGr+NtICD+FkM018IZAVSWWkDhMcv3MASaqPFu3uVKnnGlpIJpgpRBlZ3yyzM/LMEFrf8+MhesocH+SSJpAFiRhG19y7ZBKkDeA65BDT56645nQnlft+/ZjRG6dSvTUswZ8TeXn4MW2t2G/IvXRCb2WNGElvY+n1d+j2kRAUWVjtJoQUsVJ9mIo60lxrMaJh6YrYvKV6mdTTQZZXoctt0hEOXCswUbFrzfN+74226uR303M567ppM6NAEMRn8EXAlF140XvX7dRYJPU935EIYK6HwlI4KpjTACfqxK6bWPSk03bENbzAMZDmVh9k+3KQD5Y7dtrdRAildWl7zL3SEEo+fSB5JgdiWAvGMFbHDnrqB7UXhZktyklIs0l/IMoWAg18ffn0or1Ipr+nh19A0G1lraFpwlpW5rt4GYidctkJHhGH9KMCyvI5mvhSnAAfZ/Fp5OcWi53wrn9P7nv18/vrnFqQPpA+kD6QPpE+ppE/xFYWTbqcs7XPehfQpkfRZjfjJsJp0+kRqZsuFw7HA4sEWDuckd7oFBADf2cZL33lJz8mMwFWzDfMdRua37plp667qytxi9Arbvtw1IqyK5cmj6NQsbE25F7TvOhbj8kR0E4xeA1t4ryCyqh0gxCpmiCyIrFPIGz18zigvcW3jTmvY0HsVz1lU/qbSmrjP9VaPV9zEk/KWj/62rfXbuqi6gXMqXbyXQUIYDdFAD0418Hbr7SenV5R7gqLcS1qA5TZdPVloeGmnhxK8kJUxRTELAppZ+WUNBDmrnEFmZSwOiPmhBqH7aTBzVs0SyDbwGYZZYXk7Cfv9/9jFO1g/WDu0j6oZ2o3/yCcnL3LSvp+5Q38NNwn4Ob/EvBWP7adrweRJmc+/CluuGmG2uGX0n7BNDPrTv3XcrzPXrO+woVAeXbkcUzp3LYvx0Ujo8uDkvsLw1MJTC08tPLXw1MJTm0lnYfWq2fRfWo+M0emdV97lNVLvRmmdXlrL6J9X3+lV/s58NJamaXY8Fm55IxGd7nmjWX1Hefm7dLvXanW7R2DpsnfpkS64juDankIT6c19EiGKXHt6JY2Smzyr6n6I2e75Jtzzx+ien0yt0C+sXMGGoGaBX75Av/w+7E+3UXGX/BuVs/wzVsifkF+cjveeWiaGdWJwjMMxfhKO8VJsxERS8xq1V/O0sFBbLpa3onCb642qBx1oSCq3kRvddrPqHfnpqQglhDfwSXSeShg8z1oZvM96dKTi/UtHKbZ9laVzzfG1n4PTQtZytUR1lg8v8+4sZYn+QZQaWmtAqtAPvI337In9dHvXg6SD1K1fSLP7XDVdgmlnsKwstus65NjdZ+5ivBk7v5ulf82yxWKnZqFNgdJIMUOKGUgaJA2SxoLguTruGkM+REA9LwO74yF/0evXSbw366zbqdP81W6dYfVqbj26OWp32wI9ej8pIli9euw7RK96HTqbeh1iISnkQRzAoxDbBjmsHkZqSpUy9Uq15a98owraGDq2ZXaZLSL1YPGbQpe6j9B0t8uG0Fdqd4FfZs9WTME5hcnzwnE0qgVWVHhOLmuUO0v5fCxeMemJ8VSROblN1cvFNlWoKgdHEhxJR+VIogHNgivpKF1JcCOtge5R3+iP+oDu3Azca3TEsAu3EQwMtxHcRiVwG22wgCbTcaShkv0x7pA4T2hxpTFFVFIp9BH43PdWd0tU55H+BuWaCOk3Wun779m5wkURFwu6/rcnLCEJLvmNOk3i499r9RRXR5gTQp94gS53haaPDrKeZzBIoOy+C/HHbMHk1pJIhIH/Av4L+C+QCINEmGI9GGUG7FFT71d+RcksGHEtFQSjdnVc78QT84uxOEwN5wacG8fr3EguxckoDtJrl3spDhJnlhNn1PC9YS7EMafLlM4OB0qSKV9/2Cg1pp7PErVQSG6+tbcT+Dpdgw4ZcdMqNIUoVfqmG+7t5t4zuit2JVx2GXeWbuw328grto9dZrWV+avZemr+amX45rtPLCXVw+F+qZdZZPKAj0PHncVlw92lLfekif4ymAxpnE9ddSiP+WDa5gc+TbiB4kaSe3RGs1nsIouJiLtfDed2Llm2mWcSywlT2lw+G7ykqLwDNynSvOAoPcU0ry0m5v7WYNnS0ifm9j7BciK4DaLcH1FKkkqPmjfqbJcQeZrmVmHzv9BHSpZnV0KUylsdW8CO8uo6mTUc3mxVw4HFbRMm07/oN/xrtsicP0MMG+oM6gzqDOoM6ixHdZbMacySZ5oGfVZFfdaHPtuLPutDn0GfQZ9toM+aSDJEkuFRqrOlPdYPlGIo445Iw8rbwq9LvZKv3xhVftu7aFlKaY3canXFUD8OI5e6N5+POr3z8yqUr346ttHOSDpo9ffNzvXVznJXepyuWjHrJ7HZ2zRv6PG384b3RRHxiyiD6DlIPKtZnVbPektcju4iD2BGHWqgMdC4+mi8ujgXkQuw8brIRSkW4JHU0R3bFrovuyk2iM/R1K7gxn25q1JrddZsNuqs31ZVqZtVr0p9y03Zr1/6zsu54irvvkQN3m5wrZJglwyLZpFdWzvR5VAV3ZpoDnIy2im22IYo/rMC4pnLF9yR0y7Us7HZOdV+Qce12qOdsVqx28JqD2p/uoTrWJZwsd4DVXHOQeVIKETAGgHrYpD8R25aqxofIev927jMDo+jiFj/EtjlDVe3tX5bF0cSRe1kiPkGoqhVSkpeDaLqCwwZuME24VNt8uWsuEoL6kUvqERJzDURN88oHNDLVKcjmsd32cR4PYciSIwgMXAUOAocPdoY8eHjw1H5dSYn2MAVIKZCrF32ZN92o9dpNY8gbx1dOu+sB2IDDB6FWbrsA0dXDJuVL6qN0bnIRJIM51anV4Bza/UjE4vwi/R3zfTQfJE9/WJr91edhVv7nLHXbPsr6k5g+8+/ZgEL+dWFdk14mduBvQ2evSyhxPkunYwqHZ0G8l1Y7R8B3aUtfOEx+TTjsAcg9eXkU180+Brha4SvEb7GPfgaP6s7XcNVKU/vqP449tMC9dIigS7Cck6W9d5T4/OztwzZ6xYm8yFwW5Kz4vI/FeAWQ2TmIVmjZKEUnXtzq6x/NPfemvs5PN/dOlel08wol9c5z28LYJ8PV5deLMG4a46vfRTEi5OxJF7dklKPwNcckEYStn4fFb7bnsZfb3vu53K3fEj53yn3rwug7/Byj4/s4UFecUcMJ2vQXdGMyyKbsBec/iLWZDZ1KRTRA5eBy8Bl4DJwGbgMXAYuA5c9k8uScdZsMGsDzA65Nym/A5iVEMyoXQBlyMxHZj6oDPXbTnKpOMgMUh1kBjLbisx620fM+llg1ivR8u6nSA75rsl8V+DWk7j1LXVXy2MvyHSM/qWIywNyAbmAXEAuIBeQC8gF5AJyAbl2Qa6UYFgmc3X2yVx75CtEyp7CK0USzyyhteApV/iBaw/0pchPEVgVu/De6ApwBbgCXAGuAFeAK8AV1DbgCnC1JVx1to5ndbsZbNVt7juehVot+cHVvGTEgJQIiZKBK3Rh3ghjDWKJPwJu0SfSXvJvejNMI/b3CoK1V0qq6PSC1hnpW3rkAuArvJCCL3XpHbHrs3zYOvtV7rYnPI+79+wfPY+9U9Zjv0TWA32BvkBfoC/QF3aDBX+Bv8Bf4K+d+CsZ3MoGsBYA7DQBTHXdoyew9/YNGAwMBgYDg4HBwGCIgEGSg8DQ3HmnFyYiYL3zLABrA8BOA8DmwLVDCCz6cWUjYZdgMDAYGAwMBgYDg4HBIMrBYGjuwpZ4ZUPYOSDsRCHsxMJgADAAGAAMAAYAA4AtAGyyAYDlukVQP2OvTq3Ry68SNbTX9lsEJbRXtuRSuoU+uVCfXAr/1nG/XjmWqd8/vfY+ElrsO9ZpFKi26HkzdNYPm22qm3zWSGRBW0FbQVtBW0FbQVvt17l9ySdrnZ013RU0hau5LQ8/pDLltt7HkXknjItZK3zbbRjD1JaI9Sv1k9rhvMpP2tkQNH2W286i2TJazYrbOZgaZe/PojHqG9287VzQNkGZcKb1AWcVh7PTwDIAGYAMQAYgA5AByABkADIAGYCsMkCWDJb1soJl4LGq89iVxK4jpTB6NrAX2AvsBfYCe4G9wF5gL7AX2KtiwbAs+Go2AV9VD4bRYx9vDIweDvgF/AJ+Ab+AX8CvQ6zzaDbaWalEWOdRdfX0kYSNqZdWQO0knqJng36CfoJ+gn6CfoJ+KkI/JdxP2QKq2YCAqriAemcbarg8SgU1ezgPGgoaChoKGgoaChrqID4orZkloTRIqKpH8DhpCJpajtMLNX+6zTXUN9GJalNXWA5fmhXl5eXwJX943gjPRhPKtZjwRau31PtDbT1eNHRk+OjyNEmo1XW1+bV8MZnKCS82BYQvwKKnxF4rPXDd8DV4iHds09atwBBvLCs5zdSsmcZJ7VQ1W2ZN0LdvPw6ufvnnh3ef/v7u14/xAxbvd/y1rf0RCPdeKb/U0y7spS19OhZ3K3pr3iCxV2D1BYuZgFvW52iEjWz76rv42WImWpp9pAKUc+lKNdL5cF3762BA504fcTd4O6VwNmnknkkXafWBOrk3e0tvwp6i3rZ4XcTl9lsuIDpvH/rNNq2y1fUXjdV8qrGiEVFLab/w2t/EX005OT2G/dycLJRHbRTKURpXb19q17OXw3eiz2pLP5ua+tdY5s/sbgezgSU+qta0RuxVP4//oS3mjFo79m8t/sd5I/7N9eLfzdi/tSj7Zvnp/pRzhMwy8vVa9OFSpSP2QZDy1KOxIwiThm784F++EUz+DD+9mY8lysC1WyG+fqS5Q3Xd2jeP/w+LlMpfHyYEAA== cilium-operator.json.gz: H4sIAAAAAAAC/+2cbW/bNhCAv/dXECowJEOGWW6SBgP2wU3TLkDTGknaDWgDgZZomQglaiTlxDX830dSb5Qlv9StEy/lF1s+UuTx7sh7RMmaPgPA8TwcJ6ngzh/gs/wNwFR/ypIYRkhKnddXXv/yw8XZ9V9nH6+cg6KYwAEiqjxhNEJihFJeFQaI+wwnAtNYVakKxCTRjQZQQE5T5qOqLCFpiOPzYFGjWfn7XK1+VUNXmMnPG1XXgXFMBVR9q1Fl43EI5qIcYzVKWTJIMRHnSk/3oJIaCtY7M+qgGA6IKhcsRYZ8hIMWKfZpfEoJZapBFg7gXucAdF1XfhwdHQB332y6MH6vGgv4BfQIYqKmQmVPPhpQyAInL5vp7xv5OdNGQQEWc9o6YYyENnecEpJJGExG15QSgRMp72ghVlVO9CHB8a0OlczSCYwRaQkdSDDkeqyVB7KCcai0fe4Ph3A4LHUtXTyA+owhJBxVwSTH9g7FoRgpH3VqctRWfYXnnCFGJDil8RCHdfUCNIQpEXWlpdxPuaCRks5K8czwAh0jxqTTM9M0RjXEhBTWLAVvGQwwikWtIJSt9Omc0dSwj4ze7pQZuobg3mxD/p6o3w0tZFQGKL5CDLcYTfu4e1jNbhSiOGhzXu08bRvGsmHMl0Twvk2K4xYpH9G75pQRMvZJS+0xJGk1iMZIZZTqUrM1LbzDQRZEptQI6GzqycnQpzgWF1RPYy2ogocm9ZWljLl3ZVMNhRLoo7YITpCM0ljAEDUckigNVIik3HR/Jm/6T7ogQAzptWVIqKj05drhH8wILfWuBbmetOp85bcDs0SF6ytE6N011eXSg7Xywt6mN2oTZEE/a7aTH1Ue4ovsyQX0bxu24QIlCQqkf5pmFpCFSCyxCrpPWK7sHmZQoD0fE5xGHpXOg4IyT6YqH3Hu+UnqcSQX+YB7OnCnmHp5ZZgkf35xilO+OLPPbnSzvw8GE7CX0GAf/CqH0ZmzOmURVPPKEThCXubHehUZCoiN9RRx2kveQF/ojOPWLa2n95uyh4YrGBpmibjnrHRoYSK5PjxhE6nVr91Er9Y3kZxaTzmK5heO0kSnzpLpLEYMyfWfBHMLsVL4DdN5twCUUn6JwnwVnjvhaoSHonmG0OTjnPY/go9crrdAGhHEaoGvKpXYMzXzEmQoaGYmTpmYy7o6KXkFkOE4wGMcpNKsjXRQ1NG0VSlwD+/xXF4ZpP5ttjyZ41FZNM9Nasgt3DhXuz27llm0JWNN4P2yXFFFVZ7C5hby/NqgroYqoOEryFEjkjJUaFTPWKEhNoazcuZVmsqz2A7o2Qj/SdPtMkWGbZSk5e/QuFS6Bvu5CVZT+EXvH09OOYkDsfAiFFE28QYTgbiXjc95jo6g30E/GZ+72+TzmmBjQD9eDugqPHr8Or/Oa0TPFvmd4XAkrrLrXsv2j8j2j0vJCgEXkU3rirOMbvYfj2R6n96267sG36yBgE/BRMuTSLuZXn+DmeT11pMw0/n7RWZaeM11tiO0fJnrDTK9gVw4hJmnLTFvQswt64jl5V3gZZ8SAhOug3cJstbmykIgdOeAsNpbXbBhe9SIKI18xwZoFJvtN82pet7vXTTvsjBpz3UuDSzgr/TnybduwB9vwPcndv99Nxm9+1iMXjPh1hAdJzCSHzzbSlSrx+bEtCYXTaeqm9lsjW3nx0Sg8z7oBYHsjiMLPt8JPjsCFBZ8LA1snQYa232b4IBhXMsDlge2tGfXejsS+V0PJtgLUqafPirvSfI00ncdf/+203yaxkKfuH2wyLSRvc9mYDqVuTuR8YGkCsHu88bZaRf0+ufgXA1aDli3Ybnju7gjEEU42m0XSx9PgD5eztHHyYqtCPdwA/iwzwJa9tg6e7RuRKjHcvj2McGXABNSNtl5JnifRgPEAB2CumUsDthtCAsCFgQWg8DJjwCBaivDkoAlgQclATiGmOin2bZMA7r6UI6a7zgLPAeVquAOixGAxe0J0LSWJQRLCJYQLCEsJgT3+IcgwrFFBIsIj3KjQnOCTICT2M/+L/UgNxnKWwy7zgsXSEAVY+BSmwh8aCpuIWEjSKCJvZtgEeFnfLKx626CCPZ+gkWEbT/buPSZBFXoERxhselTDctaeKQHHP4HjzP4RK1FQP2VBCgLAm1BHIeWQL6PQBj61zKIZZCf83nKjSDE3smwDPIg/4FevFVRbt97PkMZQsgrSeNFLjwd6NeyHeR/t3wAnsg6ms3AnjzOu5/N/2l29/6aUVgSnOaWtEBhtzSeAk6Ub7KUc05NJzXqF53MNw73RyiCnxDj2cs1u0eZWExI/iZIdpvVlOtu5fb8BmtuN0fO8TsqK5ZdCRQlBGosX+OdmUte47nyzaGywr8pYpO513wueG+lnhSlRsNsCXFievebW0w3OdtzmVM7LcFywrHq5NyaXrE+mnPCcTsGQb0wf7hRdXxkHLvmjxcds8RYCLrGsZu/nfOm0Fut4FW+/byyF7PhY7Nhs5fuofkjqI5fBqa+hS41k32lOvM5eQCVr83SsZNvHsuUoktTjVSO+/a0c319+PfXTDquAvPZ7D/9aEz3YlYAAA== {{- end }} diff --git a/charts/kubezero-storage/Chart.yaml b/charts/kubezero-storage/Chart.yaml index ac0163d9..532b50ea 100644 --- a/charts/kubezero-storage/Chart.yaml +++ b/charts/kubezero-storage/Chart.yaml @@ -28,7 +28,7 @@ dependencies: condition: aws-ebs-csi-driver.enabled repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: aws-efs-csi-driver - version: 3.1.8 + version: 2.5.7 condition: aws-efs-csi-driver.enabled repository: https://kubernetes-sigs.github.io/aws-efs-csi-driver - name: gemini @@ -39,4 +39,4 @@ dependencies: version: 4.8.4 condition: k8up.enabled repository: https://k8up-io.github.io/k8up -kubeVersion: ">= 1.26.0" +kubeVersion: ">= 1.30.0-0" diff --git a/charts/kubezero-storage/aws-efs-csi-driver.patch b/charts/kubezero-storage/aws-efs-csi-driver.patch new file mode 100644 index 00000000..cbb41add --- /dev/null +++ b/charts/kubezero-storage/aws-efs-csi-driver.patch @@ -0,0 +1,32 @@ +diff -rtuN charts/aws-efs-csi-driver.orig/templates/controller-deployment.yaml charts/aws-efs-csi-driver/templates/controller-deployment.yaml +--- charts/aws-efs-csi-driver.orig/templates/controller-deployment.yaml 2023-08-23 11:32:48.964952023 +0000 ++++ charts/aws-efs-csi-driver/templates/controller-deployment.yaml 2023-08-23 11:32:48.968285371 +0000 +@@ -76,9 +76,14 @@ + - name: AWS_USE_FIPS_ENDPOINT + value: "true" + {{- end }} ++ {{- if .Values.controller.extraEnv }} ++ {{- toYaml .Values.controller.extraEnv | nindent 12 }} ++ {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ ++ - name: aws-token ++ mountPath: /var/run/secrets/sts.amazonaws.com/serviceaccount/ + ports: + - name: healthz + containerPort: {{ .Values.controller.healthPort }} +@@ -137,6 +142,13 @@ + volumes: + - name: socket-dir + emptyDir: {} ++ - name: aws-token ++ projected: ++ sources: ++ - serviceAccountToken: ++ path: token ++ expirationSeconds: 86400 ++ audience: "sts.amazonaws.com" + {{- with .Values.controller.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md index 96dc090e..96e6b218 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md @@ -1,5 +1,62 @@ # Helm chart +## v2.42.0 + +### Feature + +- Set internal traffic policy to local for node metric service ([#2432](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2432), [@ElijahQuinones](https://github.com/ElijahQuinones)) + +## v2.41.0 + +### Feature + +- Add `enabled` flag to schema for use in sub-charting ([#2361](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2361), [@ConnorJC3](https://github.com/ConnorJC3)) +- Add Prometheus Annotations to the Node Service ([#2363](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2363), [@mdzraf](https://github.com/mdzraf)) + +### Bug or regression + +- Prevent nil pointer deref in Helm chart when `node.enableWindows` and `node.otelTracing` are both set ([#2357](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2357), [@ConnorJC3](https://github.com/ConnorJC3)) + +## v2.40.3 + +### Feature + +- Upgrade csi-attacher to v4.8.1, csi-snapshotter to v8.2.1, csi-resizer to v1.13.2 + +### Bug or regression + +- Fix incorrect schema entry for controller.podDisruptionBudget.unhealthyPodEvictionPolicy ([#2389](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2389),[@jamesalford](https://github.com/jamesalford)) + +## v2.40.2 + +### Bug or Regression + +- Add enabled flag to schema for sub-charting ([#2359](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2359), [@ConnorJC3](https://github.com/ConnorJC3)) + +## v2.40.1 + +### Bug or Regression + +- Prevent null deref when enableWindows and otelTracing enabled on node ([#2357](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2357), [@ConnorJC3](https://github.com/ConnorJC3)) +- Fix incorrect properties validation in Helm schema ([#2356](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2356), [@ConnorJC3](https://github.com/ConnorJC3)) + +## v2.40.0 + +#### Default for enable windows changed + +The default value for enableWindows has been changed from false to true. This change makes it so the node damemonset will be scheduled on windows nodes by default. If you wish to not have the node daemonset scheduled on your windows nodes you will need to change enableWindows to false. + +### Feature + +- Add values.schema.json to validate changes in values.yaml. ([#2286](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2286), [@ElijahQuinones](https://github.com/ElijahQuinones)) + +### Bug or Regression + +- Fix helm regression with values.schema.yaml. ([#2322](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2322), [@ElijahQuinones](https://github.com/ElijahQuinones)) +- `global` has been added to the values schema, allowing aws-ebs-csi-driver to be used in a Helm sub chart ([#2321](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2321), [@kejne](https://github.com/kejne)) +- Reconcile some differences between helm chart and values.schema.json ([#2335](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2335), [@ElijahQuinones](https://github.com/ElijahQuinones)) +- Fix helm regression with a1CompatibilityDaemonSet=true ([#2316](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2316), [@AndrewSirenko](https://github.com/AndrewSirenko)) + ## v2.39.3 ### Urgent Upgrade Notes diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml index d017be89..05fab0ed 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.39.0 +appVersion: 1.42.0 description: A Helm chart for AWS EBS CSI Driver home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver keywords: @@ -13,4 +13,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 2.39.3 +version: 2.42.0 diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt index fc4cab42..a6e2aa34 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt @@ -2,6 +2,6 @@ To verify that aws-ebs-csi-driver has started, run: kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -[ACTION REQUIRED] Update to the EBS CSI Driver IAM Policy +[Deprecation announcement] AWS Snow Family device support for the EBS CSI Driver -Due to an upcoming change in handling of IAM polices for the CreateVolume API when creating a volume from an EBS snapshot, a change to your EBS CSI Driver policy may be needed. For more information and remediation steps, see GitHub issue #2190 (https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/2190). This change affects all versions of the EBS CSI Driver and action may be required even on clusters where the driver is not upgraded. +Support for the EBS CSI Driver on [AWS Snow Family devices](https://aws.amazon.com/snowball/) is deprecated, effective immediately. No further Snow-specific bugfixes or feature requests will be merged. The existing functionality for Snow devices will be removed the 1.44 release of the EBS CSI Driver. This announcement does not affect the support of the EBS CSI Driver on other platforms, such as [Amazon EC2](https://aws.amazon.com/ec2/) or EC2 on [AWS Outposts](https://aws.amazon.com/outposts/). For any questions related to this announcement, please comment on this issue [#2365](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/2365) or open a new issue. \ No newline at end of file diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node-windows.tpl b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node-windows.tpl index f53bffd0..dbbcc935 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node-windows.tpl +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node-windows.tpl @@ -17,7 +17,7 @@ spec: app: {{ .NodeName }} {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} updateStrategy: - {{ toYaml .Values.node.updateStrategy | nindent 4 }} + {{- toYaml .Values.node.updateStrategy | nindent 4 }} template: metadata: labels: @@ -111,11 +111,11 @@ spec: value: {{ .otelServiceName }} - name: OTEL_EXPORTER_OTLP_ENDPOINT value: {{ .otelExporterEndpoint }} + {{- end }} {{- if .Values.fips }} - name: AWS_USE_FIPS_ENDPOINT value: "true" {{- end }} - {{- end }} {{- with .Values.node.env }} {{- . | toYaml | nindent 12 }} {{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml index 598984ed..000a2b6b 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -429,6 +429,9 @@ spec: {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.resizer.additionalArgs)) }} - --timeout=60s {{- end }} + {{- if .Values.controller.extraCreateMetadata }} + - --extra-modify-metadata + {{- end}} - --csi-address=$(ADDRESS) - --v={{ .Values.sidecars.resizer.logLevel }} - --handle-volume-inuse-error=false diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml index 5fc2c254..5cd1deb5 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml @@ -47,6 +47,9 @@ kind: Service metadata: name: ebs-csi-node namespace: {{ .Release.Namespace }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "3302" labels: app: ebs-csi-node spec: @@ -56,5 +59,6 @@ spec: - name: metrics port: 3302 targetPort: 3302 + internalTrafficPolicy: Local type: ClusterIP {{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.schema.json b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.schema.json new file mode 100644 index 00000000..4b4d01e7 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.schema.json @@ -0,0 +1,1163 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "type": "object", + "additionalProperties": false, + "description": "Configurable parameters of the AWS EBS CSI Driver", + "properties": { + "global": { + "type": "object", + "additionalProperties": true, + "description": "Allowing global parameters for sub-charting" + }, + "enabled": { + "type": ["boolean", "null"], + "description": "No effect - reserved for use in sub-charting" + }, + "a1CompatibilityDaemonSet": { + "type": "boolean", + "description": "Enable compatibility for the A1 instance family via use of an AL2-based image in a separate DaemonSet", + "default": false + }, + "additionalDaemonSets": { + "type": ["object", "null"], + "additionalProperties": false, + "description": "Additional DaemonSets of the node pod", + "default": null, + "patternProperties": { + "^.*$": { + "type": "object", + "$ref": "#/properties/node" + } + } + }, + "defaultStorageClass": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "description": "Creates a default StorageClass.", + "default": false + } + } + }, + "helmTester": { + "type": "object", + "additionalProperties": false, + "description": "Supply a custom image to the ebs-csi-driver-test pod in helm-tester.yaml", + "properties": { + "enabled": { + "type": "boolean", + "default": true + }, + "image": { + "type": "string" + } + } + }, + "imagePullSecrets": { + "type": "array", + "default": [] + }, + "volumeSnapshotClasses": { + "type": "array", + "description": "Add VolumeSnapshotClass resources", + "default": [] + }, + "awsAccessSecret": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "default": "aws-secret" + }, + "keyId": { + "type": "string", + "default": "key_id" + }, + "accessKey": { + "type": "string", + "default": "access_key" + } + } + }, + "image": { + "type": "object", + "additionalProperties": false, + "properties": { + "pullPolicy": { + "type": "string", + "default": "IfNotPresent" + }, + "tag": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "containerRegistry": { + "type": "string" + } + } + }, + "useOldCSIDriver": { + "type": "boolean", + "description": "Use old CSIDriver without an fsGroupPolicy set Intended for use with older clusters that cannot easily replace the CSIDriver objectThis parameter should always be false for new installations", + "default": false + }, + "nodeComponentOnly": { + "type": "boolean", + "description": "Deploy EBS CSI Driver without controller and associated resources", + "default": false + }, + "storageClasses": { + "type": "array", + "description": "Add StorageClass resources", + "default": [] + }, + "fips": { + "type": "boolean", + "description": "Instruct the AWS SDK to use AWS FIPS endpoints, and deploy container built with BoringCrypto (a FIPS-validated cryptographic library) instead of the Go default. The EBS CSI Driver FIPS images have not undergone FIPS certification, and no official guarantee is made about the compliance of these images under the FIPS standard. Users relying on these images for FIPS compliance should perform their own independent evaluation", + "default": "false" + }, + "fullnameOverride": { + "type": ["string", "null"], + "default": "" + }, + "nameOverride": { + "type": ["string", "null"], + "default": "" + }, + "controller": { + "type": "object", + "additionalProperties": false, + "properties": { + "additionalArgs": { + "type": "array", + "description": "Additional arguments passed to the controller pod", + "default": [], + "items": { + "type": "string" + } + }, + "affinity": { + "type": ["object", "null"], + "description": "Affinity of the controller pod", + "default": { + "affinity": { + "nodeAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "weight": 1, + "preference": { + "matchExpressions": [ + { + "key": "eks.amazonaws.com/compute-type", + "operator": "NotIn", + "values": ["fargate", "auto", "hybrid"] + } + ] + } + } + ] + }, + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "podAffinityTerm": { + "labelSelector": { + "matchExpressions": [ + { + "key": "app", + "operator": "In", + "values": ["ebs-csi-controller"] + } + ] + }, + "topologyKey": "kubernetes.io/hostname" + }, + "weight": 100 + } + ] + } + } + } + }, + "batching": { + "type": "boolean", + "description": "Enable batching of API calls. Improves performance in workloads that are sensitive to EC2 rate limits.", + "default": true + }, + "dnsConfig": { + "type": ["object", "null"], + "description": "DNS configuration for the controller pod", + "default": null + }, + "enableMetrics": { + "type": "boolean", + "description": "Enable metrics collection for the controller pod", + "default": false + }, + "extraVolumeTags": { + "$ref": "#/$defs/extraVolumeTags", + "description": "Additional tags to be added to all EBS volumes", + "default": {} + }, + "loggingFormat": { + "type": "string", + "description": "Log format for the driver container on the controller pod", + "default": "text", + "enum": ["text", "json"] + }, + "nodeSelector": { + "type": ["object", "null"], + "description": "Node selector of the controller pod", + "default": null + }, + "deploymentAnnotations": { + "type": ["object", "null"], + "default": null + }, + "podLabels": { + "type": ["object", "null"], + "default": null + }, + "region": { + "type": ["string", "null"], + "description": "AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata", + "default": "" + }, + "logLevel": { + "type": "integer", + "description": "Set the level of verbosity of the logs", + "default": 2 + }, + "securityContext": { + "type": "object", + "description": "SecurityContext on the controller pod" + }, + "httpEndpoint": { + "type": ["string", "null"], + "description": "(deprecated) The TCP network address where the prometheus metrics endpoint. Will run (example: `:8080` which corresponds to port 8080 on local host). The default is empty string, which means metrics endpoint is disabled.", + "default": "" + }, + "revisionHistoryLimit": { + "type": "integer", + "default": 10 + }, + "extraCreateMetadata": { + "type": "boolean", + "description": "If set, add pv/pvc metadata to plugin create and modify requests as parameters.", + "default": true + }, + "k8sTagClusterId": { + "type": ["string", "null"], + "description": "ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional).", + "default": "" + }, + "defaultFsType": { + "type": ["string", "null"], + "description": "The default filesystem type of the volume to provision when fstype is unspecified in the StorageClass. If the default is not set and fstype is unset in the StorageClass, then no fstype will be set", + "default": "ext4" + }, + "envFrom": { + "type": "array", + "description": "Use envFrom to reference ConfigMaps and Secrets across all containers in the deployment", + "default": [] + }, + "env": { + "type": "array", + "default": [] + }, + "podAnnotations": { + "type": ["object", "null"], + "description": "Annotations applied to the controller pod", + "default": null + }, + "podDisruptionBudget": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "description": "Enables/Disables the EBS CSI Controller Pod's PodDisruptionBudget which ensures that at least 1 controller Pod is always able to Attach/Detach/Create/Delete volumes. Warning: Disabling PodDisruptionBudget may lead to delays in stateful workloads starting due to controller pod restarts or evictions.", + "default": true + }, + "unhealthyPodEvictionPolicy": { + "type": ["string", "null"], + "description": "Unhealthy pod eviction policy for the EBS CSI Controller Pod's PodDisruptionBudget", + "default": null + } + } + }, + "priorityClassName": { + "description": "Priority class for the controller Deployment", + "type": "string", + "default": "system-cluster-critical" + }, + "replicaCount": { + "type": "integer", + "description": "Number of replicas in the controller Deployment", + "default": 2, + "minimum": 1 + }, + "resources": { + "type": ["object", "null"], + "description": "Resource requests/limits of the controller pod", + "default": { + "requests": { + "cpu": "10m", + "memory": "40Mi" + }, + "limits": { + "memory": "256Mi" + } + } + }, + "sdkDebugLog": { + "type": "boolean", + "description": "Enables debug logging of the AWS SDK via --aws-sdk-debug-log=true", + "default": false + }, + "serviceAccount": { + "type": "object", + "additionalProperties": false, + "properties": { + "create": { + "type": "boolean", + "default": true + }, + "name": { + "type": "string", + "default": "ebs-csi-controller-sa" + }, + "automountServiceAccountToken": { + "type": "boolean", + "description": "Enable if EKS IAM for SA is used", + "default": "true" + }, + "annotations": { + "type": ["object", "null"], + "description": "Additional annotations added to the ebs-csi-controller-sa service account", + "default": null + } + } + }, + "tolerations": { + "type": "array", + "description": "Tolerations of the controller pod", + "default": [ + { + "key": "CriticalAddonsOnly", + "operator": "Exists" + }, + { + "effect": "NoExecute", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "items": { + "type": "object" + } + }, + "topologySpreadConstraints": { + "type": "array", + "description": "Topology spread constraints for the controller pod", + "default": [], + "items": { + "type": "object" + } + }, + "otelTracing": { + "type": ["object", "null"], + "additionalProperties": false, + "description": "Enable opentelemetry tracing for the plugin running on the daemonset", + "properties": { + "otelServiceName": { + "type": "string" + }, + "otelExporterEndpoint": { + "type": "string" + } + }, + "default": null + }, + "volumes": { + "type": "array", + "description": "Add additional volumes to be mounted onto the controller", + "default": [] + }, + "volumeMounts": { + "type": "array", + "description": "Add additional volume mounts on the controller", + "default": [] + }, + "containerSecurityContext": { + "type": "object", + "description": "SecurityContext on the controller container (see sidecars for securityContext on sidecar containers)" + }, + "serviceMonitor": { + "type": "object", + "additionalProperties": false, + "properties": { + "forceEnable": { + "type": "boolean", + "default": false + }, + "labels": { + "type": "object", + "description": "Additional labels for ServiceMonitor object" + }, + "interval": { + "type": "string", + "default": "15s" + } + } + }, + "updateStrategy": { + "type": "object" + }, + "initContainers": { + "type": "array", + "description": "Containers to be run before the controller's container starts", + "default": [] + }, + "socketDirVolume": { + "type": "object" + }, + "nameOverride": { + "type": ["string", "null"], + "default": "" + }, + "userAgentExtra": { + "type": "string", + "default": "helm" + }, + "volumeModificationFeature": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable modification of volume type, iops, etc via volume-modifier-for-k8s sidecar", + "default": false + } + } + } + } + }, + "customLabels": { + "type": ["object", "null"], + "description": "Custom labels added to Kubernetes objects", + "default": null + }, + "node": { + "type": "object", + "additionalProperties": false, + "properties": { + "additionalArgs": { + "type": "array", + "description": "Additional arguments passed to the node pod", + "default": [], + "items": { + "type": "string" + } + }, + "affinity": { + "type": ["object", "null"], + "description": "Affinity of the node pod", + "default": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "eks.amazonaws.com/compute-type", + "operator": "NotIn", + "values": ["fargate", "auto", "hybrid"] + }, + { + "key": "node.kubernetes.io/instance-type", + "operator": "NotIn", + "values": [ + "a1.medium", + "a1.large", + "a1.xlarge", + "a1.2xlarge", + "a1.4xlarge" + ] + } + ] + } + ] + } + } + } + }, + "initContainers": { + "type": "array", + "description": "Containers to be run before the csi-node's container starts.", + "default": [] + }, + "logLevel": { + "type": "integer", + "description": "Set the level of verbosity of the node logs", + "default": 2 + }, + "volumes": { + "type": "array", + "description": "Add additional volumes to be mounted onto the node pods", + "default": [] + }, + "probeDirVolume": { + "type": "object" + }, + "namespaceOverride": { + "type": ["string", "null"], + "description": "Allows you to deploy aws-ebs-csi-node daemonset to separate namespace (make sure namespace exists before deploy)", + "default": "" + }, + "env": { + "type": "array", + "default": [] + }, + "enableLinux": { + "type": "boolean", + "description": "Enable the node Linux DaemonSet", + "default": true + }, + "legacyXFS": { + "type": "boolean", + "description": "Warning: This option will be removed in a future release. It is a temporary workaround for users unable to immediately migrate off of older kernel versions. Formats XFS volumes with bigtime=0,inobtcount=0,reflink=0, for mounting onto nodes with linux kernel version <= 5.4. Note that XFS volumes formatted with this option will only have timestamp records until 2038", + "default": "false" + }, + "enableMetrics": { + "type": "boolean", + "description": "Enable metrics collection for the node pods", + "default": false + }, + "enableWindows": { + "type": "boolean", + "description": "Enable the node Windows DaemonSet", + "default": true + }, + "hostNetwork": { + "type": "boolean", + "description": "Run node pods on the host network", + "default": false + }, + "kubeletPath": { + "type": "string", + "description": "Kubelet path of the node", + "default": "/var/lib/kubelet" + }, + "loggingFormat": { + "type": "string", + "description": "Log format for the driver container on the node pod", + "default": "text", + "enum": ["text", "json"] + }, + "podLabels": { + "type": ["object", "null"], + "default": null + }, + "windowsHostProcess": { + "type": "boolean" + }, + "revisionHistoryLimit": { + "type": "integer", + "default": 10 + }, + "nodeSelector": { + "type": ["object", "null"], + "description": "Node selector of the node pod", + "default": null + }, + "volumeMounts": { + "type": "array", + "description": "Add additional volume mounts on the node pods", + "default": [] + }, + "podAnnotations": { + "type": ["object", "null"], + "description": "Annotations applied to the node pod", + "default": null + }, + "securityContext": { + "type": "object", + "description": "SecurityContext on the node pod" + }, + "priorityClassName": { + "description": "Priority class for the Node Daemonset", + "type": ["string", "null"], + "default": "system-node-critical" + }, + "selinux": { + "type": "boolean", + "description": "Enable SELinux-only optimizations on the EBS CSI Driver node pods. Must only be set true if all linux nodes in the DaemonSet have SELinux enabled", + "default": false + }, + "reservedVolumeAttachments": { + "type": ["integer", "null"], + "description": "The number of attachment slots to reserve for system use (and not to be used for CSI volumes)\nWhen this parameter is not specified (or set to -1), the EBS CSI Driver will attempt to determine the number of reserved slots via heuristic", + "default": null, + "minimum": -1 + }, + "resources": { + "type": ["object", "null"], + "description": "Resource requests/limits of the node pod", + "default": { + "requests": { + "cpu": "10m", + "memory": "40Mi" + }, + "limits": { + "memory": "256Mi" + } + } + }, + "serviceAccount": { + "type": "object", + "additionalProperties": false, + "properties": { + "create": { + "type": "boolean", + "default": true + }, + "name": { + "type": "string", + "default": "ebs-csi-node-sa" + }, + "automountServiceAccountToken": { + "type": "boolean", + "description": "Enable if EKS IAM for SA is used", + "default": "true" + }, + "annotations": { + "type": ["object", "null"], + "description": "Additional annotations added to the ebs-csi-node-sa service account", + "default": null + } + } + }, + "terminationGracePeriodSeconds": { + "type": "integer", + "description": "Number of seconds to wait before forcefully terminating a stopping node pod", + "default": 30, + "minimum": 0 + }, + "tolerateAllTaints": { + "type": "boolean", + "description": "Node pods will tolerate all taints", + "default": true + }, + "tolerations": { + "type": "array", + "description": "Tolerations of the node pod", + "default": [ + { + "effect": "NoExecute", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "items": { + "type": "object" + } + }, + "volumeAttachLimit": { + "type": ["integer", "null"], + "description": "Overrides the maximum number of volumes that can be attached per node (for all nodes)", + "default": null, + "minimum": 0 + }, + "envFrom": { + "type": "array", + "default": [] + }, + "containerSecurityContext": { + "type": "object", + "description": "securityContext on the node container (see sidecars for securityContext on sidecar containers). Privileged containers always run as `Unconfined`, which means that they are not restricted by a seccomp profile." + }, + "updateStrategy": { + "type": "object" + }, + "daemonSetAnnotations": { + "type": ["object", "null"], + "default": null + }, + "otelTracing": { + "type": ["object", "null"], + "additionalProperties": false, + "properties": { + "otelServiceName": { + "type": "string" + }, + "otelExporterEndpoint": { + "type": "string" + } + }, + "description": "Enable opentelemetry tracing for the plugin running on the daemonset", + "default": null + } + } + }, + "proxy": { + "type": "object", + "additionalProperties": false, + "properties": { + "http_proxy": { + "type": ["string", "null"], + "description": "Value of HTTP_PROXY and HTTPS_PROXY environment variables", + "default": "" + }, + "no_proxy": { + "type": ["string", "null"], + "description": "Value of NO_PROXY environment variable", + "default": "" + } + } + }, + "sidecars": { + "type": "object", + "additionalProperties": false, + "properties": { + "provisioner": { + "type": "object", + "additionalProperties": false, + "properties": { + "additionalArgs": { + "type": "array", + "description": "Additional arguments passed to the provisioner container", + "default": [], + "items": { + "type": "string" + } + }, + "resources": { + "type": ["object", "null"], + "default": null + }, + "env": { + "type": "array", + "default": [] + }, + "additionalClusterRoleRules": { + "type": ["array", "null"], + "description": "Grant additional permissions to external-provisioner", + "default": [] + }, + "logLevel": { + "type": "integer", + "description": "Set the level of verbosity of the logs", + "default": 2 + }, + "leaderElection": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": true + } + } + }, + "securityContext": { + "type": "object" + }, + "image": { + "type": "object", + "additionalProperties": false, + "properties": { + "pullPolicy": { + "type": "string", + "default": "IfNotPresent" + }, + "tag": { + "type": "string" + }, + "repository": { + "type": "string", + "default": "public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner" + } + } + } + } + }, + "attacher": { + "type": "object", + "additionalProperties": false, + "properties": { + "additionalArgs": { + "type": "array", + "description": "Additional arguments passed to the attacher container", + "default": [], + "items": { + "type": "string" + } + }, + "env": { + "type": "array", + "default": [] + }, + "resources": { + "type": ["object", "null"], + "default": null + }, + "logLevel": { + "type": "integer", + "description": "Set the level of verbosity of the logs", + "default": 2 + }, + "leaderElection": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": true + } + } + }, + "securityContext": { + "type": "object" + }, + "additionalClusterRoleRules": { + "type": ["array", "null"], + "description": "Grant additional permissions to external-attacher", + "default": [] + }, + "image": { + "type": "object", + "additionalProperties": false, + "properties": { + "pullPolicy": { + "type": "string", + "default": "IfNotPresent" + }, + "tag": { + "type": "string" + }, + "repository": { + "type": "string", + "default": "public.ecr.aws/eks-distro/kubernetes-csi/external-attacher" + } + } + } + } + }, + "snapshotter": { + "type": "object", + "additionalProperties": false, + "properties": { + "forceEnable": { + "type": "boolean", + "description": "Enables/disables the external-snapshotter sidecar", + "default": true + }, + "additionalArgs": { + "type": "array", + "description": "Additional arguments passed to the snapshotter container", + "default": [], + "items": { + "type": "string" + } + }, + "additionalClusterRoleRules": { + "type": ["array", "null"], + "description": "Grant additional permissions to external-snapshotter", + "default": [] + }, + "logLevel": { + "type": "integer", + "description": "Set the level of verbosity of the logs", + "default": 2 + }, + "env": { + "type": "array", + "default": [] + }, + "resources": { + "type": ["object", "null"], + "default": null + }, + "image": { + "type": "object", + "additionalProperties": false, + "properties": { + "pullPolicy": { + "type": "string", + "default": "IfNotPresent" + }, + "tag": { + "type": "string" + }, + "repository": { + "type": "string", + "default": "public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter" + } + } + }, + "securityContext": { + "type": "object" + } + } + }, + "resizer": { + "type": "object", + "additionalProperties": false, + "properties": { + "additionalArgs": { + "type": "array", + "description": "Additional arguments passed to the resizer container", + "default": [], + "items": { + "type": "string" + } + }, + "resources": { + "type": ["object", "null"], + "default": null + }, + "logLevel": { + "type": "integer", + "description": "Set the level of verbosity of the logs", + "default": 2 + }, + "leaderElection": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": true + } + } + }, + "additionalClusterRoleRules": { + "type": ["array", "null"], + "description": "Grant additional permissions to external-snapshotter", + "default": [] + }, + "env": { + "type": "array", + "default": [] + }, + "image": { + "type": "object", + "additionalProperties": false, + "properties": { + "pullPolicy": { + "type": "string", + "default": "IfNotPresent" + }, + "tag": { + "type": "string" + }, + "repository": { + "type": "string", + "default": "public.ecr.aws/eks-distro/kubernetes-csi/external-resizer" + } + } + }, + "securityContext": { + "type": "object" + } + } + }, + "livenessProbe": { + "type": "object", + "additionalProperties": false, + "properties": { + "image": { + "type": "object", + "additionalProperties": false, + "properties": { + "pullPolicy": { + "type": "string", + "default": "IfNotPresent" + }, + "tag": { + "type": "string" + }, + "repository": { + "type": "string", + "default": "public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe" + } + } + }, + "additionalArgs": { + "type": "array", + "default": [] + }, + "resources": { + "type": ["object", "null"], + "default": null + }, + "securityContext": { + "type": "object" + } + } + }, + "nodeDriverRegistrar": { + "type": "object", + "additionalProperties": false, + "properties": { + "additionalArgs": { + "type": "array", + "description": "Additional arguments passed to the node-driver-registrar container", + "default": [], + "items": { + "type": "string" + } + }, + "logLevel": { + "type": "integer", + "description": "Set the level of verbosity of the logs", + "default": 2 + }, + "env": { + "type": "array", + "default": [] + }, + "image": { + "type": "object", + "additionalProperties": false, + "properties": { + "pullPolicy": { + "type": "string", + "default": "IfNotPresent" + }, + "tag": { + "type": "string" + }, + "repository": { + "type": "string", + "default": "public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar" + } + } + }, + "resources": { + "type": ["object", "null"], + "default": null + }, + "securityContext": { + "type": "object" + }, + "livenessProbe": { + "type": "object", + "additionalProperties": false, + "properties": { + "exec": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { + "type": "array", + "default": [ + "/csi-node-driver-registrar", + "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)", + "--mode=kubelet-registration-probe" + ] + } + } + }, + "initialDelaySeconds": { + "type": "integer", + "default": 30 + }, + "periodSeconds": { + "type": "integer", + "default": 90 + }, + "timeoutSeconds": { + "type": "integer", + "default": 15 + } + } + } + } + }, + "volumemodifier": { + "type": "object", + "additionalProperties": false, + "properties": { + "additionalArgs": { + "type": "array", + "description": "Additional arguments passed to the volumemodifier container", + "default": [], + "items": { + "type": "string" + } + }, + "resources": { + "type": ["object", "null"], + "default": null + }, + "logLevel": { + "type": "integer", + "description": "Set the level of verbosity of the logs", + "default": 2 + }, + "leaderElection": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": true + } + } + }, + "env": { + "type": "array", + "default": [] + }, + "image": { + "type": "object", + "additionalProperties": false, + "properties": { + "pullPolicy": { + "type": "string", + "default": "IfNotPresent" + }, + "tag": { + "type": "string" + }, + "repository": { + "type": "string", + "default": "public.ecr.aws/ebs-csi-driver/volume-modifier-for-k8s" + } + } + }, + "securityContext": { + "type": "object" + } + } + } + } + } + }, + "$defs": { + "extraVolumeTags": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[a-zA-Z0-9 _\\.:\\/=+\\-@]*$" + }, + "patternProperties": { + "^.*$": { + "type": "string", + "minLength": 0, + "maxLength": 256, + "pattern": "^[a-zA-Z0-9 _\\.:\\/=+\\-@]*$" + } + } + } + } +} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml index 3ef73a82..0cfe397e 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml @@ -11,9 +11,9 @@ image: customLabels: {} # k8s-app: aws-ebs-csi-driver -# Instruct the AWS SDK to use AWS FIPS endpoints, and deploy container built with BoringCrypto (a FIPS-validated cryptographic library) instead of the Go default +# Instruct the AWS SDK to use AWS FIPS endpoints, and deploy container built with Boring Crypto (a FIPS-validated cryptographic library) instead of the Go default # -# The EBS CSI Driver FIPS images have not undergone FIPS certification, and no official guarnatee is made about the compliance of these images under the FIPS standard +# The EBS CSI Driver FIPS images have not undergone FIPS certification, and no official guarantee is made about the compliance of these images under the FIPS standard # Users relying on these images for FIPS compliance should perform their own independent evaluation fips: false sidecars: @@ -22,7 +22,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner - tag: "v5.1.0-eks-1-31-12" + tag: "v5.2.0-eks-1-33-1" logLevel: 2 # Additional parameters provided by external-provisioner. additionalArgs: [] @@ -49,7 +49,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher - tag: "v4.8.0-eks-1-31-12" + tag: "v4.8.1-eks-1-33-1" # Tune leader lease election for csi-attacher. # Leader election is on by default. leaderElection: @@ -78,7 +78,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter - tag: "v8.2.0-eks-1-31-12" + tag: "v8.2.1-eks-1-33-1" logLevel: 2 # Additional parameters provided by csi-snapshotter. additionalArgs: [] @@ -94,7 +94,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe - tag: "v2.14.0-eks-1-31-12" + tag: "v2.15.0-eks-1-33-1" # Additional parameters provided by livenessprobe. additionalArgs: [] resources: {} @@ -106,7 +106,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer - tag: "v1.12.0-eks-1-31-11" + tag: "v1.13.2-eks-1-33-1" # Tune leader lease election for csi-resizer. # Leader election is on by default. leaderElection: @@ -133,7 +133,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar - tag: "v2.13.0-eks-1-31-12" + tag: "v2.13.0-eks-1-33-1" logLevel: 2 # Additional parameters provided by node-driver-registrar. additionalArgs: [] @@ -220,7 +220,7 @@ controller: env: [] # Use envFrom to reference ConfigMaps and Secrets across all containers in the deployment envFrom: [] - # If set, add pv/pvc metadata to plugin create requests as parameters. + # If set, add pv/pvc metadata to plugin create and modify requests as parameters. extraCreateMetadata: true # Extra volume tags to attach to each dynamically provisioned volume. # --- @@ -337,7 +337,7 @@ controller: # Example: # # - name: wait - # image: busybox + # image: public.ecr.aws/amazonlinux/amazonlinux # command: [ 'sh', '-c', "sleep 20" ] # Enable opentelemetry tracing for the plugin running on the daemonset otelTracing: {} @@ -405,7 +405,7 @@ node: automountServiceAccountToken: true # Enable the linux daemonset creation enableLinux: true - enableWindows: false + enableWindows: true # Warning: This option will be removed in a future release. It is a temporary workaround for users unable to immediately migrate off of older kernel versions. # Formats XFS volumes with bigtime=0,inobtcount=0,reflink=0, for mounting onto nodes with linux kernel version <= 5.4. # Note that XFS volumes formatted with this option will only have timestamp records until 2038. @@ -454,7 +454,7 @@ node: # Example: # # - name: wait - # image: busybox + # image: public.ecr.aws/amazonlinux/amazonlinux # command: [ 'sh', '-c', "sleep 20" ] # Enable opentelemetry tracing for the plugin running on the daemonset otelTracing: {} @@ -511,4 +511,4 @@ nodeComponentOnly: false helmTester: enabled: true # Supply a custom image to the ebs-csi-driver-test pod in helm-tester.yaml - image: "us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20241230-3006692a6f-master" + image: "us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20250411-0688312353-master" diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md b/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md index 707d9944..85f2d4d7 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md @@ -1,38 +1,4 @@ # Helm chart -# v3.1.6 -* Bump app/driver version to `v2.1.5` -# v3.1.5 -* Bump app/driver version to `v2.1.4` -# v3.1.4 -* Bump app/driver version to `v2.1.3` -# v3.1.3 -* Bump app/driver version to `v2.1.2` -# v3.1.2 -* Bump app/driver version to `v2.1.1` -# v3.1.1 -* Bump app/driver version to `v2.1.0` -# v3.1.0 -* Bump app/driver version to `v2.0.9` -# v3.0.9 -* Bump app/driver version to `v2.0.8` -# v3.0.8 -* Bump app/driver version to `v2.0.7` -# v3.0.7 -* Bump app/driver version to `v2.0.6` -# v3.0.6 -* Bump app/driver version to `v2.0.5` -# v3.0.5 -* Bump app/driver version to `v2.0.4` -# v3.0.4 -* Bump app/driver version to `v2.0.3` -# v3.0.3 -* Bump app/driver version to `v2.0.2` -# v3.0.2 -* Update Helm to use the image from Public ECR rather than DockerHub -# v3.0.1 -* Bump app/driver version to `v2.0.1` -# v3.0.0 -* Bump app/driver version to `v2.0.0` # v2.5.7 * Bump app/driver version to `v1.7.7` # v2.5.6 @@ -244,4 +210,4 @@ for Controller deployment and Node daemonset * Fixing Controller deployment using `podAnnotations` and `tolerations` values from Node daemonset * Let the user define the whole `tolerations` array, default to `- operator: Exists` * Default `logLevel` lowered from `5` to `2` -* Default `imagePullPolicy` everywhere set to `IfNotPresent` \ No newline at end of file +* Default `imagePullPolicy` everywhere set to `IfNotPresent` diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml index 3c2b059d..51c0a36a 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 2.1.5 +appVersion: 1.7.7 description: A Helm chart for AWS EFS CSI Driver home: https://github.com/kubernetes-sigs/aws-efs-csi-driver keywords: @@ -15,4 +15,4 @@ maintainers: name: aws-efs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-efs-csi-driver -version: 3.1.6 +version: 2.5.7 diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml index 4753b16f..3aa6cfb3 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml @@ -3,18 +3,17 @@ kind: Deployment apiVersion: apps/v1 metadata: - name: {{ .Values.controller.name }} - namespace: {{ .Release.Namespace }} + name: efs-csi-controller labels: app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} {{- with .Values.controller.additionalLabels }} {{ toYaml . | nindent 4 }} {{- end }} spec: - replicas: {{ .Values.controller.replicaCount }} + replicas: {{ .Values.replicaCount }} selector: matchLabels: - app: {{ .Values.controller.name }} + app: efs-csi-controller app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- with .Values.controller.updateStrategy }} @@ -24,7 +23,7 @@ spec: template: metadata: labels: - app: {{ .Values.controller.name }} + app: efs-csi-controller app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- with .Values.controller.podLabels }} @@ -94,17 +93,14 @@ spec: - name: AWS_USE_FIPS_ENDPOINT value: "true" {{- end }} - - name: PORT_RANGE_UPPER_BOUND - value: "{{ .Values.portRangeUpperBound }}" - {{- with .Values.controller.env }} - {{- toYaml . | nindent 12 }} + {{- if .Values.controller.extraEnv }} + {{- toYaml .Values.controller.extraEnv | nindent 12 }} {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.controller.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} + - name: aws-token + mountPath: /var/run/secrets/sts.amazonaws.com/serviceaccount/ ports: - name: healthz containerPort: {{ .Values.controller.healthPort }} @@ -137,16 +133,13 @@ spec: {{- if hasKey .Values.controller "leaderElectionLeaseDuration" }} - --leader-election-lease-duration={{ .Values.controller.leaderElectionLeaseDuration }} {{- end }} - {{- range .Values.sidecars.csiProvisioner.additionalArgs }} - - {{ . }} - {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with default .Values.controller.resources .Values.sidecars.csiProvisioner.resources }} + {{- with .Values.sidecars.csiProvisioner.resources }} resources: {{ toYaml . | nindent 12 }} {{- end }} {{- with .Values.sidecars.csiProvisioner.securityContext }} @@ -162,10 +155,7 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi - {{- with .Values.controller.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with default .Values.controller.resources .Values.sidecars.livenessProbe.resources }} + {{- with .Values.sidecars.livenessProbe.resources }} resources: {{ toYaml . | nindent 12 }} {{- end }} {{- with .Values.sidecars.livenessProbe.securityContext }} @@ -175,19 +165,14 @@ spec: volumes: - name: socket-dir emptyDir: {} - {{- with .Values.controller.volumes }} - {{- toYaml . | nindent 8 }} - {{- end }} + - name: aws-token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 86400 + audience: "sts.amazonaws.com" {{- with .Values.controller.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.controller.topologySpreadConstraints }} - {{- $tscLabelSelector := dict "labelSelector" ( dict "matchLabels" ( dict "app" "efs-csi-controller" ) ) }} - {{- $constraints := list }} - {{- range .Values.controller.topologySpreadConstraints }} - {{- $constraints = mustAppend $constraints (mergeOverwrite . $tscLabelSelector) }} - {{- end }} - topologySpreadConstraints: - {{- $constraints | toYaml | nindent 8 }} - {{- end }} {{- end }} diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-pdb.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-pdb.yaml deleted file mode 100644 index 43d5e697..00000000 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-pdb.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{- if .Values.controller.podDisruptionBudget.enabled -}} -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ .Values.controller.name }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "aws-efs-csi-driver.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - app: {{ .Values.controller.name }} - app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - {{- if .Values.controller.podDisruptionBudget.unhealthyPodEvictionPolicy }} - unhealthyPodEvictionPolicy: {{ .Values.controller.podDisruptionBudget.unhealthyPodEvictionPolicy }} - {{- end }} - {{- if .Values.controller.podDisruptionBudget.maxUnavailable }} - maxUnavailable: {{ .Values.controller.podDisruptionBudget.maxUnavailable }} - {{- end }} - {{- if .Values.controller.podDisruptionBudget.minAvailable }} - minAvailable: {{ .Values.controller.podDisruptionBudget.minAvailable }} - {{- end }} -{{- end -}} \ No newline at end of file diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml index 1a113307..48d1a90c 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ .Values.controller.serviceAccount.name }} - namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} {{- with .Values.controller.serviceAccount.annotations }} @@ -22,7 +21,7 @@ metadata: rules: - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "patch", "delete"] + verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] @@ -75,7 +74,6 @@ kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: efs-csi-provisioner-binding-describe-secrets - namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} subjects: diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/csidriver.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/csidriver.yaml index 6b68cc14..e6b4d419 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/csidriver.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/csidriver.yaml @@ -3,10 +3,8 @@ kind: CSIDriver metadata: name: efs.csi.aws.com annotations: - {{- if .Values.useHelmHooksForCSIDriver }} "helm.sh/hook": pre-install, pre-upgrade "helm.sh/hook-delete-policy": before-hook-creation - {{- end }} "helm.sh/resource-policy": keep spec: attachRequired: false diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml index 324350ce..5eb8a9bd 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml @@ -3,12 +3,8 @@ kind: DaemonSet apiVersion: apps/v1 metadata: name: efs-csi-node - namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} - {{- with .Values.node.additionalLabels }} - {{ toYaml . | nindent 4 }} - {{- end }} spec: selector: matchLabels: @@ -25,9 +21,6 @@ spec: app: efs-csi-node app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} - {{- with .Values.node.podLabels }} - {{ toYaml . | nindent 8 }} - {{- end }} {{- if .Values.node.podAnnotations }} annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} {{- end }} @@ -60,7 +53,7 @@ spec: dnsConfig: {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ .Values.node.serviceAccount.name }} - priorityClassName: {{ .Values.node.priorityClassName}} + priorityClassName: system-node-critical {{- with .Values.node.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} @@ -92,14 +85,9 @@ spec: - name: AWS_USE_FIPS_ENDPOINT value: "true" {{- end }} - - name: PORT_RANGE_UPPER_BOUND - value: "{{ .Values.portRangeUpperBound }}" - {{- with .Values.node.env }} - {{- toYaml . | nindent 12 }} - {{- end }} volumeMounts: - name: kubelet-dir - mountPath: {{ .Values.node.kubeletPath }} + mountPath: /var/lib/kubelet mountPropagation: "Bidirectional" - name: plugin-dir mountPath: /csi @@ -109,9 +97,6 @@ spec: mountPath: /var/amazon/efs - name: efs-utils-config-legacy mountPath: /etc/amazon/efs-legacy - {{- with .Values.node.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} ports: - name: healthz containerPort: {{ .Values.node.healthPort }} @@ -138,7 +123,7 @@ spec: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH - value: {{ printf "%s/plugins/efs.csi.aws.com/csi.sock" (trimSuffix "/" .Values.node.kubeletPath) }} + value: /var/lib/kubelet/plugins/efs.csi.aws.com/csi.sock - name: KUBE_NODE_NAME valueFrom: fieldRef: @@ -175,15 +160,15 @@ spec: volumes: - name: kubelet-dir hostPath: - path: {{ .Values.node.kubeletPath }} + path: /var/lib/kubelet type: Directory - name: plugin-dir hostPath: - path: {{ printf "%s/plugins/efs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} + path: /var/lib/kubelet/plugins/efs.csi.aws.com/ type: DirectoryOrCreate - name: registration-dir hostPath: - path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} + path: /var/lib/kubelet/plugins_registry/ type: Directory - name: efs-state-dir hostPath: @@ -197,6 +182,3 @@ spec: hostPath: path: /etc/amazon/efs type: DirectoryOrCreate - {{- with .Values.node.volumes }} - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml index 797f1b34..c90d62be 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ .Values.node.serviceAccount.name }} - namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} {{- with .Values.node.serviceAccount.annotations }} diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml index 4db8025a..13456b77 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml @@ -5,20 +5,20 @@ nameOverride: "" fullnameOverride: "" +replicaCount: 2 + useFIPS: false -portRangeUpperBound: "21049" - image: - repository: public.ecr.aws/efs-csi-driver/amazon/aws-efs-csi-driver - tag: "v2.1.5" + repository: amazon/aws-efs-csi-driver + tag: "v1.7.7" pullPolicy: IfNotPresent sidecars: livenessProbe: image: repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe - tag: v2.14.0-eks-1-31-5 + tag: v2.11.0-eks-1-29-2 pullPolicy: IfNotPresent resources: {} securityContext: @@ -27,7 +27,7 @@ sidecars: nodeDriverRegistrar: image: repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar - tag: v2.12.0-eks-1-31-5 + tag: v2.9.3-eks-1-29-2 pullPolicy: IfNotPresent resources: {} securityContext: @@ -36,13 +36,12 @@ sidecars: csiProvisioner: image: repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner - tag: v5.1.0-eks-1-31-5 + tag: v3.6.3-eks-1-29-2 pullPolicy: IfNotPresent resources: {} securityContext: readOnlyRootFilesystem: true allowPrivilegeEscalation: false - additionalArgs: [] imagePullSecrets: [] @@ -51,10 +50,6 @@ imagePullSecrets: [] controller: # Specifies whether a deployment should be created create: true - # Name of the CSI controller service - name: efs-csi-controller - # Number of replicas for the CSI controller service deployment - replicaCount: 2 # Number for the log level verbosity logLevel: 2 # If set, add pv/pvc metadata to plugin create requests as parameters. @@ -68,7 +63,7 @@ controller: # path on efs when deleteing an access point deleteAccessPointRootDir: false podAnnotations: {} - podLabels: {} + podLabel: {} hostNetwork: false priorityClassName: system-cluster-critical dnsPolicy: ClusterFirst @@ -94,9 +89,6 @@ controller: - key: efs.csi.aws.com/agent-not-ready operator: Exists affinity: {} - env: [] - volumes: [] - volumeMounts: [] # Specifies whether a service account should be created serviceAccount: create: true @@ -106,12 +98,6 @@ controller: # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/efs-csi-role healthPort: 9909 regionalStsEndpoints: false - # Pod Disruption Budget - podDisruptionBudget: - enabled: false - # maxUnavailable: 1 - minAvailable: 1 - unhealthyPodEvictionPolicy: IfHealthyBudget # securityContext on the controller pod securityContext: runAsNonRoot: false @@ -124,18 +110,7 @@ controller: privileged: true leaderElectionRenewDeadline: 10s leaderElectionLeaseDuration: 15s - # TSCs without the label selector stanza - # - # Example: - # - # topologySpreadConstraints: - # - maxSkew: 1 - # topologyKey: topology.kubernetes.io/zone - # whenUnsatisfiable: ScheduleAnyway - # - maxSkew: 1 - # topologyKey: kubernetes.io/hostname - # whenUnsatisfiable: ScheduleAnyway - topologySpreadConstraints: [] + ## Node daemonset variables @@ -155,7 +130,6 @@ node: # "fs-01234567": # ip: 10.10.2.2 # region: us-east-2 - priorityClassName: system-node-critical dnsPolicy: ClusterFirst dnsConfig: {} @@ -164,9 +138,7 @@ node: # dnsConfig: # nameservers: # - 169.254.169.253 - podLabels: {} podAnnotations: {} - additionalLabels: {} resources: {} # limits: @@ -176,8 +148,7 @@ node: # cpu: 100m # memory: 128Mi nodeSelector: {} - updateStrategy: - {} + updateStrategy: {} # Override default strategy (RollingUpdate) to speed up deployment. # This can be useful if helm timeouts are observed. # type: OnDelete @@ -192,7 +163,6 @@ node: operator: NotIn values: - fargate - - hybrid # Specifies whether a service account should be created serviceAccount: create: true @@ -208,10 +178,6 @@ node: runAsUser: 0 runAsGroup: 0 fsGroup: 0 - env: [] - volumes: [] - volumeMounts: [] - kubeletPath: /var/lib/kubelet storageClasses: [] # Add StorageClass resources like: @@ -232,6 +198,3 @@ storageClasses: [] # ensureUniqueDirectory: true # reclaimPolicy: Delete # volumeBindingMode: Immediate - -# Specifies wether to use helm hooks to apply the CSI driver -useHelmHooksForCSIDriver: true diff --git a/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json b/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json index 2b7c226d..6ce8e923 100644 --- a/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json +++ b/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json @@ -18,7 +18,7 @@ "subdir": "contrib/mixin" } }, - "version": "8c52b414f324d6369b77096af98d8f0416fe20cb", + "version": "8f933a5b5867d078c714fd6a9584aa47f450d8d0", "sum": "XmXkOCriQIZmXwlIIFhqlJMa0e6qGWdxZD+ZDYaN0Po=" }, { @@ -78,8 +78,18 @@ "subdir": "grafana-builder" } }, - "version": "393630ca7ba9b25258c098f1fd4c81962e3ca046", - "sum": "yxqWcq/N3E/a/XreeU6EuE6X7kYPnG0AspAQFKOjASo=" + "version": "42da78cf7f2735c0cf57dee8f80cc52e9e7e57d8", + "sum": "G7B6E5sqWirDbMWRhifbLRfGgRFbIh9WCYa6X3kMh6g=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "mixin-utils" + } + }, + "version": "42da78cf7f2735c0cf57dee8f80cc52e9e7e57d8", + "sum": "SRElwa/XrKAN8aZA9zvdRUx8iebl2It7KNQ7VFvMcBA=" }, { "source": { @@ -98,8 +108,8 @@ "subdir": "" } }, - "version": "1199b50e9d2ff53d4bb5fb2304ad1fb69d38e609", - "sum": "LfbgcJbilu4uBdKYZSvmkoOTPwEAzg10L3/VqKAIWtA=" + "version": "4eee017d21cb63a303925d1dcd9fc5c496809b46", + "sum": "Kh0GbIycNmJPzk6IOMXn1BbtLNyaiiimclYk7+mvsns=" }, { "source": { @@ -108,8 +118,8 @@ "subdir": "" } }, - "version": "4ff562d5e8145940cf355f62cf2308895c4dca81", - "sum": "kiL19fTbXOtNglsmT62kOzIf/Xpu+YwoiMPAApDXhkE=" + "version": "aad557d746a4e05d028a2ce542f61dde3b13c621", + "sum": "H+gpR450rmG2/USp9Y4vMfiz9FCUhKiG7xgqPNB1FJk=" }, { "source": { @@ -118,7 +128,7 @@ "subdir": "jsonnet/kube-state-metrics" } }, - "version": "2a95d4649b2fea55799032fb9c0b571c4ba7f776", + "version": "0b01e3abce1da521b5e620b8aaa76774bb0fda87", "sum": "3bioG7CfTfY9zeu5xU4yon6Zt3kYvNkyl492nOhQxnM=" }, { @@ -128,7 +138,7 @@ "subdir": "jsonnet/kube-state-metrics-mixin" } }, - "version": "2a95d4649b2fea55799032fb9c0b571c4ba7f776", + "version": "0b01e3abce1da521b5e620b8aaa76774bb0fda87", "sum": "qclI7LwucTjBef3PkGBkKxF0mfZPbHnn4rlNWKGtR4c=" }, { @@ -138,8 +148,8 @@ "subdir": "" } }, - "version": "d2dc72021d0247a5199007ed6e425d4615f9fa5c", - "sum": "rHh5ItS3fs1kwz8GKNEPiBBn58m4Bn5v9KAdBU+tf1U=" + "version": "9abc7566be4b58233d7b2aa29665bf47425b30e6", + "sum": "lL17qG4Ejhae7giWBzD2y6HDSxaNgkg8kX7p0i4eUNA=" }, { "source": { @@ -148,8 +158,8 @@ "subdir": "jsonnet/kube-prometheus" } }, - "version": "1eea946a1532f1e8cccfceea98d907bf3a10b1d9", - "sum": "17LhiwefVfoNDsF3DcFZw/UL4PMU7YpNNUaOdaYd1gE=" + "version": "696ce89f1f4d9107bd3a3b026178b320bac03b8e", + "sum": "NYKZ3k27E/3sk27DCNct1X7gqv8tmYxqACnOm96W7pc=" }, { "source": { @@ -158,7 +168,7 @@ "subdir": "jsonnet/mixin" } }, - "version": "7deab71d6d5921eeaf8c79e3ae8e31efe63783a9", + "version": "8ce76ccb32d054cb26898f498ec6bc947cd87d6c", "sum": "gi+knjdxs2T715iIQIntrimbHRgHnpM8IFBJDD1gYfs=", "name": "prometheus-operator-mixin" }, @@ -169,8 +179,8 @@ "subdir": "jsonnet/prometheus-operator" } }, - "version": "7deab71d6d5921eeaf8c79e3ae8e31efe63783a9", - "sum": "LctDdofQostvviE5y8vpRKWGGO1ZKO3dgJe7P9xifW0=" + "version": "8ce76ccb32d054cb26898f498ec6bc947cd87d6c", + "sum": "D8bNt3/sB6EO2AirgMZDt1M/5MwbLMpiQtKqCzfTrE4=" }, { "source": { @@ -179,8 +189,8 @@ "subdir": "doc/alertmanager-mixin" } }, - "version": "b5d1a64ad5bb0ff879705714d1e40cea82efbd5c", - "sum": "Mf4h1BYLle2nrgjf/HXrBbl0Zk8N+xaoEM017o0BC+k=", + "version": "79805945102a7ba3566f38a627ca3f1edd27756e", + "sum": "j5prvRrJdoCv7n45l5Uy2ghl1IDb9BBUqjwCDs4ZJoQ=", "name": "alertmanager" }, { @@ -190,8 +200,8 @@ "subdir": "docs/node-mixin" } }, - "version": "11365f97bef6cb0e6259d536a7e21c49e3f5c065", - "sum": "xYj6VYFT/eafsbleNlC+Z2VfLy1CndyYrJs9BcTmnX8=" + "version": "38d32a397720dfdaf547429ea1b40ab8cfa57e85", + "sum": "NcpQ0Hz0qciUqmOYoAR0X8GUK5pH/QiUXm1aDNgvua0=" }, { "source": { @@ -200,7 +210,7 @@ "subdir": "documentation/prometheus-mixin" } }, - "version": "a5ffa83be83be22e2ec9fd1d4765299d8d16119e", + "version": "9659e30dec7073703fb8548e7b0ad80dd0df48f0", "sum": "2c+wttfee9TwuQJZIkNV7Tekem74Qgc7iZ842P28rNw=", "name": "prometheus" }, @@ -222,7 +232,7 @@ "subdir": "mixin" } }, - "version": "346d18bb0f8011c63d7106de494cf3b9253161a1", + "version": "7d7ea650b76cd201de8ee2c73f31497914026293", "sum": "ieCD4eMgGbOlrI8GmckGPHBGQDcLasE1rULYq56W/bs=", "name": "thanos-mixin" } diff --git a/charts/kubezero-storage/templates/k8up/crds.yaml b/charts/kubezero-storage/templates/k8up/crds.yaml index 6132e25c..e2c665a2 100644 --- a/charts/kubezero-storage/templates/k8up/crds.yaml +++ b/charts/kubezero-storage/templates/k8up/crds.yaml @@ -1432,6 +1432,9 @@ spec: type: object type: array type: object + clusterName: + description: ClusterName sets the kubernetes cluster name to send to pushgateway for grouping metrics + type: string failedJobsHistoryLimit: description: |- FailedJobsHistoryLimit amount of failed jobs to keep for later analysis. @@ -1444,6 +1447,56 @@ spec: Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively. type: integer + labelSelectors: + description: |- + LabelSelectors is a list of selectors that we filter for. + When defined, only PVCs and PreBackupPods matching them are backed up. + items: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array podConfigRef: description: |- PodConfigRef describes the pod spec with wich this action shall be executed. @@ -2346,6 +2399,9 @@ spec: type: object type: array type: object + clusterName: + description: ClusterName sets the kubernetes cluster name to send to pushgateway for grouping metrics + type: string failedJobsHistoryLimit: description: |- FailedJobsHistoryLimit amount of failed jobs to keep for later analysis. @@ -20718,6 +20774,9 @@ spec: type: object type: array type: object + clusterName: + description: ClusterName sets the kubernetes cluster name to send to pushgateway for grouping metrics + type: string concurrentRunsAllowed: type: boolean failedJobsHistoryLimit: @@ -20732,6 +20791,56 @@ spec: Deprecated: Use FailedJobsHistoryLimit and SuccessfulJobsHistoryLimit respectively. type: integer + labelSelectors: + description: |- + LabelSelectors is a list of selectors that we filter for. + When defined, only PVCs and PreBackupPods matching them are backed up. + items: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array podConfigRef: description: |- PodConfigRef describes the pod spec with wich this action shall be executed. @@ -21504,6 +21613,9 @@ spec: type: object type: array type: object + clusterName: + description: ClusterName sets the kubernetes cluster name to send to pushgateway for grouping metrics + type: string concurrentRunsAllowed: type: boolean failedJobsHistoryLimit: diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index e9ecc084..15e2ea6d 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero - Root App of Apps chart type: application -version: 1.31.6 +version: 1.32.3 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -15,4 +15,4 @@ dependencies: - name: kubezero-lib version: 0.2.1 repository: https://cdn.zero-downtime.net/charts -kubeVersion: ">= 1.31.0-0" +kubeVersion: ">= 1.32.0-0" diff --git a/charts/kubezero/README.md b/charts/kubezero/README.md index 865b37f0..ba3cba94 100644 --- a/charts/kubezero/README.md +++ b/charts/kubezero/README.md @@ -1,6 +1,6 @@ # kubezero -![Version: 1.31.6](https://img.shields.io/badge/Version-1.31.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 1.32.3](https://img.shields.io/badge/Version-1.32.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero - Root App of Apps chart @@ -14,7 +14,7 @@ KubeZero - Root App of Apps chart ## Requirements -Kubernetes: `>= 1.31.0-0` +Kubernetes: `>= 1.32.0-0` | Repository | Name | Version | |------------|------|---------| @@ -38,14 +38,15 @@ Kubernetes: `>= 1.31.0-0` | argo.argocd-image-updater.enabled | bool | `false` | | | argo.enabled | bool | `false` | | | argo.namespace | string | `"argocd"` | | -| argo.targetRevision | string | `"0.3.1"` | | +| argo.targetRevision | string | `"0.3.2"` | | | cert-manager.enabled | bool | `false` | | | cert-manager.namespace | string | `"cert-manager"` | | | cert-manager.targetRevision | string | `"0.9.12"` | | | falco.enabled | bool | `false` | | | falco.k8saudit.enabled | bool | `false` | | | falco.targetRevision | string | `"0.1.2"` | | -| global.aws | object | `{}` | | +| global.aws.accountId | string | `"123456789012"` | | +| global.aws.region | string | `"the-moon"` | | | global.clusterName | string | `"zdt-trial-cluster"` | | | global.gcp | object | `{}` | | | global.highAvailable | bool | `false` | | diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 31070a67..9d3a40cd 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -13,7 +13,7 @@ global: addons: enabled: true - targetRevision: 0.8.13 + targetRevision: 0.8.14 external-dns: enabled: false forseti: @@ -32,7 +32,7 @@ addons: network: enabled: true retain: true - targetRevision: 0.5.7 + targetRevision: 0.5.8 cilium: cluster: {} @@ -43,7 +43,7 @@ cert-manager: storage: enabled: false - targetRevision: 0.8.10 + targetRevision: 0.8.11 lvm-localpv: enabled: false aws-ebs-csi-driver: