From a3166859afa3c10212fbbe60e83404d3c163cbd4 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 16 Oct 2024 12:20:20 +0100 Subject: [PATCH] feat: first working v1.30 base --- Dockerfile | 12 ++--- admin/README.md | 6 +-- admin/kubezero.sh | 48 +++++++++---------- admin/libhelm.sh | 19 +++++--- admin/migrate_argo_values.py | 8 ---- admin/upgrade_cluster.sh | 8 ++-- charts/kubeadm/Chart.yaml | 2 +- .../templates/ClusterConfiguration.yaml | 3 +- .../kubeadm/templates/InitConfiguration.yaml | 4 +- charts/kubeadm/templates/_helpers.tpl | 3 +- charts/kubeadm/values.yaml | 2 + charts/kubezero-addons/README.md | 16 +++---- .../aws-node-termination-handler/Chart.yaml | 4 +- charts/kubezero-addons/update.sh | 8 ++++ charts/kubezero-addons/values.yaml | 4 +- .../kubezero-istio-gateway/README.md.gotmpl | 8 ---- charts/kubezero-network/README.md | 11 +++-- .../templates/cilium-grafana-dashboards.yaml | 2 +- .../templates/multus/daemonset.yaml | 2 +- charts/kubezero-network/values.yaml | 6 +-- charts/kubezero/Chart.yaml | 2 +- charts/kubezero/templates/_app.tpl | 4 +- charts/kubezero/templates/addons.yaml | 1 - charts/kubezero/templates/network.yaml | 11 ++++- charts/kubezero/values.yaml | 4 +- docs/v1.30.md | 16 +++++++ 26 files changed, 121 insertions(+), 93 deletions(-) create mode 100644 docs/v1.30.md diff --git a/Dockerfile b/Dockerfile index 7ac9b370..f22259d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,9 @@ ARG ALPINE_VERSION=3.20 FROM docker.io/alpine:${ALPINE_VERSION} ARG ALPINE_VERSION -ARG KUBE_VERSION=1.29.7 -ARG SECRETS_VERSION=4.6.0 -ARG VALS_VERSION=0.37.3 +ARG KUBE_VERSION=1.30.5 +ARG SECRETS_VERSION=4.6.1 +ARG VALS_VERSION=0.37.5 RUN cd /etc/apk/keys && \ wget "https://cdn.zero-downtime.net/alpine/stefan@zero-downtime.net-61bb6bfb.rsa.pub" && \ @@ -22,11 +22,11 @@ RUN cd /etc/apk/keys && \ py3-yaml \ restic \ helm \ + etcd-ctl@edge-community \ cri-tools@kubezero \ - kubeadm@kubezero~=${KUBE_VERSION} \ - kubectl@kubezero~=${KUBE_VERSION} \ etcdhelper@kubezero \ - etcd-ctl@edge-testing + kubeadm@kubezero~=${KUBE_VERSION} \ + kubectl@kubezero~=${KUBE_VERSION} RUN helm repo add kubezero https://cdn.zero-downtime.net/charts && \ mkdir -p /var/lib/kubezero diff --git a/admin/README.md b/admin/README.md index 389f57ca..073b6a97 100644 --- a/admin/README.md +++ b/admin/README.md @@ -1,8 +1,8 @@ # Cluster upgrade flow -## During 1.23 upgrade -- create new kubezero-values CM if not exists yet, by merging parts of the legacy /etc/kubernetes/kubeadm-values.yaml values with potentially existing values from kubezero ArgoCD app values - +## Hard refresh +```kubectl annotate app/kubezero -n argocd argocd.argoproj.io/refresh="hard" +``` # General flow diff --git a/admin/kubezero.sh b/admin/kubezero.sh index ff59db1e..08cd542f 100755 --- a/admin/kubezero.sh +++ b/admin/kubezero.sh @@ -47,15 +47,24 @@ _kubeadm() { # Render cluster config render_kubeadm() { - helm template $CHARTS/kubeadm --output-dir ${WORKDIR} -f ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml + local phase=$1 + + helm template $CHARTS/kubeadm --output-dir ${WORKDIR} \ + -f ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml \ + --set patches=/etc/kubernetes/patches # Assemble kubeadm config cat /dev/null > ${HOSTFS}/etc/kubernetes/kubeadm.yaml - for f in Cluster Init Join KubeProxy Kubelet; do + for f in Cluster KubeProxy Kubelet; do # echo "---" >> /etc/kubernetes/kubeadm.yaml cat ${WORKDIR}/kubeadm/templates/${f}Configuration.yaml >> ${HOSTFS}/etc/kubernetes/kubeadm.yaml done + # skip InitConfig during upgrade + if [ "$phase" != "upgrade" ]; then + cat ${WORKDIR}/kubeadm/templates/InitConfiguration.yaml >> ${HOSTFS}/etc/kubernetes/kubeadm.yaml + fi + # "uncloak" the json patches after they got processed by helm for s in apiserver controller-manager scheduler; do yq eval '.json' ${WORKDIR}/kubeadm/templates/patches/kube-${s}1\+json.yaml > /tmp/_tmp.yaml && \ @@ -98,7 +107,7 @@ pre_kubeadm() { fi # copy patches to host to make --rootfs of kubeadm work - cp -r ${WORKDIR}/kubeadm/templates/patches /host/tmp/ + cp -r ${WORKDIR}/kubeadm/templates/patches ${HOSTFS}/etc/kubernetes } @@ -111,8 +120,6 @@ post_kubeadm() { # Patch coreDNS addon, ideally we prevent kubeadm to reset coreDNS to its defaults kubectl patch deployment coredns -n kube-system --patch-file ${WORKDIR}/kubeadm/templates/patches/coredns0.yaml $LOG - - rm -rf /host/tmp/patches } @@ -126,26 +133,28 @@ kubeadm_upgrade() { migrate_argo_values.py < "$WORKDIR"/kubezero-values.yaml > "$WORKDIR"/new-kubezero-values.yaml # Update kubezero-values CM - kubectl get cm -n kube-system kubezero-values -o=yaml | \ + kubectl get cm -n kubezero kubezero-values -o=yaml | \ yq e '.data."values.yaml" |= load_str("/tmp/kubezero/new-kubezero-values.yaml")' | \ kubectl replace -f - # update argo app kubectl get application kubezero -n argocd -o yaml | \ kubezero_chart_version=$(yq .version /charts/kubezero/Chart.yaml) \ - yq '.spec.source.helm.values |= load_str("/tmp/kubezero/new-kubezero-values.yaml") | .spec.source.targetRevision = strenv(kubezero_chart_version)' | \ + yq 'del (.spec.source.helm.values) | .spec.source.helm.valuesObject |= load("/tmp/kubezero/new-kubezero-values.yaml") | .spec.source.targetRevision = strenv(kubezero_chart_version)' | \ kubectl apply -f - # finally remove annotation to allow argo to sync again kubectl patch app kubezero -n argocd --type json -p='[{"op": "remove", "path": "/metadata/annotations"}]' # Local node upgrade - render_kubeadm + render_kubeadm upgrade pre_kubeadm - # Upgrade - _kubeadm upgrade apply -y --patches /tmp/patches + # Upgrade - we upload the new config first so we can use --patch during 1.30 + _kubeadm init phase upload-config kubeadm + + kubeadm upgrade apply --yes --patches /etc/kubernetes/patches $KUBE_VERSION --rootfs ${HOSTFS} $LOG post_kubeadm @@ -172,7 +181,7 @@ kubeadm_upgrade() { control_plane_node() { CMD=$1 - render_kubeadm + render_kubeadm $CMD # Ensure clean slate if bootstrap, restore PKI otherwise if [[ "$CMD" =~ ^(bootstrap)$ ]]; then @@ -193,9 +202,7 @@ control_plane_node() { cp -r ${WORKDIR}/pki ${HOSTFS}/etc/kubernetes # Always use kubeadm kubectl config to never run into chicken egg with custom auth hooks - # Fallback to old config remove with 1.30 !! - cp ${WORKDIR}/super-admin.conf ${HOSTFS}/root/.kube/config || \ - cp ${WORKDIR}/admin.conf ${HOSTFS}/root/.kube/config + cp ${WORKDIR}/super-admin.conf ${HOSTFS}/root/.kube/config # Only restore etcd data during "restore" and none exists already if [[ "$CMD" =~ ^(restore)$ ]]; then @@ -254,7 +261,7 @@ control_plane_node() { yq eval -i '.etcd.state = "existing" | .etcd.initialCluster = strenv(ETCD_INITIAL_CLUSTER) ' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml - render_kubeadm + render_kubeadm join fi # Generate our custom etcd yaml @@ -263,12 +270,7 @@ control_plane_node() { _kubeadm init phase kubelet-start - # Remove conditional with 1.30 - if [ -f ${HOSTFS}/etc/kubernetes/super-admin.conf ]; then - cp ${HOSTFS}/etc/kubernetes/super-admin.conf ${HOSTFS}/root/.kube/config - else - cp ${HOSTFS}/etc/kubernetes/admin.conf ${HOSTFS}/root/.kube/config - fi + cp ${HOSTFS}/etc/kubernetes/super-admin.conf ${HOSTFS}/root/.kube/config # Wait for api to be online echo "Waiting for Kubernetes API to be online ..." @@ -372,9 +374,7 @@ backup() { # pki & cluster-admin access cp -r ${HOSTFS}/etc/kubernetes/pki ${WORKDIR} cp ${HOSTFS}/etc/kubernetes/admin.conf ${WORKDIR} - - # Remove conditional with 1.30 - [ -f ${HOSTFS}/etc/kubernetes/super-admin.conf ] && cp ${HOSTFS}/etc/kubernetes/super-admin.conf ${WORKDIR} + cp ${HOSTFS}/etc/kubernetes/super-admin.conf ${WORKDIR} # Backup via restic restic backup ${WORKDIR} -H $CLUSTERNAME --tag $CLUSTER_VERSION diff --git a/admin/libhelm.sh b/admin/libhelm.sh index 1774adb6..24851882 100644 --- a/admin/libhelm.sh +++ b/admin/libhelm.sh @@ -34,11 +34,18 @@ function argo_used() { # get kubezero-values from ArgoCD if available or use in-cluster CM without Argo function get_kubezero_values() { - local _namespace="kube-system" - [ "$PLATFORM" == "gke" ] && _namespace=kubezero + ### Remove with 1.31 + ### Migrate the kubezero CM from kube-system to kubezero NS during the 1.30 cycle + kubectl get cm kubezero-values -n kubezero > /dev/null || \ + { create_ns kubezero; kubectl get cm kubezero-values -n kube-system -o yaml | \ + sed 's/^ namespace: kube-system/ namespace: kubezero/' | \ + kubectl create -f - && \ + kubectl delete cm kubezero-values -n kube-system ; } + ### + argo_used && \ - { kubectl get application kubezero -n argocd -o yaml | yq .spec.source.helm.values > ${WORKDIR}/kubezero-values.yaml; } || \ - { kubectl get configmap -n $_namespace kubezero-values -o yaml | yq '.data."values.yaml"' > ${WORKDIR}/kubezero-values.yaml ;} + { kubectl get application kubezero -n argocd -o yaml | yq .spec.source.helm.valuesObject > ${WORKDIR}/kubezero-values.yaml ; } || \ + { kubectl get configmap kubezero-values -n kubezero -o yaml | yq '.data."values.yaml"' > ${WORKDIR}/kubezero-values.yaml ; } } @@ -96,7 +103,7 @@ function argo_app_synced() { function create_ns() { local namespace=$1 if [ "$namespace" != "kube-system" ]; then - kubectl get ns $namespace || kubectl create ns $namespace + kubectl get ns $namespace > /dev/null || kubectl create ns $namespace fi } @@ -169,7 +176,7 @@ function _helm() { [ -n "$_version" ] && targetRevision="--version $_version" fi - yq eval '.spec.source.helm.values' $WORKDIR/kubezero/templates/${module}.yaml > $WORKDIR/values.yaml + yq eval '.spec.source.helm.valuesObject' $WORKDIR/kubezero/templates/${module}.yaml > $WORKDIR/values.yaml if [ $action == "crds" ]; then # Allow custom CRD handling diff --git a/admin/migrate_argo_values.py b/admin/migrate_argo_values.py index 38fafd80..04322bad 100755 --- a/admin/migrate_argo_values.py +++ b/admin/migrate_argo_values.py @@ -8,14 +8,6 @@ import yaml def migrate(values): """Actual changes here""" - # argoCD moves to argo module - try: - if values["argocd"]["enabled"]: - values["argo"] = { "enabled": True, "argo-cd": values["argocd"] } - values.pop("argocd") - except KeyError: - pass - return values diff --git a/admin/upgrade_cluster.sh b/admin/upgrade_cluster.sh index cf0f7cef..499af57b 100755 --- a/admin/upgrade_cluster.sh +++ b/admin/upgrade_cluster.sh @@ -2,7 +2,7 @@ set -eE set -o pipefail -KUBE_VERSION=v1.29 +KUBE_VERSION=v1.30 ARGO_APP=${1:-/tmp/new-kubezero-argoapp.yaml} @@ -26,9 +26,9 @@ read -r #echo "Adjust kubezero values as needed:" # shellcheck disable=SC2015 -#argo_used && kubectl edit app kubezero -n argocd || kubectl edit cm kubezero-values -n kube-system +#argo_used && kubectl edit app kubezero -n argocd || kubectl edit cm kubezero-values -n kubezero -### v1.29 +### v1.30 # # upgrade modules @@ -42,7 +42,7 @@ echo "Applying remaining KubeZero modules..." control_plane_upgrade "apply_cert-manager, apply_istio, apply_istio-ingress, apply_istio-private-ingress, apply_logging, apply_metrics, apply_telemetry, apply_argo" # Final step is to commit the new argocd kubezero app -kubectl get app kubezero -n argocd -o yaml | yq 'del(.status) | del(.metadata) | del(.operation) | .metadata.name="kubezero" | .metadata.namespace="argocd"' | yq 'sort_keys(..) | .spec.source.helm.values |= (from_yaml | to_yaml)' > $ARGO_APP +kubectl get app kubezero -n argocd -o yaml | yq 'del(.status) | del(.metadata) | del(.operation) | .metadata.name="kubezero" | .metadata.namespace="argocd"' | yq 'sort_keys(..)' > $ARGO_APP # Trigger backup of upgraded cluster state kubectl create job --from=cronjob/kubezero-backup kubezero-backup-$KUBE_VERSION -n kube-system diff --git a/charts/kubeadm/Chart.yaml b/charts/kubeadm/Chart.yaml index fe98b91a..a7ed166f 100644 --- a/charts/kubeadm/Chart.yaml +++ b/charts/kubeadm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubeadm description: KubeZero Kubeadm cluster config type: application -version: 1.29.7 +version: 1.30.5 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubeadm/templates/ClusterConfiguration.yaml b/charts/kubeadm/templates/ClusterConfiguration.yaml index 8a18a15a..16ef0a91 100644 --- a/charts/kubeadm/templates/ClusterConfiguration.yaml +++ b/charts/kubeadm/templates/ClusterConfiguration.yaml @@ -2,8 +2,7 @@ apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration kubernetesVersion: {{ .Chart.Version }} clusterName: {{ .Values.global.clusterName }} -featureGates: - EtcdLearnerMode: true # becomes beta in 1.29 +#featureGates: # NonGracefulFailover: true controlPlaneEndpoint: {{ .Values.api.endpoint }} networking: diff --git a/charts/kubeadm/templates/InitConfiguration.yaml b/charts/kubeadm/templates/InitConfiguration.yaml index f334d04b..dba07d92 100644 --- a/charts/kubeadm/templates/InitConfiguration.yaml +++ b/charts/kubeadm/templates/InitConfiguration.yaml @@ -3,8 +3,10 @@ kind: InitConfiguration localAPIEndpoint: advertiseAddress: {{ .Values.listenAddress }} bindPort: {{ .Values.api.listenPort }} +{{- with .Values.patches }} patches: - directory: /tmp/patches + directory: {{ . }} +{{- end }} nodeRegistration: criSocket: "unix:///var/run/crio/crio.sock" ignorePreflightErrors: diff --git a/charts/kubeadm/templates/_helpers.tpl b/charts/kubeadm/templates/_helpers.tpl index 95f2d325..e211fc7f 100644 --- a/charts/kubeadm/templates/_helpers.tpl +++ b/charts/kubeadm/templates/_helpers.tpl @@ -2,9 +2,8 @@ {{- /* Issues: MemoryQoS */ -}} {{- /* v1.28: PodAndContainerStatsFromCRI still not working */ -}} {{- /* v1.28: UnknownVersionInteroperabilityProxy requires StorageVersionAPI which is still alpha in 1.30 */ -}} -{{- /* v1.30: remove/beta KubeProxyDrainingTerminatingNodes */ -}} {{- define "kubeadm.featuregates" }} -{{- $gates := list "CustomCPUCFSQuotaPeriod" "KubeProxyDrainingTerminatingNodes" "ImageMaximumGCAge" }} +{{- $gates := list "CustomCPUCFSQuotaPeriod" }} {{- if eq .return "csv" }} {{- range $key := $gates }} {{- $key }}=true, diff --git a/charts/kubeadm/values.yaml b/charts/kubeadm/values.yaml index 86955934..670d2f5a 100644 --- a/charts/kubeadm/values.yaml +++ b/charts/kubeadm/values.yaml @@ -36,3 +36,5 @@ etcd: # -- Set to false for openrc, eg. on Gentoo or Alpine systemd: false protectKernelDefaults: false + +# patches: /tmp/patches diff --git a/charts/kubezero-addons/README.md b/charts/kubezero-addons/README.md index 88ae687e..105b0f9e 100644 --- a/charts/kubezero-addons/README.md +++ b/charts/kubezero-addons/README.md @@ -1,6 +1,6 @@ # kubezero-addons -![Version: 0.8.8](https://img.shields.io/badge/Version-0.8.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.29](https://img.shields.io/badge/AppVersion-v1.29-informational?style=flat-square) +![Version: 0.8.9](https://img.shields.io/badge/Version-0.8.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.29](https://img.shields.io/badge/AppVersion-v1.29-informational?style=flat-square) KubeZero umbrella chart for various optional cluster addons @@ -18,12 +18,12 @@ Kubernetes: `>= 1.26.0` | Repository | Name | Version | |------------|------|---------| -| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.16.0 | -| https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.14.5 | -| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.37.0 | -| https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.16.0 | +| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.16.1 | +| https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.15.0 | +| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.43.0 | +| https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.16.2 | | https://twin.github.io/helm-charts | aws-eks-asg-rolling-update-handler | 1.5.0 | -| oci://public.ecr.aws/aws-ec2/helm | aws-node-termination-handler | 0.24.0 | +| oci://public.ecr.aws/aws-ec2/helm | aws-node-termination-handler | 0.24.1 | # MetalLB @@ -101,7 +101,7 @@ Device plugin for [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/) | aws-node-termination-handler.useProviderId | bool | `true` | | | awsNeuron.enabled | bool | `false` | | | awsNeuron.image.name | string | `"public.ecr.aws/neuron/neuron-device-plugin"` | | -| awsNeuron.image.tag | string | `"2.19.16.0"` | | +| awsNeuron.image.tag | string | `"2.22.4.0"` | | | cluster-autoscaler.autoDiscovery.clusterName | string | `""` | | | cluster-autoscaler.awsRegion | string | `"us-west-2"` | | | cluster-autoscaler.enabled | bool | `false` | | @@ -110,7 +110,7 @@ Device plugin for [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/) | cluster-autoscaler.extraArgs.scan-interval | string | `"30s"` | | | cluster-autoscaler.extraArgs.skip-nodes-with-local-storage | bool | `false` | | | cluster-autoscaler.image.repository | string | `"registry.k8s.io/autoscaling/cluster-autoscaler"` | | -| cluster-autoscaler.image.tag | string | `"v1.29.4"` | | +| cluster-autoscaler.image.tag | string | `"v1.30.2"` | | | cluster-autoscaler.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | | cluster-autoscaler.podDisruptionBudget | bool | `false` | | | cluster-autoscaler.prometheusRule.enabled | bool | `false` | | diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml index f028f1bd..52ac19fa 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.22.0 +appVersion: 1.22.1 description: A Helm chart for the AWS Node Termination Handler. home: https://github.com/aws/aws-node-termination-handler/ icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png @@ -21,4 +21,4 @@ name: aws-node-termination-handler sources: - https://github.com/aws/aws-node-termination-handler/ type: application -version: 0.24.0 +version: 0.24.1 diff --git a/charts/kubezero-addons/update.sh b/charts/kubezero-addons/update.sh index 55edb204..4e360546 100755 --- a/charts/kubezero-addons/update.sh +++ b/charts/kubezero-addons/update.sh @@ -6,6 +6,14 @@ set -ex login_ecr_public update_helm +# Abandon for now in favor of KRR +# get latest VPA resources, from https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/hack/vpa-process-yamls.sh +# COMPONENTS="vpa-v1-crd-gen vpa-rbac updater-deployment recommender-deployment admission-controller-deployment" +# mkdir -p templates/vertical-pod-autoscaler +#for c in $COMPONENTS; do +# wget -q -O templates/vertical-pod-autoscaler/${c}.yaml https://raw.githubusercontent.com/kubernetes/autoscaler/refs/heads/master/vertical-pod-autoscaler/deploy/${c}.yaml +#done + patch_chart aws-node-termination-handler patch_chart aws-eks-asg-rolling-update-handler diff --git a/charts/kubezero-addons/values.yaml b/charts/kubezero-addons/values.yaml index 6fbcc3da..0a8eae69 100644 --- a/charts/kubezero-addons/values.yaml +++ b/charts/kubezero-addons/values.yaml @@ -160,7 +160,7 @@ awsNeuron: image: name: public.ecr.aws/neuron/neuron-device-plugin - tag: 2.19.16.0 + tag: 2.22.4.0 nvidia-device-plugin: enabled: false @@ -200,7 +200,7 @@ cluster-autoscaler: image: repository: registry.k8s.io/autoscaling/cluster-autoscaler - tag: v1.29.4 + tag: v1.30.2 autoDiscovery: clusterName: "" diff --git a/charts/kubezero-istio-gateway/README.md.gotmpl b/charts/kubezero-istio-gateway/README.md.gotmpl index f2d80bb5..9627be13 100644 --- a/charts/kubezero-istio-gateway/README.md.gotmpl +++ b/charts/kubezero-istio-gateway/README.md.gotmpl @@ -17,14 +17,6 @@ Installs Istio Ingress Gateways, requires kubezero-istio to be installed ! {{ template "chart.valuesSection" . }} -## ToDo -- exclude certain ports from any Envoyfilters -``` - - filter_disabled: - destination_port_range: - end: 1026 - start: 1025 -``` ## Resources - https://github.com/cilium/cilium/blob/main/operator/pkg/model/translation/envoy_listener.go#L134 diff --git a/charts/kubezero-network/README.md b/charts/kubezero-network/README.md index 7edf86a2..5ff78811 100644 --- a/charts/kubezero-network/README.md +++ b/charts/kubezero-network/README.md @@ -1,6 +1,6 @@ # kubezero-network -![Version: 0.5.3](https://img.shields.io/badge/Version-0.5.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.5.4](https://img.shields.io/badge/Version-0.5.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero umbrella chart for all things network @@ -19,9 +19,9 @@ Kubernetes: `>= 1.26.0` | Repository | Name | Version | |------------|------|---------| | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 | -| https://haproxytech.github.io/helm-charts | haproxy | 1.22.0 | -| https://helm.cilium.io/ | cilium | 1.15.7 | -| https://metallb.github.io/metallb | metallb | 0.14.7 | +| https://haproxytech.github.io/helm-charts | haproxy | 1.23.0 | +| https://helm.cilium.io/ | cilium | 1.16.2 | +| https://metallb.github.io/metallb | metallb | 0.14.8 | ## Values @@ -35,6 +35,7 @@ Kubernetes: `>= 1.26.0` | cilium.cni.exclusive | bool | `false` | | | cilium.cni.logFile | string | `"/var/log/cilium-cni.log"` | | | cilium.enabled | bool | `false` | | +| cilium.envoy.enabled | bool | `false` | | | cilium.hubble.enabled | bool | `false` | | | cilium.hubble.relay.enabled | bool | `false` | | | cilium.hubble.tls.auto.certManagerIssuerRef.group | string | `"cert-manager.io"` | | @@ -42,6 +43,7 @@ Kubernetes: `>= 1.26.0` | cilium.hubble.tls.auto.certManagerIssuerRef.name | string | `"kubezero-local-ca-issuer"` | | | cilium.hubble.tls.auto.method | string | `"cert-manager"` | | | cilium.hubble.ui.enabled | bool | `false` | | +| cilium.image.pullPolicy | string | `"Never"` | | | cilium.image.useDigest | bool | `false` | | | cilium.ipam.operator.clusterPoolIPv4PodCIDRList[0] | string | `"10.240.0.0/16"` | | | cilium.l7Proxy | bool | `false` | | @@ -60,6 +62,7 @@ Kubernetes: `>= 1.26.0` | cilium.resources.requests.cpu | string | `"10m"` | | | cilium.resources.requests.memory | string | `"256Mi"` | | | cilium.routingMode | string | `"tunnel"` | | +| cilium.sysctlfix.enabled | bool | `false` | | | cilium.tunnelProtocol | string | `"geneve"` | | | haproxy.PodDisruptionBudget.enable | bool | `false` | | | haproxy.PodDisruptionBudget.minAvailable | int | `1` | | diff --git a/charts/kubezero-network/templates/cilium-grafana-dashboards.yaml b/charts/kubezero-network/templates/cilium-grafana-dashboards.yaml index 270169c7..e7e753e4 100644 --- a/charts/kubezero-network/templates/cilium-grafana-dashboards.yaml +++ b/charts/kubezero-network/templates/cilium-grafana-dashboards.yaml @@ -11,7 +11,7 @@ metadata: k8s-sidecar-target-directory: KubeZero binaryData: cilium-agents.json.gz: - H4sIAAAAAAAC/+2da5PbNpaGv+dXYJWtre6sYpO6a6rmg922M64Z9/TaTmZ3k5QKIiE1yxTJ8NLujqv3ty8AUhIlkbqTAqV3PkzcEsXLAQi8zzkHB9++I6RGHccNaWi5TlD7C/nGP+If2lYQ8r9+lX+R5FP5zTCy7PC9w7/U6/NPTRrSwI18g/Evane+O2HhPYuCWuoY5tChLb4P/YilPr+3zIxPLcN1blzb9cUJ/fGQXml10tB1/n/tdp3o1+lTO3QiL/xq/izkP8grm/nhwi2ET548zqTB/dClvllLvnuW//2d//+zOLxmssDwLU+cSRz/Zno8Gbk+ubFsK5qQq/sw9IK/vHxpyL9fWO7La8Kf27eM+KI1Zlrh0jPXRlZgUPt/GPU/hdQPP7hOeM+/1+SXY596959d1w4tb2rhmmXO/mlbzhfRSr/+Lv/0qMPsYNZO01ba3Br8Lpht3rjOyBrPGj3+KRvRyA6DhU/550bSFOkP+ccTV7YdvxObhSH70bBpEFhGLXXUc33hPFEQupPVE9FHK3jt+ibzP927X/n3I2oHrL560A1zQuYz83+Z7647TNzvh+TuQvYY1jIO+gcdMlsckPXlnU0NNuFXEwfQKHSXDxpS/5VtjZ3kGG3pa9OnXz+FT7L1RcOx5d+PLNv+p0cNK3wSDbz8e94XTIufevoQjrt6CvHqvPMz7Cn6Chszx8w2kXgVZr0s+/sH68/pdwtfPS/dguUE/C27jWzZEbOaQzz7e9FonmvT6RslPqR+LePQf1mmfCP0pe8813LCT9afwhjtpe/EK8Xe8DHLt4ZRcokVg0xf/uTSa58q4J3wTlwwkKZnD2zlXgP+9q177CCkxhfLGWfdyth3I08OWLUVw0/S7b32JsN7n/Ebtc1g2s++5Z3MHY2WzpX7hi6OMfMzUc/jT5Pxzfw28scHOgxcOwpXOnAQMi9ITTTT/y0/SGoE4sZjzFmxm+iy1I7E1RzeKkvfPtd3uAAfXdadvqctn3zh799zLRs5lhxMXG8yb4zUITWXdzOfv9LLFlk2Kg2Ne+ZntbecK2rDp1sxIa50LdebTvM1fiF3wzvg+a7HX22LbdlA8bVjK64x3+ov5Vj4yMzZdP99r6+NtEbGWVJdSv6ktnLE836NU4CJv1LfEe+/qkY29Lahd4s28nfLn84swF9ky7xzF0eN2v3i8F4TWkBvpD54XJxra2Ly1FbOLe3U7cz+nrdL6lqzKXJR6lDbyBjmTCvwbPo0nY6lQl44wEsLhqEbCp2zcICYVP4xvaQQhJmDwHxiXriraYtMuDqzls7r+uHqjPG8YhPPjsaW8wvzg2QS1rUXrRfNuSrkgnTMwsWeuHATW8j8RPbOxZch/rv4/aMne2AQTa58GrKrWEAP5JgUDJL3JhiEXMvb3770ggGfe/76Wy0+7LdanXiu+df/+6327/y/v9Wef9Unv19fk+ETueIf1InNp2v7mvxAOtrihbl45y+2VITWhA24cBHv3cIhlhAq/C16R41Qvif64uQom+/d7DzfvsmLPT8vnsWnzniVacQXbPTejCf+lYb6fd4OVhjLxrfSIhxl/pXYJNVWiZoRT5I8yHep5gYLVJgFmm2wAFgALHABLMDFnyHGjFPzAH1QWKiORpT32WrTwIQ+FmJgOYe+EPPHa2a7Xz+766xdm1hObY/XYeFa84F5zZU0dQy//MhHNfyxjVEILi18sIaX+uClk/BSwkO8p15ZaSDiPdNgQTAwvIjDiuE65r5QJGhI1wrHIfGubead+jGMxSesihtLTLnZxnp9bGPxyafqPYtPn9nGutmKpG/ufiY/B3TMCB/riZP2SWyN0lzM2NQL2ApILbaGkJybB219adButDb4uNrZY7be0ecD1iwemGGBn5jDn81YfWyfMzZcB3AdwHUA1wFcB9VxHQyfwulMdULHwatffho8WH4YUXswYRPXfxos3Zhi3oS21uu0qh5bfMUbWIiZX2LLkw/S8srafNQ3+6N+xW3+4dV/V6unszY1NFZ1q9PHqvTydq/V6nZh7+O7LT9YztqbOn8v5iYLnL1Ts7cBjzvZeNyAS/PULs0ll1PWFLqF1+m6aA9TxhtWoiuzIkbK0Z0lujGr0ptWp9FD3JeLpyKvpe5ELtAFOPSwLgAOPTj04NArzaEn3By84fjFnHBhWh0UlcJSHV9HESDT2xJkWgCZE4JMhkbPfEkK1Z4rkmRn+f7LT9m3vYU4LUrBV9CM64fIbFO+ObopV9G6fFNua7D3t3kGy80YersVGH1MzkrisxKursIIaAQ0AhoBjYBG54JG3Fi+AoskbFumDiod8u2xRhUxSO9sx0H9HjjohBwk1uwuqU7e2Z3ByFRCaM7f0JJiNxkqXCV78Nt76awswC4uQpOByUp1j4fxOnOUgbtK9Q76uM4cb7YisH/yByJcmDMyLZrl+oUA2FJNrtd376bQF8nEfssh/BeED9aWz0jwxIXShH9o2JHJVRgx3InHB1OuT/kIwXmROvxHpljGENfzegHUA+oB9YB6QD1EwRAFOzL+dXYt6aTrOevdugBARQJhQ2/E3wZvJQmLvyFbhx7If5LU2firMz7odLtL5qT67MocddygmzRVehTJX/xbJISgxbaO76202GlCe+feYlwHyrG1yNZ8f5vRmgfGHT9JvPzxK38yskyiJcDvO05HJKkdJNjXHcV3wXtKnfBPErLlfxMnLXWAtkBboC3Q9ihoK2bE5cEXuLtfATj552boPSZxrVSFykWuPnIPT4lci1KQCzgWBJHPtpBq2+a/LUum2iFFaxMhQqb3WbVCO3o35zVot7ertPPq7j2q7ECzQbNBsyEcUfnMs6JDEb/IO10Ti8h4elf2x3GY5eoXFokM5ifpMO8DOfjuvZD/qIUFZkPgrnEOOy33MsMb8yEy95C8UXLH8Edd7eYWwxya+3jNXU60q5enORsHoddS92R0oRbyWjCLd67LJzPfGt+HiIWlq6rSsYgWU8+aZXslLvG4wGoQTbYtr/pyh9MabuSEe2yTIh5N/Nej4f118RulxJd7fibfvokrrmyZshtkcsYihsj25CqVOQZ/JpqUphAJbddYAAQMA4YBw4BhwDBgGDAMGAYM2yoElsthzWNy2GIKKTDs8AQ3YJgiGMYbAwgGBAOCAcGAYGe2bT0gDKocEAYIKzYW1mjlMVgHsbBziYVVlZbIQbj0PZkCU4CAFWgJtARaAi2BlkBLoCXQEmhpr5BVPi51EbI6k5AVaAlxJZASSAmkBFICKYGUQEogJZDSjnGlppYHSn3ElRBXmpNSnfgsjHyHnyzNG4VhU+pqHJmusjjq+uAsvfgiRFwEsScQFYgKRAWiAlGBqEBUICoQ1X6xp3yk6gGplN7DC0h1bKTiViWz7ceuq1b0sJmz3V63sV3Nw3gPpX3LHhb1UHmVHFsbCpoa8Uu98I7Z1BlHorr3X+SIIrrkY9ZAwsX/bTQZ8rc5s6y5OOaD5VgfqLcyzqeHG/7uhcmQlbrIfLih/hfT/TobFwsbTdJNs8+L8vruXca7IEwHz0IlPAv82GCTZwGuBbgWFl0LfCq24VyAcwHOBTgX4FyAc2Fhv+/twrV5pVhaR18GWEeuaxHuBtMK50LWWNlnOC/CK/Y5CJ6ELLIHZuRL2XWwM0K8svJMJTgfZtdaKa7CEZJlvJ+7J8YmuxtjJSGYC8wF5jotc5nMsPilg+VeDRgDjAHGAGOAscrB2GqkN5fG2qCxdTQmNm5VeeXhGdPWYVCFBYdIjwVPIT0WEaxzgyYgEzQ0kAnIVOxyw1Ynj5g0EFOVVyCuJaZdNhcg4LAMDktRWHpztzFgDDAGGAOMAcYuDsYKXcOUr9N60GlVrqkHnVa+ToPTHDoNOg06DToNmUZwm8OPulmg9za4UduNPHneOEier1XbNh+V14jtoRuGS013cWo7dL0vV+06yfKO8mF3wMeLQeiG1N5GDreXKgfw3zt0wtLCuIzy1cllZZWAY8lkflL+FEjShzyGPIY8hjyGPIY8hjzeWx73tpbHXchjBeRxllMa8jhDHsNzDGkMaQxpDGkMaQxpfO7SGAnXIKEDSUjvbI1CTaCQQuWGt0IgNwoNflr+0Yha9n5MVHUkEhWFxdNH/l4FhcuosNvOWxbea6DCbkkVdr888InCZ6iyi4pPQGZUfELFJ9A0aBo0DZoGTc+VfXvX5cvtfp6yb2FZTJUKPs3wO+GEwYxTp/i9fm1KYPAf1AlH4nKYWl5PAnV8yYPLOs2f9/BNegBOACeAE8AJ4ARwAjgBnABOFw9OKwUF8skJG5dUs1QuyAn5mcjPBDMhPxP5mecHRsAi6GRgEbCo2HhSp5VHRSizdhaFHzIgKbPq2npmisklYadr8nKvy8Rl2Ha5UAl0NmUyMgO1g+gsVRoXdSUAZ4AzwBngDHAGOAOcAc4AZzvGrHLprI/NSs6i7gjo7DR0hrAZyAxkhlRDsBkyCkFnoDPQGQqb7Bw6y0so7KMEukobQ07hh+sMLjkGf0QsYjtRT9kJg1tDz+utoOetfG7iM4NZD8w8bmiqoAIm3Xb2u7WpfCrqlxytfsmNfHuIw8Kvrv+FWE7c1cW9oaQJaBm0jDgmWLnqrOx5ZW/0uTLT5+zz2dMhohWqDsgn/6/UN2O9vH2J9CUlbVpchZakomfXOjBm8C5+ci6c7/goKkyMkAFEEEQQRBBE0HmIoKF38t3O81SQrkMGqSiDhk8hC44gg8gPpFdFKfTZp6MR1xWQQhcghZptSCFIIUihC5BC3Fh+WHT2xC2drI2m1/jY9cAIsbyHVq2ISLe05q7x7ZH1yMybaUN8r9G2RvVaZhR72rXkT2qny1vYxdCEv9Q/fr65U9bgo77ZH/XPxuAdZQ3d0Vtmv3k+hiYq9+qe1mHD7hkZW/Vh5Ix6981nwkW4YC21B5QhNUejVsVN/oZxjOTku2h0dfXJcKQPNe1MjK60pXt9baQ1zsjSyo/h52bxDkaRIi39jy4xmWNxW/vsj4gFITp2weYezRzFqlu8y4aNjllxi9OHsbIGZhoH96oDjhmP1epCDTM1pp2HkcnS0nOlDN1q8QHDqLihVTZwp9Fn7aq7VSeWAwNXsQfHS+BE9PM1s92vn9euwVxt5u2COQvXmocV917+dw49uxhjlLIMsJezv3m7g8ydU+4SYDnTpB3xQ4+G9wPDdZzQp8aXwdgYJF68zPSdIKRhFPCPqHC0ik9GdGLZT/wT4SCRGT6+G7p8ROYffb65W835idN94p/V4/Ptn/S8kgiyaw6QeG83Z/7Uj7X88mwMv6V5BYBtXqJZP9buF5dmXjHdZ5v3Rs3em6BMZQw85dtsI79Rsw9X1Mgkvze/3SoL8/3dQ4vcTM0lo9tIw0wdhDTM1DFIw0xkKtIwkYaJNEyCNEyl86eQhll2whoS1UoyNNIwkYaJNEykYSINs8qpakjDLM3SSMMs2eJIwyzU0kjDRBpmnsWRhok0TKRhqmRkpGEiDRNpmDAw0jCRhnmgMQpJw1wpoZaXh6k3ekjEPM9EzA4SMU+TiNlBImaRiZgdJGIWm4jZQSJm8YmYnQtIxOwgEROJmEjERCImEjEzRiAkYiIRkyARsxiDIxETqWpIxFTe2EjEJEjERCImUqiQiIlETFh8K4sjEbNQSyMRE4mYeRZHIiYSMZGIqZKRkYiJREwkYsLASMREIuaBxiinHmYvLw+zjzzMyyiImfhLkIt5iqKY5Rj/ogtjKmXicy2OqZSRz7lApoqGPnaRzNvEg4/8zNRByM9MHYP8zES+Ij8T+ZnIzyTIz1Q6sQr5mWVnsiGDrSRDIz8T+ZnIz0R+JvIzq5zDhvzM0iyN/MySLY78zEItjfxM5GfmWRz5mcjPRH6mSkZGfibyM5GfCQMjPxP5mQcao6RCmXkJmk0NCZqXUSgTCZrLxi+xWCYSNC/LxOdaNBMJmhds6GMXz0SC5gUlaGrIz0R+JvIzcyET+ZnH9QIoHY5v6/22wSru4FI7QdDUR53Co5T7OG/auy6u7beyfTe97kGum6UOwaizMl3RILx15eS3/NWyEiQrbre1nqGQDm22xjXkW+P78LI9Q0E0mdKG5Q2oafJZKMiBiyxC4P+eEsb1b07RhPDtW3yp5+ctnDf5gPDKtl2DCtR4NX1g0MEl0IEOPAAeAA+AB2XhgRlNvIGc5PzICwMyc+SpncfLaKfVqXwSzTrbq4sU7Z7e0oxKIsVKPDiPKbqo16OI6M8IMcj3hs82/IKD0A2pvRsMUJ/RKRLUicPf0uvisUBc5vmZfPsmri7/cRRQeJNYJxVNeMOtQz5K64AZwAxgBjADmOFMmMH1gi2IoVAnbD8ngU7rQDGdUDGJ7Befhmwqm/jQ+mAZLBjwAcjZQSf9qk9+v06LJUO0ZQkCKb7QgXLoU/zU5GePWw8+U+gf6B/oH+ifi/OZFuszypFAPfiMFPEZRY7PqHEvgusDhz/JTtHiwrVO6u6IvLuy8vxzLHTPqB3eD/g9yolCXWPFN0pmN7pF8n6+VBQOM8Y15wMXEeRv8syQi5CLkIuQi5CLcJcdyV2mazkBxibE4qnFYtpdZvquNzDcyAkTT5lp+Ux6pLgKevvTx7efPgkVtJP/jAuXoBTXWXyhQyOJ3AIeM8nbscg3JHd8TEUE8UIkEYrGQxJBEl2IJJqcvmS8wiXQDIM2K19E8YO6RaP88fBK73TrRG9r/P90/Rq2zrT1buWNPlxeeaMPlSpvVHRVSdMUcFKEQeTMsa7vLQVlTp5VLYs0wBqxNSKZCQFrzF8TsbLO/lEEXiphkXLy4HP9VP0m/FQKpXXJeOGA2vY0r8tnBrMemLl3gpc8z0DwaZ3E93VNfiBLa2sKcVvJK3/mF5bp8PG1c51Y2wX1brl5yFtpGXiuEMyD5wqeK3iuzsRzNVQgmJeX/K5DJSkXzRs+hSw4ejSPa6NeNSN6n306GlkGdBEietBF0EXQRagjctSYHvPpmJFbmdCsbtmQYa/Zr37ESXEzG3rb0Lsw8zHDe1n3chFBvsIbQYWdTPbx2OfAaF8Hi57YY7/irHeiiVKrahZn65KWH6X2eFHXMvMBp/ydQRS2ymwu3LyZx/rozKFxGaHGNw+n+tJw2mhtcu3pOfHPRm/9aGrEI9RCB7CpM4746yVdGTYVtn7MquLL553baDLkQ1MW9MljeHe0PlBvhZ3TY6fh8gvEA3DqIvOxk/pfTPfrLFmmsKEx3TT7+KvuXNsynjL6h7AePFOI2MEzBc8UPFPV8Uz57I+tYnbYUf6MitpiR/nyE15F71bWxsORPix+RUUpO5DoejubEtqI/6sU//ckSQzs7iB+N3ZMjix+G0H5wpZY9SXbOLNxWjH7zO6rLB9Mvon2TK8t2kLT2zrIHZM5UyM7BAwOBgeDg8FRAudYkco81dzsQjVXpQbO+9td0mbbFS2CcyMfiAhLBOS9IxNnIYmqL4n4scEmSQRNBE20qIn44GJDFZ1lzuzJoxIio8JjviwATDzfNfhMwxv3M59a1I1PNFpmq3EGaZ0zw0ceH7IYnSht9nL22yu68sI8LW7uXfP4tMwG/J1z/elKrURIxoe++EFkOV14ZKPghpFtQOI2gKXPaMivWm2WUgfmyhWuUWv4rJr5ih7kqlH1ZyWc3dBylhC0DnLMLbUfowuF2uC2I0ctCcR77OPTYDpgDnzm2U+DgBmuYwaDYKtE8uWaQIHBX4Jr8pKsBEbXXUu6Dve+2t4ewhUH0u6+Q3kHW7kO60deIrJ2MFdpBcDCCHpIiaY70YWIzwKPDyuMiPsjV68extfwsyL0DDcrQs9wsqJg09FCz3kKV9dbiD1XpWLTbrHncyjZlISeUbMJNZsgjSCNII0Qfz6+Q1nZaE+nNdSMXsWjPdIbpqyJ+9Ro9WjFTTxRd5ub79t6v22wqhu4oC1WjjJIMFNjGnpwpoF3qy42ubzNgybYPGhBC5DCpquNodGFHekUeCErY4qSYsQ5izd0DR60U3rQRDW1tAeNg+94zK8+DSnGGz8NDFFOzIx8id1FhYV3uvaJwsRbOvLExFBiGBgtePQWFGRbYu0+tODx30Guig9Z2B4XkiOfY4uQN8nTw4tefS86FnLBjb67Gx0LueBIL4oYsZDrRKiOhVwnMTvWC52Lpau6VXGeR6rTwaoFdd1VS6h86asWdii8v2l9QuFujQMXKSxaToWVCq/2X6nAGwQrFeBIgCMB+XhwI2APxaMKfq4T71EqvUgLs7iim7J+mTY1tKonjVmO4kbWuu1G1Tvy5qkIFf+38DZ0NmW/NHPWjzWbR/Q1cIke3rpSsR3L5eBb4/uwAI+DcG3brDprzBYJlj+71Kz8H5w7Y2sNuCoMo2Drxfapn+6OsoIOMvd645o9pLLpRMvk1jnQD12llrr7rfwJ+XT8NrFlQGLjktSpSWxTcHL1ORnL1oDJwGRgskKYrPLStWHXHNIhVqUUZWB/PKRXvX6dK/NGnXQ7daK9aLeusc6qsB7dGLW7bYYenWlgrLM6q3VWRbgUOtu6FFLxJiyoOdGCmoWEAa6mZBGaQKlCfCUvUKmARUpe8FGFPnLoAgqZ+fBx+mzw48CPAz8O/Djw45zNpoATBbw4zKdjRqZrE5UFYLPTa1a+kstIvh3K1nLRW2a/WX0fg/qd+WwsnSx3VrfAVqfb1BrV90uq36XbvVar2z0DS6vepUcGowZqxh3JE5zd3BfhES60p1fSKIXJs6qu8ct3kjfgJFe/6tSeS8PID6SjleBAnw1AalV6UtlqU1xfb7lTVVhSur+laGWzU75Mq8lZR1GrxTNitr3e7FMF6mPkIJCBQAYCGQhkIJCBhNQLyZf8ftQw+pVftpmsMDHuqTNG7aESLQ5TIzf1nD2RF5aburnaezOntlavrfZ6V/jmFhNY5fC9ZU7iOaetKmeHEyWrqtcftkpRrRezDjwWktt6DevEjUKDX4MfMqKWXe6+61nS92C3GL8rcsd8citGOTjFKu8UQzUzeMV294qhLPq5+sVcD4XR93EooDA6CqPv7TJDYfQzsfTmpKmDXWWLCnDdtbgEM5mjYIZW3r6AjSbKsCvrI1rORKpwafRdyqLn4/Cb20/ESwp9/xGxAKtdkSQCHEaSCGAYSSJHFa0qVy0b9c3+qI84emEG7mkdNuyiShkMjEwQZIKomAmyCvitvCVYehdLsFRK8xjwcYOT+4MlbgC1ypS3yilTQBTuK8fZ9P1j8mwHenEE02weVfWlUbXR2pRf18rxmnY3OE2NeJxa6Ao2dcYRHccOIZsKez9m7fnAp5/baDLkA1Tm1gPimA+WY32g3ooHIj2CGi6/QDwMpy4yH0Gp/8V0v84m0sIGyHTT5A8qW2xckNFFhAHh4kPKC3x8SHmBl69aXr4tPHw7AFN/59T5Vid7am+3EQ9VOh56z83k8qlhMvgjok5o2exKe9HX6iQdKJ1tHeUL67C4JtYgFr8hDYNZ0HQYGV9YdtRUxiv/jX8kE48z4qhtEUeNJbcaMdTdFBVJ2ybZL7mvhfciG8lg0rLYOxnSC9IL0gvSC9Jrb+mV4avu5PmqIb6qKL76EF9HEV99iC+IL4ivBnLbkNt2ltJLhZ0cxJpoLHgp2sIvlU6/6mujyle7DyJDrFFU1sitVpcNjfMwstK9uTnq9JqFb5JRTkGfdk7CQat/bDCuL3eWR+VZWaxHtlk1YFlUMdnIxMH2lZCbWhAv2rpKqpvsw7tcmMomWs0q2ZmFk7sogoaxpAvcC+5VnHvFGI6YA8B325iDEou6uI4xXMdhRii6KTZ9K9DUPqPmk6om9sdDetXT66TR0Oqk364T7UW7cV1xk3+llujXP4bujzM5pW4RGY22NapXktpWA5p52NbWL7QOa9VQbZnSRJySbb3gYvFnJUQiFy94IITdyGcj03MSeU5FF2R0cla5dVtYkFHSggwO0aHv2jbzsSTj3KuugNCRFojINCLT5eD5O2rZy3ofsenj21hl58dZhKbl5oXKVm/V+22DnUe4tKPn4ICGcGmVUouXo6XGnDEGfrRLnHRx79OiSXxhl9J8mDr2vhhp84ziAV2lzUKSefyQOp/rIRPRYESDwZpgTbBmNYPBpw8ER8nCGzF7Rj4DDpVibdVTdttar1P5/UTQpctIb+DCH4NHaZZWfeDosmGj8tt2Y3QuMWMkz3PV6ZXguVr+yLrsdfI+8xgN31i+SAyMD7k/natrqpZmq+T5L/bZ1SWkYRRck5dk9yvuvZNMfM0SVuLLCx2a9zKzA3kTHWfpQWFpLzkFMzoa0l5KSnv5e8RN4bCQBUSYbBx3GGTAnHcGjA6vJLyS8ErCK3kEr+Qv8k7XEFjG07uyP47DrHi9sEhksLg2k22/D+Tgu/emFUfdRGM2BO7KfHYaFDJRbz5E5h6SN0qWytuFN7dcCIDmPlpz70P+3d1TVro5te86zeL2aQnpcHk1xgK2+9b4PkR1uzQlC/o1bCH1OARbA66RmGM8JVXsdifzl7uee18GFw8p/uvR8L4EEo8v9/xMvn0TVzwQybk1+F3xGZckNiFXlP/FaVVuj46KeIAuQBegC9AF6AJ0AboAXYCuLcOtudTVBnWdcndM+gjqUpC6eLuAuJB9D+S6XORCMbZzhS7XA3ZBhwO7gF07YVdv51hXt5VHXT2F1mdvwjQs0F7NWgVLbWSp73l3tQNyxU1H+L8kTgXgKfAUeAo8BZ4CT4GnwFPgKfDUDmGsfKDqHBOojghPiHFtYieJCXsWuJrDks/CyHcGxkLMpgxmSl34aOgEcgI5gZxATiAnkBPICeQEcgI5pcmps3MkqtfIAadu49iRKJQFLo6cZjUYBlyJcFEy8JnBrAdmruEn9kdEbf6JsJf4m78Zlpn6e4mv2kslTQz+gtYJlZVhSiCr+EKSrOSlD2SqX8TD1snPYtM7FgTUfyJ/7wXkrbQe+ZhYD2gFtAJaAa2AVthxFXAFuAJcAa52CUvl01ULdHWZdCW77tnj1XvnAYAFwAJgAbAAWIhdAa+AV8Ar4NVxV1H1enl01QZdXQZdzWjqgOBV8uPKxrBuAVgALAAWAAuABcACYAGwAFgArOMsq8onrCYI60IJ68ICWKAr0BXoCnQFuroYuppsQVeFboXTz9nVUtd6xRVlhrDafSucFWGVr6ekKOGf3MhPbln41fW/3Lm2ZTxtXsyeqCjyA+loJUop/rw5Iur1dhvJrj5roqAgnCCcIJwgnCCcLk04HeSnvKWTtW7KmuEzPj/LiasID6I05a5+w5H1yMybaSt839XMYWZLpPqV/EntdP7gjXY2GZ8b1bYza7TMVqPido48U/X+zLRR3+wWbeeStsPJJS+9D/KqOHldBnOBtkBboC3QFmgLtAXaAm2BtkBbatDWcpiroTXzwlyArarD1p1gqjNFLP5sACuAFcAKYAWwAlgBrABWACuAlTphrHyyajRAVlUPY/HHPt/oFX84sBXYCmwFtgJbXShbFep01rW8DB+srai6NPrEVYtlKKuODlJGybNBHEEcQRxBHEEcQRwV4DjKVUcNDeqo4urorWPK4fIs5dH04QIIJAgkCCQIJAgkCKQCvEfdPH2kQx9VPbBGuUDgU8t5+o9mT7e9QPouOZG4lBiqxEFNLf4lnzzu2YTOW7gpc3Z5u47njZoYObkUnxDk0rTa7Lwhm3hicksN93Fnn/eK1CtkRL4fd/lv6U4sZvOlV9VyDDsy2SvbXp1lavZUv2T2qaTTr/7METkO/FdvPg3uPv7zw9vPf3v786f0D+evffptrv0RMf9Jqr3My8mvf0lGyLSkShldX/h0zB6XDw2+WN7Pvv3pyTEy7nzavqk3avl9TVmZ2vbsduLme/FD+lqpVlgYGJjNjJBlKqdYSop5e6kI6GxqqP37YMAvnD26bzESCAVu8VliKpNEEw/kyYPpiPAQ91T5ZqfLEW7sP4ulPDf2D376XXrFTrc67xKNw7pEMoynOxZ/cWW7B/81vbXa4rc5XySdK36g1BdRwD7HY0FaDccDlJiin+MRwJrM9VdtFItyPrt8/VG/nw4boZt8Vlv4mWcZX1JpSVPjDKbDa3pu4dNPasBrpv/Q5zNnrZ36t57+o6mlv7mf/7uR+reepAb9Pr1vMQGk2n/jVdIn7qRPnL5Ko5X+I1WBsmum73d6Lwsm+1PM2iIdKzRqyYcLlZrIB8ZZwEhG+CjOrnoIo3+FZjT5M/70YTbiy95T+8rYl098NpcveO275/8HM4Xl7vlQBAA= + H4sIAAAAAAAC/+2da5PbNpaGv+dXYJWtre6sYpO6a6rmg922M64Z9/TaTmZ3k5QKIiE1yxTJ8NLujqv3ty8AUhIlkbqTAqV3PkzcEsXLAQi8zzkHB9++I6RGHccNaWi5TlD7C/nGP+If2lYQ8r9+lX+R5FP5zTCy7PC9w7/U6/NPTRrSwI18g/Evane+O2HhPYuCWuoY5tChLb4P/YilPr+3zIxPLcN1blzb9cUJ/fGQXml10tB1/n/tdp3o1+lTO3QiL/xq/izkP8grm/nhwi2ET548zqTB/dClvllLvnuW//2d//+zOLxmssDwLU+cSRz/Zno8Gbk+ubFsK5qQq/sw9IK/vHxpyL9fWO7La8Kf27eM+KI1Zlrh0jPXRlZgUPt/GPU/hdQPP7hOeM+/1+SXY596959d1w4tb2rhmmXO/mlbzhfRSr/+Lv/0qMPsYNZO01ba3Br8Lpht3rjOyBrPGj3+KRvRyA6DhU/550bSFOkP+ccTV7YdvxObhSH70bBpEFhGLXXUc33hPFEQupPVE9FHK3jt+ibzP927X/n3I2oHrL560A1zQuYz83+Z7647TNzvh+TuQvYY1jIO+gcdMlsckPXlnU0NNuFXEwfQKHSXDxpS/5VtjZ3kGG3pa9OnXz+FT7L1RcOx5d+PLNv+p0cNK3wSDbz8e94XTIufevoQjrt6CvHqvPMz7Cn6Chszx8w2kXgVZr0s+/sH68/pdwtfPS/dguUE/C27jWzZEbOaQzz7e9FonmvT6RslPqR+LePQf1mmfCP0pe8813LCT9afwhjtpe/EK8Xe8DHLt4ZRcokVg0xf/uTSa58q4J3wTlwwkKZnD2zlXgP+9q177CCkxhfLGWfdyth3I08OWLUVw0/S7b32JsN7n/Ebtc1g2s++5Z3MHY2WzpX7hi6OMfMzUc/jT5Pxzfw28scHOgxcOwpXOnAQMi9ITTTT/y0/SGoE4sZjzFmxm+iy1I7E1RzeKkvfPtd3uAAfXdadvqctn3zh799zLRs5lhxMXG8yb4zUITWXdzOfv9LLFlk2Kg2Ne+ZntbecK2rDp1sxIa50LdebTvM1fiF3wzvg+a7HX22LbdlA8bVjK64x3+ov5Vj4yMzZdP99r6+NtEbGWVJdSv6ktnLE836NU4CJv1LfEe+/qkY29Lahd4s28nfLn84swF9ky7xzF0eN2v3i8F4TWkBvpD54XJxra2Ly1FbOLe3U7cz+nrdL6lqzKXJR6lDbyBjmTCvwbPo0nY6lQl44wEsLhqEbCp2zcICYVP4xvaQQhJmDwHxiXriraYtMuDqzls7r+uHqjPG8YhPPjsaW8wvzg2QS1rUXrRfNuSrkgnTMwsWeuHATW8j8RPbOxZch/rv4/aMne2AQTa58GrKrWEAP5JgUDJL3JhiEXMvb3770ggGfe/76Wy0+7LdanXiu+df/+6327/y/v9Wef9Unv19fk+ETueIf1InNp2v7mvxAOtrihbl45y+2VITWhA24cBHv3cIhlhAq/C16R41Qvif64uQom+/d7DzfvsmLPT8vnsWnzniVacQXbPTejCf+lYb6fd4OVhjLxrfSIhxl/pXYJNVWiZoRT5I8yHep5gYLVJgFmm2wAFgALHABLMDFnyHGjFPzAH1QWKiORpT32WrTwIQ+FmJgOYe+EPPHa2a7Xz+766xdm1hObY/XYeFa84F5zZU0dQy//MhHNfyxjVEILi18sIaX+uClk/BSwkO8p15ZaSDiPdNgQTAwvIjDiuE65r5QJGhI1wrHIfGubead+jGMxSesihtLTLnZxnp9bGPxyafqPYtPn9nGutmKpG/ufiY/B3TMCB/riZP2SWyN0lzM2NQL2ApILbaGkJybB219adButDb4uNrZY7be0ecD1iwemGGBn5jDn81YfWyfMzZcB3AdwHUA1wFcB9VxHQyfwulMdULHwatffho8WH4YUXswYRPXfxos3Zhi3oS21uu0qh5bfMUbWIiZX2LLkw/S8srafNQ3+6N+xW3+4dV/V6unszY1NFZ1q9PHqvTydq/V6nZh7+O7LT9YztqbOn8v5iYLnL1Ts7cBjzvZeNyAS/PULs0ll1PWFLqF1+m6aA9TxhtWoiuzIkbK0Z0lujGr0ptWp9FD3JeLpyKvpe5ELtAFOPSwLgAOPTj04NArzaEn3By84fjFnHBhWh0UlcJSHV9HESDT2xJkWgCZE4JMhkbPfEkK1Z4rkmRn+f7LT9m3vYU4LUrBV9CM64fIbFO+ObopV9G6fFNua7D3t3kGy80YersVGH1MzkrisxKursIIaAQ0AhoBjYBG54JG3Fi+AoskbFumDiod8u2xRhUxSO9sx0H9HjjohBwk1uwuqU7e2Z3ByFRCaM7f0JJiNxkqXCV78Nt76awswC4uQpOByUp1j4fxOnOUgbtK9Q76uM4cb7YisH/yByJcmDMyLZrl+oUA2FJNrtd376bQF8nEfssh/BeED9aWz0jwxIXShH9o2JHJVRgx3InHB1OuT/kIwXmROvxHpljGENfzegHUA+oB9YB6QD1EwRAFOzL+dXYt6aTrOevdugBARQJhQ2/E3wZvJQmLvyFbhx7If5LU2firMz7odLtL5qT67MocddygmzRVehTJX/xbJISgxbaO76202GlCe+feYlwHyrG1yNZ8f5vRmgfGHT9JvPzxK38yskyiJcDvO05HJKkdJNjXHcV3wXtKnfBPErLlfxMnLXWAtkBboC3Q9ihoK2bE5cEXuLtfATj552boPSZxrVSFykWuPnIPT4lci1KQCzgWBJHPtpBq2+a/LUum2iFFaxMhQqb3WbVCO3o35zVot7ertPPq7j2q7ECzQbNBsyEcUfnMs6JDEb/IO10Ti8h4elf2x3GY5eoXFokM5ifpMO8DOfjuvZD/qIUFZkPgrnEOOy33MsMb8yEy95C8UXLH8Edd7eYWwxya+3jNXU60q5enORsHoddS92R0oRbyWjCLd67LJzPfGt+HiIWlq6rSsYgWU8+aZXslLvG4wGoQTbYtr/pyh9MabuSEe2yTIh5N/Nej4f118RulxJd7fibfvokrrmyZshtkcsYihsj25CqVOQZ/JpqUphAJbddYAAQMA4YBw4BhwDBgGDAMGAYM2yoElsthzWNy2GIKKTDs8AQ3YJgiGMYbAwgGBAOCAcGAYGe2bT0gDKocEAYIKzYW1mjlMVgHsbBziYVVlZbIQbj0PZkCU4CAFWgJtARaAi2BlkBLoCXQEmhpr5BVPi51EbI6k5AVaAlxJZASSAmkBFICKYGUQEogJZDSjnGlppYHSn3ElRBXmpNSnfgsjHyHnyzNG4VhU+pqHJmusjjq+uAsvfgiRFwEsScQFYgKRAWiAlGBqEBUICoQ1X6xp3yk6gGplN7DC0h1bKTiViWz7ceuq1b0sJmz3V63sV3Nw3gPpX3LHhb1UHmVHFsbCpoa8Uu98I7Z1BlHorr3X+SIIrrkY9ZAwsX/bTQZ8rc5s6y5OOaD5VgfqLcyzqeHG/7uhcmQlbrIfLih/hfT/TobFwsbTdJNs8+L8vruXca7IEwHz0IlPAv82GCTZwGuBbgWFl0LfCq24VyAcwHOBTgX4FyAc2Fhv+/twrV5pVhaR18GWEeuaxHuBtMK50LWWNlnOC/CK/Y5CJ6ELLIHZuRL2XWwM0K8svJMJTgfZtdaKa7CEZJlvJ+7J8YmuxtjJSGYC8wF5jotc5nMsPilg+VeDRgDjAHGAGOAscrB2GqkN5fG2qCxdTQmNm5VeeXhGdPWYVCFBYdIjwVPIT0WEaxzgyYgEzQ0kAnIVOxyw1Ynj5g0EFOVVyCuJaZdNhcg4LAMDktRWHpztzFgDDAGGAOMAcYuDsYKXcOUr9N60GlVrqkHnVa+ToPTHDoNOg06DToNmUZwm8OPulmg9za4UduNPHneOEier1XbNh+V14jtoRuGS013cWo7dL0vV+06yfKO8mF3wMeLQeiG1N5GDreXKgfw3zt0wtLCuIzy1cllZZWAY8lkflL+FEjShzyGPIY8hjyGPIY8hjzeWx73tpbHXchjBeRxllMa8jhDHsNzDGkMaQxpDGkMaQxpfO7SGAnXIKEDSUjvbI1CTaCQQuWGt0IgNwoNflr+0Yha9n5MVHUkEhWFxdNH/l4FhcuosNvOWxbea6DCbkkVdr888InCZ6iyi4pPQGZUfELFJ9A0aBo0DZoGTc+VfXvX5cvtfp6yb2FZTJUKPs3wO+GEwYxTp/i9fm1KYPAf1AlH4nKYWl5PAnV8yYPLOs2f9/BNegBOACeAE8AJ4ARwAjgBnABOFw9OKwUF8skJG5dUs1QuyAn5mcjPBDMhPxP5mecHRsAi6GRgEbCo2HhSp5VHRSizdhaFHzIgKbPq2npmisklYadr8nKvy8Rl2Ha5UAl0NmUyMgO1g+gsVRoXdSUAZ4AzwBngDHAGOAOcAc4AZzvGrHLprI/NSs6i7gjo7DR0hrAZyAxkhlRDsBkyCkFnoDPQGQqb7Bw6y0so7KMEukobQ07hh+sMLjkGf0QsYjtRT9kJg1tDz+utoOetfG7iM4NZD8w8bmiqoAIm3Xb2u7WpfCrqlxytfsmNfHuIw8Kvrv+FWE7c1cW9oaQJaBm0jDgmWLnqrOx5ZW/0uTLT5+zz2dMhohWqDsgn/6/UN2O9vH2J9CUlbVpchZakomfXOjBm8C5+ci6c7/goKkyMkAFEEEQQRBBE0HmIoKF38t3O81SQrkMGqSiDhk8hC44gg8gPpFdFKfTZp6MR1xWQQhcghZptSCFIIUihC5BC3Fh+WHT2xC2drI2m1/jY9cAIsbyHVq2ISLe05q7x7ZH1yMybaUN8r9G2RvVaZhR72rXkT2qny1vYxdCEv9Q/fr65U9bgo77ZH/XPxuAdZQ3d0Vtmv3k+hiYq9+qe1mHD7hkZW/Vh5Ix6981nwkW4YC21B5QhNUejVsVN/oZxjOTku2h0dfXJcKQPNe1MjK60pXt9baQ1zsjSyo/h52bxDkaRIi39jy4xmWNxW/vsj4gFITp2weYezRzFqlu8y4aNjllxi9OHsbIGZhoH96oDjhmP1epCDTM1pp2HkcnS0nOlDN1q8QHDqLihVTZwp9Fn7aq7VSeWAwNXsQfHS+BE9PM1s92vn9euwVxt5u2COQvXmocV917+dw49uxhjlLIMsJezv3m7g8ydU+4SYDnTpB3xQ4+G9wPDdZzQp8aXwdgYJF68zPSdIKRhFPCPqHC0ik9GdGLZT/wT4SCRGT6+G7p8ROYffb65W835idN94p/V4/Ptn/S8kgiyaw6QeG83Z/7Uj7X88mwMv6V5BYBtXqJZP9buF5dmXjHdZ5v3Rs3em6BMZQw85dtsI79Rsw9X1Mgkvze/3SoL8/3dQ4vcTM0lo9tIw0wdhDTM1DFIw0xkKtIwkYaJNEyCNEyl86eQhll2whoS1UoyNNIwkYaJNEykYSINs8qpakjDLM3SSMMs2eJIwyzU0kjDRBpmnsWRhok0TKRhqmRkpGEiDRNpmDAw0jCRhnmgMQpJw1wpoZaXh6k3ekjEPM9EzA4SMU+TiNlBImaRiZgdJGIWm4jZQSJm8YmYnQtIxOwgEROJmEjERCImEjEzRiAkYiIRkyARsxiDIxETqWpIxFTe2EjEJEjERCImUqiQiIlETFh8K4sjEbNQSyMRE4mYeRZHIiYSMZGIqZKRkYiJREwkYsLASMREIuaBxiinHmYvLw+zjzzMyyiImfhLkIt5iqKY5Rj/ogtjKmXicy2OqZSRz7lApoqGPnaRzNvEg4/8zNRByM9MHYP8zES+Ij8T+ZnIzyTIz1Q6sQr5mWVnsiGDrSRDIz8T+ZnIz0R+JvIzq5zDhvzM0iyN/MySLY78zEItjfxM5GfmWRz5mcjPRH6mSkZGfibyM5GfCQMjPxP5mQcao6RCmXkJmk0NCZqXUSgTCZrLxi+xWCYSNC/LxOdaNBMJmhds6GMXz0SC5gUlaGrIz0R+JvIzcyET+ZnH9QIoHY5v6/22wSru4FI7QdDUR53Co5T7OG/auy6u7beyfTe97kGum6UOwaizMl3RILx15eS3/NWyEiQrbre1nqGQDm22xjXkW+P78LI9Q0E0mdKG5Q2oafJZKMiBiyxC4P+eEsb1b07RhPDtW3yp5+ctnDf5gPDKtl2DCtR4NX1g0MEl0IEOPAAeAA+AB2XhgRlNvIGc5PzICwMyc+SpncfLaKfVqXwSzTrbq4sU7Z7e0oxKIsVKPDiPKbqo16OI6M8IMcj3hs82/IKD0A2pvRsMUJ/RKRLUicPf0uvisUBc5vmZfPsmri7/cRRQeJNYJxVNeMOtQz5K64AZwAxgBjADmOFMmMH1gi2IoVAnbD8ngU7rQDGdUDGJ7Befhmwqm/jQ+mAZLBjwAcjZQSf9qk9+v06LJUO0ZQkCKb7QgXLoU/zU5GePWw8+U+gf6B/oH+ifi/OZFuszypFAPfiMFPEZRY7PqHEvgusDhz/JTtHiwrVO6u6IvLuy8vxzLHTPqB3eD/g9yolCXWPFN0pmN7pF8n6+VBQOM8Y15wMXEeRv8syQi5CLkIuQi5CLcJcdyV2mazkBxibE4qnFYtpdZvquNzDcyAkTT5lp+Ux6pLgKevvTx7efPgkVtJP/jAuXoBTXWXyhQyOJ3AIeM8nbscg3JHd8TEUE8UIkEYrGQxJBEl2IJJqcvmS8wiXQDIM2K19E8YO6RaP88fBK73TrRG9r/P90/Rq2zrT1buWNPlxeeaMPlSpvVHRVSdMUcFKEQeTMsa7vLQVlTp5VLYs0wBqxNSKZCQFrzF8TsbLO/lEEXiphkXLy4HP9VP0m/FQKpXXJeOGA2vY0r8tnBrMemLl3gpc8z0DwaZ3E93VNfiBLa2sKcVvJK3/mF5bp8PG1c51Y2wX1brl5yFtpGXiuEMyD5wqeK3iuzsRzNVQgmJeX/K5DJSkXzRs+hSw4ejSPa6NeNSN6n306GlkGdBEietBF0EXQRagjctSYHvPpmJFbmdCsbtmQYa/Zr37ESXEzG3rb0Lsw8zHDe1n3chFBvsIbQYWdTPbx2OfAaF8Hi57YY7/irHeiiVKrahZn65KWH6X2eFHXMvMBp/ydQRS2ymwu3LyZx/rozKFxGaHGNw+n+tJw2mhtcu3pOfHPRm/9aGrEI9RCB7CpM4746yVdGTYVtn7MquLL553baDLkQ1MW9MljeHe0PlBvhZ3TY6fh8gvEA3DqIvOxk/pfTPfrLFmmsKEx3TT7+KvuXNsynjL6h7AePFOI2MEzBc8UPFPV8Uz57I+tYnbYUf6MitpiR/nyE15F71bWxsORPix+RUUpO5DoejubEtqI/6sU//ckSQzs7rZZkXXiRzaTuxKKd2kW+S9+Q0H56pZY/+U4ZpqN8KVZanbFsvwyxzPWNEW3NFtNL3iQsyZzHkfuCAgdhA5CB6GjQM6x4ph5mrrZhaauSoWc97e7JNW2K1oi50Y+EBGWCMh7R6bVQhJVXxLxY4NNkgiaCJpoURPxwcWGKjrLjNqTxyxEvoXHfFkemHi+a/CZhjfuZz61qBu9aLTMVuMMkj5nho88PmQxOlHa7OXsxld0XYZ50tyyny3RjvG3L35I+dU8Pm2zAX8nXT8Q2VAXHgEpuImktUlsbVj6jAb/qtVwKXWIrlyBG1UH0qoZsujhrhp1glYC4A0tZ9FB6yBn3VL7MbpQ2g2uPHLUIkK8xz4+DaZD58Bnnv00CJjhOmYwCLZKPV+uIhQY/CW4Ji/JSth03bWkO3Hvq+3tNVxxKu3uT5R3sJU7sX7kRSX7x6GXh/nSVg8sjKWHlHe6E52J+Czw+ADDiLg/cvXqYXwNLywC03DCIjANFyyKPR0tMJ2ndXW9hch0Vao97RaZPodyT0lgGvWeUO8J0gjSCNII0enjO5mVjQB1WkPN6FU8AiT9YsqauE+NVo9W3MQTdbfI+b6t99sGq7qBC9qe5SiDBDM1pqEHZxp4t8pkk8vbeGiCjYcWtAApbLraGCRd2M1OgReyMqYoKVqcs7RD1+BBO6UHTVRiS3vQOPiOx/zq0+BivGnUwBClyMzIl9hdVIB4p2ufKGC8pSNPTAwlBoTRgkdvQUG2Jdb9Qwse/x3kqviQZe9xETryObYIeZM8Pbzo1feiY5kX3Oi7u9GxzAuO9KKIEcu8ToTqWOZ1ErNjDdG5WLqq2xzneaQ6HaxfUNddtYTKl75+YYei/ZtWKhTu1jjacoVFG6qwZuHV/msWeNNgzQJcCnApIDMPDgXsxHhU6c8V4z0KrhdpYRZXflPWQ9Omhlb19DHLUdzIWrfdqHpH3jwVYd+ALfwOnU15MM2clWTN5hG9Dlyih7euVGzHcj741vg+LMD3IJzcNqvOarNFluXPLjUr/wfnzthaA64KwyjYetO+1E93R1lBB5k7xnHNHlLZdKJlcmsf6IeuV0vd/VaehXw6fpvYMiCxcUnq1CS2KTi5+pyMBWzAZGAyMFkhTFZ5Eduwaw7pEOtTijKwPx7Sq16/zpV5o066nTrRXrRb11hxVViPboza3TZDj840MFZcndWKqyJcCp1tXQqpeBOW1pxoac1C6gBXU7Iczfa+gTLC2yUvVamARUpe+lGFPnLoUgqZ+fBx+mzw48CPAz8O/Djw45zN5oETBbw4zKdjRqarFJUFYLPTa1a+pstIvh3KVnXRW2a/WX0fg/qd+WwsnSx8VrfUVqfb1BrV90uq36XbvVar2z0DS6vepUcGowaqxx3JE5zd3BfhES60p1fSKIXJs6qu9st3kjfgJFe//tSWK8aWl4aRH0hHK8GBPhuA1Kr5pLLVpri+3nKnqrWkdH9L0cpmp3yZVpOzjqJWi2fEbHu92ace1MfIQSADgQwEMhDIQCADCakXki/5/ahh9Cu/bDNZYWLcU2eMKkQlWhymRm7qOXsiLyw3dXPd92ZOla1eW+31rvDNLSawyuF7y5zEc05bVc4OJ0pWVa8/bJWiWi9mHXgsJLeva+ZGocGvwQ8ZUcsudwf2LOl7sFuM3xW5Yz65FaMcnGKVd4qhmhm8Yrt7xVAg/Vz9Yq6HEun7OBRQIh0l0vd2maFE+plYenPS1MGuskUFuO5aXIKZzFEwQytvh8BGEwXZlfURLWciVbhI+i4F0vNx+M3tJ+Ilhb7/iFiA1a5IEgEOI0kEMIwkkaOKVpWrlo36Zn/URxy9MAP3tA4bdlGlDAZGJggyQVTMBFkF/FbeEiy9iyVYKqV5DPi4wcn9wRI3gFplylvllCkgCveV42z//jF5tgO9OIJpNo+q+tKo2mhtyq9r5XhNuxucpkY8Ti10BZs644iOY4eQTYW9H7P2fODTz200GfIBKnPrAXHMB8uxPlBvxQORHkENl18gHoZTF5mPoNT/YrpfZxNpYQNkumnyB5UtNi7I6CLCgHDxIeUFPj6kvMDLVy0v3xYevh2Aqb9z6nyrkz21t9uIhyodD73nZnL51DAZ/BFRJ7RsdqW96Gt1kg6UzraO8oV1WFwTaxCL35CGwSxoOoyMLyw7airjlf/GP5KJxxlx1LaIo8aSW40Y6m6KiqRtk+yX3NfCe5GNZDBpWeydDOkF6QXpBekF6bW39MrwVXfyfNUQX1UUX32Ir6OIrz7EF8QXxFcDuW3IbTtL6aXCTg5iTTQWvBRt4ZdKp1/1tVHlq90HkSHWKCpr5Fary4bGeRhZ6d7cHHV6zcI3ySinoE87J+Gg1T82GNeXO8uj8qws1iPbrBqwLKqYbGTiYPtKyE0tiBdtXSXVTfbhXS5MZROtZpXszMLJXRRBw1jSBe4F9yrOvWIMR8wB4LttzEGJRV1cxxiu4zAjFN0Um74VaGqfUfNJVRP74yG96ul10mhoddJv14n2ot24rrjJv1JL9OsfQ/fHmZxSt4iMRtsa1StJbasBzTxsa+sXWoe1aqi2TGkiTsm2XnCx+LMSIpGLFzwQwm7ks5HpOYk8p6ILMjo5q9y6LSzIKGlBBofo0Hdtm/lYknHuVVdA6EgLRGQakely8PwdtexlvY/Y9PFtrLLz4yxC03LzQmWrt+r9tsHOI1za0XNwQEO4tEqpxcvRUmPOGAM/2iVOurj3adEkvrBLaT5MHXtfjLR5RvGArtJmIck8fkidz/WQiWgwosFgTbAmWLOaweDTB4KjZOGNmD0jnwGHSrG26im7ba3Xqfx+IujSZaQ3cOGPwaM0S6s+cHTZsFH5bbsxOpeYMZLnuer0SvBcLX9kXfY6eZ95jIZvLF8kBsaH3J/O1TVVS7NV8vwX++zqEtIwCq7JS7L7FffeSSa+Zgkr8eWFDs17mdmBvImOs/SgsLSXnIIZHQ1pLyWlvfw94qZwWMgCIkw2jjsMMmDOOwNGh1cSXkl4JeGVPIJX8hd5p2sILOPpXdkfx2FWvF5YJDJYXJvJtt8HcvDde9OKo26iMRsCd2U+Ow0Kmag3HyJzD8kbJUvl7cKbWy4EQHMfrbn3If/u7ikr3Zzad51mcfu0hHS4vBpjAdt9a3wforpdmpIF/Rq2kHocgq0B10jMMZ6SKna7k/nLXc+9L4OLhxT/9Wh4XwKJx5d7fibfvokrHojk3Br8rviMSxKbkCvK/+K0KrdHR0U8QBegC9AF6AJ0AboAXYAuQNeW4dZc6mqDuk65OyZ9BHUpSF28XUBcyL4Hcl0ucqEY27lCl+sBu6DDgV3Arp2wq7dzrKvbyqOunkLrszdhGhZor2atgqU2stT3vLvaAbnipiP8XxKnAvAUeAo8BZ4CT4GnwFPgKfAUeGqHMFY+UHWOCVRHhCfEuDaxk8SEPQtczWHJZ2HkOwNjIWZTBjOlLnw0dAI5gZxATiAnkBPICeQEcgI5gZzS5NTZORLVa+SAU7dx7EgUygIXR06zGgwDrkS4KBn4zGDWAzPX8BP7I6I2/0TYS/zN3wzLTP29xFftpZImBn9B64TKyjAlkFV8IUlW8tIHMtUv4mHr5Gex6R0LAuo/kb/3AvJWWo98TKwHtAJaAa2AVkAr7LgKuAJcAa4AV7uEpfLpqgW6uky6kl337PHqvfMAwAJgAbAAWAAsxK6AV8Ar4BXw6rirqHq9PLpqg64ug65mNHVA8Cr5cWVjWLcALAAWAAuABcACYAGwAFgALADWcZZV5RNWE4R1oYR1YQEs0BXoCnQFugJdXQxdTbagq0K3wunn7Gqpa73iijJDWO2+Fc6KsMrXU1KU8E9u5Ce3LPzq+l/uXNsynjYvZk9UFPmBdLQSpRR/3hwR9Xq7jWRXnzVRUBBOEE4QThBOEE6XJpwO8lPe0slaN2XN8Bmfn+XEVYQHUZpyV7/hyHpk5s20Fb7vauYwsyVS/Ur+pHY6f/BGO5uMz41q25k1WmarUXE7R56pen9m2qhvdou2c0nb4eSSl94HeVWcvC6DuUBboC3QFmgLtAXaAm2BtkBboC01aGs5zNXQmnlhLsBW1WHrTjDVmSIWfzaAFcAKYAWwAlgBrABWACuAFcBKnTBWPlk1GiCrqoex+GOfb/SKPxzYCmwFtgJbga0ulK0KdTrrWl6GD9ZWVF0afeKqxTKUVUcHKaPk2SCOII4gjiCOII4gjgpwHOWqo4YGdVRxdfTWMeVweZbyaPpwAQQSBBIEEgQSBBIEUgHeo26ePtKhj6oeWKNcIPCp5Tz9R7On214gfZecSFxKDFXioKYW/5JPHvdsQuct3JQ5u7xdx/NGTYycXIpPCHJpWm123pBNPDG5pYb7uLPPe0XqFTIi34+7/Ld0Jxaz+dKrajmGHZnslW2vzjI1e6pfMvtU0ulXf+aIHAf+qzefBncf//nh7ee/vf35U/qH89c+/TbX/oiY/yTVXubl5Ne/JCNkWlKljK4vfDpmj8uHBl8s72ff/vTkGBl3Pm3f1Bu1/L6mrExte3Y7cfO9+CF9rVQrLAwMzGZGyDKVUywlxby9VAR0NjXU/n0w4BfOHt23GAmEArf4LDGVSaKJB/LkwXREeIh7qnyz0+UIN/afxVKeG/sHP/0uvWKnW513icZhXSIZxtMdi7+4st2D/5reWm3x25wvks4VP1Dqiyhgn+OxIK2G4wFKTNHP8QhgTeb6qzaKRTmfXb7+qN9Ph43QTT6rLfzMs4wvqbSkqXEG0+E1Pbfw6Sc14DXTf+jzmbPWTv1bT//R1NLf3M//3Uj9W09Sg36f3reYAFLtv/Eq6RN30idOX6XRSv+RqkDZNdP3O72XBZP9KWZtkY4VGrXkw4VKTeQD4yxgJCN8FGdXPYTRv0IzmvwZf/owG/Fl76l9ZezLJz6byxe89t3z/wOShkCLP1EEAA== cilium-operator.json.gz: H4sIAAAAAAAC/+2cX2/bNhDA3/spCBUYkiHDLDdJgwF7cNO0C9C0RpJ2A9pAoCVKJkKJGkk5cQ1/95HUP8qSndStEy/liy0dKfJ4dyR/IiXNngHgeB5O0kxw5w/wWZ4DMNO/MiWBMZJS5/WFNzz/cHZy+dfJxwtnr0wmcISISk8ZjZEYo4zXiQHiPsOpwDRRWeoEMU11oQEUkNOM+ahOS0kW4eQ0WFZonv6+UGtY59AZ5vL3SuV1YJJQAVXdqlV5exyCuajaWLdSpowyTMSp0tPdq6WGgs3KjDwogSOi0gXLkCEf46BDin2aHFNCmSqQRSO409sDfdeVPwcHe8DdNYsujT+o2wJ+AQOCmGioUNuTj0cUssAp0ub6/0r+zrVRUIDFgrZOlCChzZ1khOQSBtPxJaVE4FTKe1qIVZYjfUhwcq1DJbd0ChNEOkIHEgy5bmvtgTxhEiltn/thCMOw0rVy8QjqK0JIOKqDSbbtHUoiMVY+6jXkqCv7HZ5zQoxIcEyTEEdN9QIUwoyIptJS7mdc0FhJ55V4bniBThBj0um5aVqtCjEhpTUrwVsGA4wS0UiIZClDumA01ewDo7YbZYa+Ibg1y5DnU3Xe0kJGZYCSC8Rwh9G0j/v7de9GEUqCLuc1rtO2YSxvxmJKDG+7pDjpkPIxvWl3GSFjn3TknkCS1Y1otVRGqU41S9PCGxzkQWRKjYDOu57sDEOKE3FGdTfWgjp4aNocWaqYe1cV1VIohT7qiuAUyShNBIxQyyGp0kCFSMZN9+fytv+kCwLEkB5bQkJFrS/XDv9gRmildyPIdadV1yu/7ZkpKlxfIUJvLqlOlx5spJf2Nr3R6CBL6rlnOcVR7SG+zJ5cQP+6ZRsuUJqiQPqnbWYBWYTECqug25QVyu5gBgXa8THBWexR6TwoKPPkVOUjzj0/zTyO5CAfcE8H7gxTr8gM0/TPL055yRdn/tmNr3Z3wWgKdlIa7IJfZTN6C1anLIaqXzkCx8jL/djMIkMBsYnuIk53yhvoCz3juE1L6+79pqqh5QqGwnwiHjh3OrQ0kRwfnrCJ1OjXbaJX9zeR7FpPOYoWB47KRMfOiu4sxgzJ8Z8ECwOxUvgN0/NuCSiV/BxFxSi8cMHFGIeifYXQ5OMcDz+Cj1yOt0AaESRqgK8zVdgzM+clyFDQnpk4ZWJh1tWTklcCGU4CPMFBJs3amg7KPJq2agVu4S1emFdGmX+dD09me9QsWsxNqskd3LiQu3t2rWbRjhlrCm9XzRV1VBVT2MJAXtwbNNVQCTR6BTlqRVKOCq3sOSu0xEZz7ux5tabyKrYFerbCf9p2u5wioy5K0vJ3aFIp3YD9wgR3U/jZ4B9PdjmJA4nwYhRTNvVGU4G4l7fPeY4OoN9DPxmfu5vk84ZgbUA/XA3oKjwG/LK4z2tFzwb5neFoLC7y+17L9o/I9o9LyQoBl5FN54izim52H49kBp/edut7D765BwI+BROtnkS6zfT6G8wk77eehJlO3y8z09J7rpMtoeXzQm+Q6w3kwCHMedoS8zrE3DGOWF7eBl72KSEw5Tp4VyBro68sBUJ3AQjrtdUlC7YHrYjSyHdogEa52H7V7qqnw8FZe5eFSXve59bAAv6d/jz61gX4wzX4/siuv28no/cfi9EbJtwYouMUxvKH50uJavRYn5juyUWzmapmPr/HsvNjItDpEAyCQFbHkQWf7wSfLQEKCz6WBjZOA63lvnVwwDCu5QHLAxtas+vcjkR+34Mp9oKM6aePqj1JnsV61/H3b7vMp1ki9IWbB4tcG1n7fA5mMzl3pzI+kFQh2H7eODnug8HwFJyqRssG6zIsd3wXdwSiDEe77GLp4wnQx8sF+ji6YynC3V8DPuyzgJY9Ns4enQsR6rEcvnlM8CXARJRNt54J3mfxCDFAQ9C0jMUBuwxhQcCCwHIQOPoRIFAvZVgSsCTwoCQAJxAT/TTbhmlAZw9lq/mWs8BzUKsKbrAYA1huT4C2tSwhWEKwhGAJYTkhuIc/BBEOLSJYRHiUjQrNCXICnCZ+/r7Ug2wyVFsM284LZ0hAFWPgXJsIfGgrbiFhLUigqd1NsIjwMz7Z2HfXQQS7n2ARYdPPNq58JkElegTHWKz7VMOqEh7pAYf/weMMPlFjEVCvkgBlQaAtiJPIEsj3EQhD/1oGsQzycz5PuRaE2J0MyyAP8g708qWKavne8xnKEULeSRofcuHZSH+Wba943fIBeCKvaD4HO/K4qH6++NLs9r2aUVoSHBeWtEBhlzSeAk5UX7KUfU51J9XqF73cNw73xyiGnxDj+cc1+we5WExJ8SVIdp3nlONu7fZig7WwmyP7+A2VGauqBIpTAjWWt76ZWX9FUodolR7mHdpJ6M1vbhn8su8VMqdxWYpl+LP64qJtXjlamRHquD2DZ16YJ25cHx8Yx6558qJnphjdsm8cu8W3Mq9KvdV4Ws9+n++sxSz40CzYrKW/b54E9fHLwNS31KVhsq9Uz0NO4c7qI1bak8VSrhzgdWqmAcdx3x73Li/3//6aSyd1mDyb/wfhH1MR8FUAAA== {{- end }} diff --git a/charts/kubezero-network/templates/multus/daemonset.yaml b/charts/kubezero-network/templates/multus/daemonset.yaml index e88cf5fd..f1313ae5 100644 --- a/charts/kubezero-network/templates/multus/daemonset.yaml +++ b/charts/kubezero-network/templates/multus/daemonset.yaml @@ -29,7 +29,7 @@ spec: - name: kube-multus image: {{ .Values.multus.image.repository }}:{{ .Values.multus.image.tag }} # Always used cached images - imagePullPolicy: Never + imagePullPolicy: {{ .Values.multus.image.pullPolicy }} command: ["/entrypoint.sh"] args: - "--multus-conf-file=/tmp/multus-conf/00-multus.conf" diff --git a/charts/kubezero-network/values.yaml b/charts/kubezero-network/values.yaml index 336d590f..53e2ff74 100644 --- a/charts/kubezero-network/values.yaml +++ b/charts/kubezero-network/values.yaml @@ -30,12 +30,11 @@ cilium: # Always use cached images image: useDigest: false - pullPolicy: Never resources: requests: cpu: 10m - memory: 256Mi + memory: 160Mi limits: memory: 1024Mi # cpu: 4000m @@ -60,7 +59,8 @@ cilium: # Keep it simple for now l7Proxy: false - + envoy: + enabled: false #rollOutCiliumPods: true cgroup: diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index 862dcabd..3957b32f 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero - Root App of Apps chart type: application -version: 1.29.7-1 +version: 1.30.5 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero/templates/_app.tpl b/charts/kubezero/templates/_app.tpl index c70dfe5f..41ab27b9 100644 --- a/charts/kubezero/templates/_app.tpl +++ b/charts/kubezero/templates/_app.tpl @@ -25,8 +25,8 @@ spec: repoURL: {{ .Values.kubezero.repoURL }} targetRevision: {{ default .Values.kubezero.targetRevision ( index .Values $name "targetRevision" ) | quote }} helm: - values: | -{{- include (print $name "-values") $ | nindent 8 }} + valuesObject: + {{- include (print $name "-values") $ | nindent 8 }} destination: server: {{ .Values.kubezero.server }} diff --git a/charts/kubezero/templates/addons.yaml b/charts/kubezero/templates/addons.yaml index 72085432..ec530463 100644 --- a/charts/kubezero/templates/addons.yaml +++ b/charts/kubezero/templates/addons.yaml @@ -1,5 +1,4 @@ {{- define "addons-values" }} - clusterBackup: enabled: {{ ternary "true" "false" (or (hasKey .Values.global.aws "region") .Values.addons.clusterBackup.enabled) }} diff --git a/charts/kubezero/templates/network.yaml b/charts/kubezero/templates/network.yaml index b4777061..c1c127b7 100644 --- a/charts/kubezero/templates/network.yaml +++ b/charts/kubezero/templates/network.yaml @@ -1,12 +1,21 @@ {{- define "network-values" }} - multus: enabled: true clusterNetwork: "cilium" + {{- if eq .Values.global.platform "aws" }} + image: + pullPolicy: Never + {{- end }} + cilium: enabled: true + {{- if eq .Values.global.platform "aws" }} + image: + pullPolicy: Never + {{- end }} + cluster: name: {{ .Values.global.clusterName }} {{- with .Values.network.cilium.cluster.id }} diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 26ba9218..9e690369 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -17,7 +17,7 @@ global: addons: enabled: true - targetRevision: 0.8.8 + targetRevision: 0.8.9 external-dns: enabled: false forseti: @@ -36,7 +36,7 @@ addons: network: enabled: true retain: true - targetRevision: 0.5.3 + targetRevision: 0.5.4 cilium: cluster: {} diff --git a/docs/v1.30.md b/docs/v1.30.md new file mode 100644 index 00000000..dc2a7a8e --- /dev/null +++ b/docs/v1.30.md @@ -0,0 +1,16 @@ +# ![k8s-v1.30](images/k8s-v130.png) KubeZero 1.30 - Uwubernetes + +## What's new - Major themes +- all KubeZero and support AMIs based on Alpine 3.20.3 +- reduced memory consumption of CNI agent on each node + + +## Version upgrades +- cilium 1.16.2 +- istio 1.22.3 +- ArgoCD 2.11.5 +- Prometheus 2.53 / Grafana 11.1 ( fixing many of the previous warnings ) +- ... + +## Resources +- [Kubernetes v1.30 upstream release blog](https://kubernetes.io/blog/2024/04/17/kubernetes-v1-30-release/)