V1.27: Istio bump to 1.19.4 incl. hardening settings, latest network CNI, initial operators and telemetry chart

This commit is contained in:
Stefan Reimer 2023-11-15 14:39:02 +00:00
parent 698df3b556
commit 214d758fbc
33 changed files with 361 additions and 52 deletions

View File

@ -150,13 +150,25 @@ echo "Adjust kubezero values as needed:"
# shellcheck disable=SC2015
argo_used && kubectl edit app kubezero -n argocd || kubectl edit cm kubezero-values -n kube-system
control_plane_upgrade "apply_network, apply_addons, apply_storage"
# We need to restore the network ready file as cilium decided to rename it
control_plane_upgrade apply_network
echo "Wait for all CNI agents to be running ..."
waitSystemPodsRunning
all_nodes_upgrade "cd /host/etc/cni/net.d && ln -s 05-cilium.conflist 05-cilium.conf || true"
# now the rest
control_plane_upgrade "apply_addons, apply_storage"
echo "Checking that all pods in kube-system are running ..."
waitSystemPodsRunning
echo "Applying remaining KubeZero modules..."
### Cleanup of some deprecated Istio Crds
for crd in clusterrbacconfigs.rbac.istio.io rbacconfigs.rbac.istio.io servicerolebindings.rbac.istio.io serviceroles.rbac.istio.io; do
kubectl delete crds $crd || true
done
control_plane_upgrade "apply_cert-manager, apply_istio, apply_istio-ingress, apply_istio-private-ingress, apply_logging, apply_metrics, apply_argocd"
# Trigger backup of upgraded cluster state

View File

@ -2,7 +2,7 @@ apiVersion: v2
name: kubezero-istio-gateway
description: KubeZero Umbrella Chart for Istio gateways
type: application
version: 0.18.3
version: 0.19.4
home: https://kubezero.com
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
keywords:

View File

@ -1,6 +1,6 @@
# kubezero-istio-gateway
![Version: 0.10.0](https://img.shields.io/badge/Version-0.10.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
![Version: 0.19.4](https://img.shields.io/badge/Version-0.19.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
KubeZero Umbrella Chart for Istio gateways
@ -16,12 +16,12 @@ Installs Istio Ingress Gateways, requires kubezero-istio to be installed !
## Requirements
Kubernetes: `>= 1.25.0`
Kubernetes: `>= 1.26.0`
| Repository | Name | Version |
|------------|------|---------|
| https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
| https://istio-release.storage.googleapis.com/charts | gateway | 1.17.2 |
| https://istio-release.storage.googleapis.com/charts | gateway | 1.19.4 |
## Values
@ -33,12 +33,14 @@ Kubernetes: `>= 1.25.0`
| gateway.autoscaling.minReplicas | int | `1` | |
| gateway.autoscaling.targetCPUUtilizationPercentage | int | `80` | |
| gateway.podAnnotations."proxy.istio.io/config" | string | `"{ \"terminationDrainDuration\": \"20s\" }"` | |
| gateway.priorityClassName | string | `"system-cluster-critical"` | |
| gateway.replicaCount | int | `1` | |
| gateway.resources.limits.memory | string | `"512Mi"` | |
| gateway.resources.requests.cpu | string | `"50m"` | |
| gateway.resources.requests.memory | string | `"64Mi"` | |
| gateway.service.externalTrafficPolicy | string | `"Local"` | |
| gateway.service.type | string | `"NodePort"` | |
| gateway.terminationGracePeriodSeconds | int | `120` | |
| proxyProtocol | bool | `true` | |
| telemetry.enabled | bool | `false` | |

View File

@ -1,5 +1,5 @@
apiVersion: v2
appVersion: 1.18.2
appVersion: 1.19.4
description: Helm chart for deploying Istio gateways
icon: https://istio.io/latest/favicons/android-192x192.png
keywords:
@ -9,4 +9,4 @@ name: gateway
sources:
- https://github.com/istio/istio
type: application
version: 1.18.2
version: 1.19.4

View File

@ -9,7 +9,9 @@ metadata:
{{- .Values.annotations | toYaml | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- with .Values.replicaCount }}
replicas: {{ . }}
{{- end }}
{{- end }}
# Give the LB 120s to detect and take into service
# should only be 40s by we are on AWS so ...
@ -34,7 +36,6 @@ spec:
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
terminationGracePeriodSeconds: 120
serviceAccountName: {{ include "gateway.serviceAccountName" . }}
securityContext:
{{- if .Values.securityContext }}
@ -113,6 +114,10 @@ spec:
topologySpreadConstraints:
{{- toYaml . | nindent 8 }}
{{- end }}
terminationGracePeriodSeconds: {{ $.Values.terminationGracePeriodSeconds }}
{{- with .Values.priorityClassName }}
priorityClassName: {{ . }}
{{- end }}
{{- with .Values.volumes }}
volumes:
{{- toYaml . | nindent 8 }}

View File

@ -15,6 +15,13 @@ spec:
{{- with .Values.service.loadBalancerIP }}
loadBalancerIP: "{{ . }}"
{{- end }}
{{- with .Values.service.ipFamilyPolicy }}
ipFamilyPolicy: "{{ . }}"
{{- end }}
{{- with .Values.service.ipFamilies }}
ipFamilies:
{{ toYaml . | indent 4 }}
{{- end }}
{{- with .Values.service.loadBalancerSourceRanges }}
loadBalancerSourceRanges:
{{ toYaml . | indent 4 }}

View File

@ -81,7 +81,7 @@
}
},
"replicaCount": {
"type": "integer"
"type": [ "integer", "null" ]
},
"resources": {
"type": "object",
@ -135,6 +135,16 @@
"loadBalancerSourceRanges": {
"type": "array"
},
"ipFamilies" : {
"items": {
"type": "string",
"enum": ["IPv4", "IPv6"]
}
},
"ipFamilyPolicy" : {
"type": "string",
"enum": ["", "SingleStack", "PreferDualStack", "RequireDualStack"]
},
"ports": {
"type": "array",
"items": {
@ -220,6 +230,12 @@
"enum": ["", "IfHealthyBudget", "AlwaysAllow"]
}
}
},
"terminationGracePeriodSeconds": {
"type": "number"
},
"priorityClassName": {
"type": "string"
}
}
}

View File

@ -3,7 +3,9 @@ name: ""
# revision declares which revision this gateway is a part of
revision: ""
replicaCount: 1
# Controls the spec.replicas setting for the Gateway deployment if set.
# Otherwise defaults to Kubernetes Deployment default (1).
replicaCount:
kind: Deployment
@ -55,6 +57,8 @@ service:
loadBalancerSourceRanges: []
externalTrafficPolicy: ""
externalIPs: []
ipFamilyPolicy: ""
ipFamilies: []
resources:
requests:
@ -125,3 +129,11 @@ imagePullSecrets: []
# podDisruptionBudget: {}
#
podDisruptionBudget: {}
terminationGracePeriodSeconds: 30
# Configure this to a higher priority class in order to make sure your Istio gateway pods
# will not be killed because of low priority class.
# Refer to https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass
# for more detail.
priorityClassName: ""

View File

@ -11,14 +11,6 @@ diff -tubr charts/gateway.orig/templates/deployment.yaml charts/gateway/template
selector:
matchLabels:
{{- include "gateway.selectorLabels" . | nindent 6 }}
@@ -31,6 +34,7 @@
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
+ terminationGracePeriodSeconds: 120
serviceAccountName: {{ include "gateway.serviceAccountName" . }}
securityContext:
{{- if .Values.securityContext }}
@@ -86,6 +90,10 @@
name: http-envoy-prom
resources:

View File

@ -18,7 +18,7 @@ data:
{
"name": "envoy.resource_monitors.fixed_heap",
"threshold": {
"value": 0.9
"value": 0.92
}
}
]
@ -29,7 +29,7 @@ data:
{
"name": "envoy.resource_monitors.fixed_heap",
"threshold": {
"value": 0.99
"value": 0.98
}
}
]

View File

@ -28,11 +28,16 @@ spec:
name: "envoy.filters.network.http_connection_manager"
typed_config:
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager"
stat_prefix: ingress_http
use_remote_address: true
normalize_path: true
merge_slashes: true
path_with_escaped_slashes_action: UNESCAPE_AND_REDIRECT
common_http_protocol_options:
idle_timeout: 3600s # 1 hour
# headers_with_underscores_action: REJECT_REQUEST
http2_protocol_options:
max_concurrent_streams: 500
max_concurrent_streams: 100
initial_stream_window_size: 65536 # 64 KiB
initial_connection_window_size: 1048576 # 1 MiB
#stream_idle_timeout: 300s # 5 mins, must be disabled for long-lived and streaming requests

View File

@ -1,4 +1,3 @@
{{- if .Values.proxyProtocol }}
apiVersion: networking.istio.io/v1alpha3
kind: EnvoyFilter
metadata:
@ -16,6 +15,11 @@ spec:
operation: MERGE
value:
listener_filters:
- name: envoy.filters.listener.proxy_protocol
- name: envoy.filters.listener.tls_inspector
{{- end }}
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector
{{- if .Values.proxyProtocol }}
- name: envoy.filters.listener.proxy_protocol
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.listener.proxy_protocol.v3.ProxyProtocol
{{- end }}

View File

@ -9,3 +9,5 @@ update_helm
export ISTIO_VERSION=$(yq eval '.dependencies[] | select(.name=="gateway") | .version' Chart.yaml)
patch_chart gateway
update_docs

View File

@ -7,6 +7,9 @@ gateway:
replicaCount: 1
terminationGracePeriodSeconds: 120
priorityClassName: system-cluster-critical
resources:
requests:
cpu: 50m

View File

@ -2,7 +2,7 @@ apiVersion: v2
name: kubezero-istio
description: KubeZero Umbrella Chart for Istio
type: application
version: 0.18.3
version: 0.19.4
home: https://kubezero.com
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
keywords:

View File

@ -1,6 +1,6 @@
# kubezero-istio
![Version: 0.18.2](https://img.shields.io/badge/Version-0.18.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
![Version: 0.19.4](https://img.shields.io/badge/Version-0.19.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
KubeZero Umbrella Chart for Istio
@ -21,9 +21,9 @@ Kubernetes: `>= 1.26.0`
| Repository | Name | Version |
|------------|------|---------|
| https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
| https://istio-release.storage.googleapis.com/charts | base | 1.18.2 |
| https://istio-release.storage.googleapis.com/charts | istiod | 1.18.2 |
| https://kiali.org/helm-charts | kiali-server | 1.72.0 |
| https://istio-release.storage.googleapis.com/charts | base | 1.19.4 |
| https://istio-release.storage.googleapis.com/charts | istiod | 1.19.4 |
| https://kiali.org/helm-charts | kiali-server | 1.76.0 |
## Values

View File

@ -10,7 +10,9 @@ export ISTIO_VERSION=$(yq eval '.dependencies[] | select(.name=="base") | .versi
export KIALI_VERSION=$(yq eval '.dependencies[] | select(.name=="kiali-server") | .version' Chart.yaml)
# Get matching istioctl
# [ -x istioctl ] && [ "$(./istioctl version --remote=false)" == $ISTIO_VERSION ] || { curl -sL https://github.com/istio/istio/releases/download/${ISTIO_VERSION}/istioctl-${ISTIO_VERSION}-linux-amd64.tar.gz | tar xz; chmod +x istioctl; }
[ -x istioctl ] && [ "$(./istioctl version --remote=false)" == $ISTIO_VERSION ] || { curl -sL https://github.com/istio/istio/releases/download/${ISTIO_VERSION}/istioctl-${ISTIO_VERSION}-linux-amd64.tar.gz | tar xz; chmod +x istioctl; }
# Fetch dashboards from Grafana.com and update ZDT CM
../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/grafana-dashboards.yaml
update_docs

View File

@ -2,7 +2,7 @@ apiVersion: v2
name: kubezero-network
description: KubeZero umbrella chart for all things network
type: application
version: 0.4.5
version: 0.4.6
home: https://kubezero.com
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
keywords:
@ -19,11 +19,11 @@ dependencies:
version: ">= 0.1.6"
repository: https://cdn.zero-downtime.net/charts/
- name: cilium
version: 1.13.5
version: 1.14.4
repository: https://helm.cilium.io/
condition: cilium.enabled
- name: metallb
version: 0.13.9
version: 0.13.12
repository: https://metallb.github.io/metallb
condition: metallb.enabled
kubeVersion: ">= 1.26.0"

View File

@ -1,6 +1,6 @@
# kubezero-network
![Version: 0.4.4](https://img.shields.io/badge/Version-0.4.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
![Version: 0.4.6](https://img.shields.io/badge/Version-0.4.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
KubeZero umbrella chart for all things network
@ -14,13 +14,13 @@ KubeZero umbrella chart for all things network
## Requirements
Kubernetes: `>= 1.25.0`
Kubernetes: `>= 1.26.0`
| Repository | Name | Version |
|------------|------|---------|
| https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
| https://helm.cilium.io/ | cilium | 1.13.4 |
| https://metallb.github.io/metallb | metallb | 0.13.9 |
| https://helm.cilium.io/ | cilium | 1.14.4 |
| https://metallb.github.io/metallb | metallb | 0.13.12 |
## Values
@ -65,5 +65,6 @@ Kubernetes: `>= 1.25.0`
| multus.clusterNetwork | string | `"cilium"` | |
| multus.defaultNetworks | list | `[]` | |
| multus.enabled | bool | `false` | |
| multus.readinessindicatorfile | string | `"/etc/cni/net.d/05-cilium.conf"` | |
| multus.tag | string | `"v3.9.3"` | |
| multus.image.repository | string | `"ghcr.io/k8snetworkplumbingwg/multus-cni"` | |
| multus.image.tag | string | `"v3.9.3"` | |
| multus.readinessindicatorfile | string | `"/etc/cni/net.d/05-cilium.conflist"` | |

View File

@ -37,10 +37,10 @@ spec:
resources:
requests:
cpu: "100m"
memory: "50Mi"
memory: "32Mi"
limits:
cpu: "100m"
memory: "50Mi"
#cpu: "100m"
memory: "64Mi"
securityContext:
privileged: true
capabilities:

View File

@ -1,6 +1,12 @@
#!/bin/bash
set -ex
helm dep update
. ../../scripts/lib-update.sh
#login_ecr_public
update_helm
# Create ZDT dashboard configmap
../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/grafana-dashboards.yaml
update_docs

View File

@ -22,7 +22,7 @@ multus:
clusterNetwork: "cilium"
defaultNetworks: []
readinessindicatorfile: "/etc/cni/net.d/05-cilium.conf"
readinessindicatorfile: "/etc/cni/net.d/05-cilium.conflist"
cilium:
enabled: false
@ -45,9 +45,9 @@ cilium:
#-- Ensure this is false if multus is enabled
exclusive: false
#bpf:
# hostLegacyRouting: true
# tproxy: false
# bpf:
# autoMount:
# enabled: false
cluster:
# This should match the second octet of clusterPoolIPv4PodCIDRList
@ -71,7 +71,7 @@ cilium:
enabled: false
hostRoot: "/sys/fs/cgroup"
tunnel: geneve
tunnelProtocol: geneve
prometheus:
enabled: false

View File

@ -0,0 +1,23 @@
apiVersion: v2
name: kubezero-operators
description: Various operators supported by KubeZero
type: application
version: 0.1.0
home: https://kubezero.com
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
keywords:
- kubezero
- operators
- opensearch
maintainers:
- name: Stefan Reimer
email: stefan@zero-downtime.net
dependencies:
- name: kubezero-lib
version: ">= 0.1.6"
repository: https://cdn.zero-downtime.net/charts/
- name: opensearch-operator
version: 2.4.0
repository: https://opster.github.io/opensearch-k8s-operator/
condition: opensearch-operator.enabled
kubeVersion: ">= 1.26.0"

View File

@ -0,0 +1,7 @@
#!/bin/bash
set -ex
. ../../scripts/lib-update.sh
#login_ecr_public
update_helm

View File

@ -0,0 +1,15 @@
opensearch-operator:
enabled: false
# otherwise service names will be >63 chars
fullnameOverride: telemetry
# not needed for now
kubeRbacProxy:
enable: false
tolerations:
- key: node-role.kubernetes.io/control-plane
effect: NoSchedule
nodeSelector:
node-role.kubernetes.io/control-plane: ""

View File

@ -0,0 +1,28 @@
apiVersion: v2
name: kubezero-telemetry
description: KubeZero Umbrella Chart for OpenTelemetry, Jaeger etc.
type: application
version: 0.1.0
home: https://kubezero.com
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
keywords:
- kubezero
- otel
- OpenTelemetry
- jaeger
maintainers:
- name: Stefan Reimer
email: stefan@zero-downtime.net
dependencies:
- name: kubezero-lib
version: ">= 0.1.6"
repository: https://cdn.zero-downtime.net/charts/
- name: opentelemetry-collector
version: 0.73.1
repository: https://open-telemetry.github.io/opentelemetry-helm-charts
condition: opentelemetry-collector.enabled
- name: jaeger
version: 0.72.0
repository: https://jaegertracing.github.io/helm-charts
condition: jaeger.enabled
kubeVersion: ">= 1.26.0"

View File

@ -0,0 +1,20 @@
{{- if .Values.jaeger.istio.enabled }}
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
name: {{ include "kubezero-lib.fullname" . }}
namespace: {{ .Release.Namespace }}
labels:
{{- include "kubezero-lib.labels" . | nindent 4 }}
spec:
gateways:
- {{ .Values.jaeger.istio.gateway }}
hosts:
- {{ .Values.jaeger.istio.url }}
http:
- route:
- destination:
host: {{ .Release.Name }}-jaeger-query
port:
number: 16686
{{- end }}

View File

@ -0,0 +1,39 @@
{{- if .Values.opensearch.nodeSets }}
apiVersion: opensearch.opster.io/v1
kind: OpenSearchCluster
metadata:
name: {{ template "kubezero-lib.fullname" . }}
namespace: {{ .Release.Namespace }}
labels:
{{ include "kubezero-lib.labels" . | nindent 4 }}
spec:
general:
serviceName: {{ template "kubezero-lib.fullname" . }}
version: 2.11.0
dashboards:
enable: true
version: 2.11.0
replicas: 1
resources:
requests:
memory: "512Mi"
cpu: "200m"
limits:
memory: "512Mi"
cpu: "200m"
nodePools:
- component: nodes
replicas: 2
diskSize: "16Gi"
nodeSelector:
resources:
requests:
memory: "2Gi"
cpu: "500m"
limits:
memory: "2Gi"
cpu: "500m"
roles:
- "cluster_manager"
- "data"
{{- end }}

View File

@ -0,0 +1,7 @@
#!/bin/bash
set -ex
. ../../scripts/lib-update.sh
#login_ecr_public
update_helm

View File

@ -0,0 +1,34 @@
opentelemetry-collector:
enabled: false
mode: deployment
jaeger:
enabled: false
# allInOne:
# enabled: true
# storage:
# type: none
# collector:
# enabled: false
# query:
# enabled: false
agent:
enabled: false
storage:
type: elasticsearch
provisionDataStore:
cassandra: false
elasticsearch: false
istio:
enabled: false
gateway: istio-ingress/private-ingressgateway
url: jaeger.example.com
opensearch:
nodeSets: {}

View File

@ -0,0 +1,15 @@
{{- define "operators-values" }}
{{- with index .Values "operators" "opensearch-operator" }}
opensearch-operator:
{{- toYaml . | nindent 2 }}
{{- end }}
{{- end }}
{{- define "operators-argo" }}
{{- end }}
{{ include "kubezero-app.app" . }}

View File

@ -0,0 +1,42 @@
{{- define "telemetry-values" }}
{{- if .Values.telemetry.jaeger }}
jaeger:
{{- with .Values.telemetry.jaeger }}
{{- toYaml . | nindent 2 }}
{{- end }}
{{- end }}
{{- if .Values.telemetry.opensearch }}
opensearch:
{{- if .Values.telemetry.opensearch.nodeSets }}
nodeSets:
{{- with .Values.telemetry.opensearch.nodeSets }}
{{- toYaml . | nindent 2 }}
{{- end }}
{{- end }}
prometheus: {{ .Values.metrics.enabled }}
{{- if .Values.telemetry.opensearch.s3Snapshot }}
s3Snapshot:
{{- with .Values.telemetry.opensearch.s3Snapshot }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.telemetry.dashboard }}
dashboard:
{{- with .Values.telemetry.dashboard }}
{{- toYaml . | nindent 2 }}
{{- end }}
{{- end }}
{{- end }}
{{- define "telemetry-argo" }}
{{- end }}
{{ include "kubezero-app.app" . }}

View File

@ -31,7 +31,7 @@ addons:
network:
enabled: true
retain: true
targetRevision: 0.4.5
targetRevision: 0.4.6
cilium:
cluster: {}
@ -59,13 +59,13 @@ storage:
istio:
enabled: false
namespace: istio-system
targetRevision: 0.18.2
targetRevision: 0.19.4
istio-ingress:
enabled: false
chart: kubezero-istio-gateway
namespace: istio-ingress
targetRevision: 0.18.2
targetRevision: 0.19.4
gateway:
service: {}
@ -77,6 +77,16 @@ istio-private-ingress:
gateway:
service: {}
telemetry:
enabled: false
namespace: telemetry
targetRevision: 0.1.0
operators:
enabled: false
namespace: operators
targetRevision: 0.1.0
metrics:
enabled: false
namespace: monitoring