From 6b342e72dbe14480bd5e4cd0f0dcb5e08ad96f1e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 29 Jul 2020 14:46:55 +0100 Subject: [PATCH 01/96] Calico cleanup, add efs-csi --- charts/kubezero-calico/test-pod.yaml | 34 +++++++++++++++++++++++++++ charts/kubezero/Chart.yaml | 4 ++-- charts/kubezero/README.md | 4 +--- charts/kubezero/templates/calico.yaml | 4 +--- charts/kubezero/values.yaml | 3 --- deploy/templates/values.yaml | 28 +++++++++++++--------- deploy/values.yaml | 10 +++++--- 7 files changed, 62 insertions(+), 25 deletions(-) create mode 100644 charts/kubezero-calico/test-pod.yaml diff --git a/charts/kubezero-calico/test-pod.yaml b/charts/kubezero-calico/test-pod.yaml new file mode 100644 index 0000000..e3c4f35 --- /dev/null +++ b/charts/kubezero-calico/test-pod.yaml @@ -0,0 +1,34 @@ +# Once pod is running: +# kubectl -n NAME-SPACE-TO-TEST exec -it pod/POD_NAME /bin/sh + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: netshoot + namespace: kube-system + labels: + app: netshoot +spec: + replicas: 2 + strategy: + type: Recreate + selector: + matchLabels: + app: netshoot + template: + metadata: + labels: + app: netshoot + spec: + containers: + - name: netshoot + image: nicolaka/netshoot + imagePullPolicy: Always + command: + - /bin/sleep + args: + - "3600" + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index 262a6c2..7d00c53 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero ArgoCD Application - Root App of Apps chart of KubeZero type: application -version: 0.3.10 +version: 0.3.11 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -15,4 +15,4 @@ dependencies: - name: kubezero-lib version: ">= 0.1.1" repository: https://zero-down-time.github.io/kubezero/ -kubeVersion: ">= 1.16.0" +kubeVersion: ">= 1.16.12" diff --git a/charts/kubezero/README.md b/charts/kubezero/README.md index bb93bec..7fe23c6 100644 --- a/charts/kubezero/README.md +++ b/charts/kubezero/README.md @@ -2,7 +2,7 @@ kubezero ======== KubeZero ArgoCD Application - Root App of Apps chart of KubeZero -Current chart version is `0.3.10` +Current chart version is `0.3.11` Source code can be found [here](https://kubezero.com) @@ -19,8 +19,6 @@ Source code can be found [here](https://kubezero.com) | aws-ebs-csi-driver.enabled | bool | `false` | | | aws-efs-csi-driver.enabled | bool | `false` | | | calico.enabled | bool | `true` | | -| calico.type | string | `"kustomize"` | | -| calico.values.migration | bool | `false` | | | cert-manager.enabled | bool | `true` | | | global.defaultDestination.server | string | `"https://kubernetes.default.svc"` | | | global.defaultSource.pathPrefix | string | `""` | | diff --git a/charts/kubezero/templates/calico.yaml b/charts/kubezero/templates/calico.yaml index a56ad5e..91eebd5 100644 --- a/charts/kubezero/templates/calico.yaml +++ b/charts/kubezero/templates/calico.yaml @@ -1,10 +1,8 @@ {{- if .Values.calico.enabled }} -{{ template "kubezero-app.app" dict "root" . "name" "calico" "type" .Values.calico.type "retain" true }} - {{- if not .Values.calico.values.migration }} +{{ template "kubezero-app.app" dict "root" . "name" "calico" "type" "helm" "retain" true }} syncPolicy: automated: prune: true - {{- end }} ignoreDifferences: - group: apiextensions.k8s.io diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 5fdbdac..dba7782 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -17,9 +17,6 @@ platform: aws calico: enabled: true - type: kustomize - values: - migration: false cert-manager: enabled: true diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 48cfca2..8d00320 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -7,11 +7,9 @@ kubezero: enabled: {{ .Values.calico.enabled }} type: {{ default "kustomize" .Values.calico.type }} values: - migration: {{ default false .Values.calico.migration }} - prometheus: false - # prometheus: {{ .Values.prometheus.enabled }} network: {{ default "vxlan" .Values.calico.network }} mtu: {{ default "8941" .Values.calico.mtu }} + prometheus: {{ .Values.prometheus.enabled }} cert-manager: enabled: {{ index .Values "cert-manager" "enabled" }} {{- if not .Values.bootstrap }} @@ -52,6 +50,7 @@ kubezero: iam.amazonaws.com/role: "{{ index .Values "aws-ebs-csi-driver" "IamArn" }}" extraVolumeTags: Name: {{ .Values.ClusterName }} + kiam: enabled: {{ .Values.kiam.enabled }} values: @@ -63,6 +62,17 @@ kubezero: prometheus: servicemonitor: enabled: {{ .Values.prometheus.enabled }} + + aws-efs-csi-driver: + enabled: {{ index .Values "aws-efs-csi-driver" "enabled" }} + values: + aws-efs-csi-driver: + nodeSelector: + node.kubernetes.io/csi.efs.fs: {{ index .Values "aws-efs-csi-driver" "EfsId" }} + PersistentVolume: + create: true + EfsId: {{ index .Values "aws-efs-csi-driver" "EfsId" }} + Name: {{ default "kubezero-efs-pv" ( index .Values "aws-efs-csi-driver" "PVName" ) }} {{- end }} istio: @@ -70,14 +80,10 @@ kubezero: values: istiod: replicaCount: {{ ternary 2 1 .Values.HighAvailableControlplane }} + {{- if .Values.istio.ingress }} ingress: - private: {{ .Values.istio.ingress.private }} - type: {{ .Values.istio.ingress.type }} - replicaCount: {{ default 2 .Values.istio.ingress.replicaCount }} - dnsNames: - {{- with .Values.istio.ingress.dnsNames }} - {{- . | toYaml | nindent 8 }} - {{- end }} + {{- toYaml .Values.istio.ingress | nindent 8 }} + {{- end }} {{- end }} @@ -90,7 +96,7 @@ argo-cd: configs: {{- toYaml . | nindent 4 }} {{- end }} - {{- if and ( not .Values.bootstrap ) .Values.istio.enabled }} + {{- if and ( not .Values.bootstrap ) ( index .Values "argo-cd" "istio" ) }} istio: {{- with index .Values "argo-cd" "istio" }} {{- toYaml . | nindent 4 }} diff --git a/deploy/values.yaml b/deploy/values.yaml index dceb6f7..0c00f1f 100644 --- a/deploy/values.yaml +++ b/deploy/values.yaml @@ -1,12 +1,11 @@ ClusterName: Test-cluster Domain: example.com -aws: true +platform: aws HighAvailableControlplane: false calico: enabled: true - migration: false cert-manager: enabled: true @@ -16,6 +15,9 @@ aws-ebs-csi-driver: enabled: true IamArn: "" +aws-efs-csi-driver: + enabled: false + kiam: enabled: true IamArn: "" @@ -28,4 +30,6 @@ prometheus: argo-cd: server: {} - gateway: private-ingressgateway.istio-system.svc.cluster.local + istio: + enabled: true + gateway: private-ingressgateway.istio-system.svc.cluster.local -- 2.40.1 From 4468cafd52d4782efc64aa3b985e2732feb779a8 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 29 Jul 2020 15:07:41 +0100 Subject: [PATCH 02/96] More cleanup, kiam doc update --- artifacts/kubezero-calico/README.md | 31 - .../kubezero-calico/aws/calico-v3.15.patch | 101 - artifacts/kubezero-calico/aws/calico.yaml | 3733 ---------------- artifacts/kubezero-calico/canal.yaml | 3749 ----------------- artifacts/kubezero-calico/kustomization.yaml | 8 - artifacts/kubezero-calico/logging.yaml | 16 - artifacts/kubezero-calico/prometheus.yaml | 14 - .../kubezero-calico/remove-namespace.patch | 50 - charts/kubezero-kiam/README.md | 5 +- charts/kubezero-kiam/README.md.gotmpl | 5 +- charts/kubezero-kiam/kiam_architecure.png | Bin 0 -> 43992 bytes 11 files changed, 8 insertions(+), 7704 deletions(-) delete mode 100644 artifacts/kubezero-calico/README.md delete mode 100644 artifacts/kubezero-calico/aws/calico-v3.15.patch delete mode 100644 artifacts/kubezero-calico/aws/calico.yaml delete mode 100644 artifacts/kubezero-calico/canal.yaml delete mode 100644 artifacts/kubezero-calico/kustomization.yaml delete mode 100644 artifacts/kubezero-calico/logging.yaml delete mode 100644 artifacts/kubezero-calico/prometheus.yaml delete mode 100644 artifacts/kubezero-calico/remove-namespace.patch create mode 100644 charts/kubezero-kiam/kiam_architecure.png diff --git a/artifacts/kubezero-calico/README.md b/artifacts/kubezero-calico/README.md deleted file mode 100644 index c730a4c..0000000 --- a/artifacts/kubezero-calico/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Calico CNI - -Current top-level still contains the deprecated Canal implementation. -Removed once new AWS config is tested and rolled out to all existing clusters. - -## AWS -Calico is setup based on the upstream calico-vxlan config from -`https://docs.projectcalico.org/v3.15/manifests/calico-vxlan.yaml` - -Changes: - -- VxLAN set to Always to not expose cluster communication to VPC - - -> EC2 SecurityGroups still apply and only need to allow UDP 4789 for VxLAN traffic - -> No need to disable source/destination check on EC2 instances - -> Prepared for optional WireGuard encryption for all inter node traffic - -- MTU set to 8941 - -- Removed migration init-container - -- Disable BGB and BIRD health checks - -- Set FELIX log level to warning - -- Enable Prometheus metrics - - -## Prometheus - -See: https://grafana.com/grafana/dashboards/12175 diff --git a/artifacts/kubezero-calico/aws/calico-v3.15.patch b/artifacts/kubezero-calico/aws/calico-v3.15.patch deleted file mode 100644 index 786d290..0000000 --- a/artifacts/kubezero-calico/aws/calico-v3.15.patch +++ /dev/null @@ -1,101 +0,0 @@ ---- calico-vxlan.yaml 2020-07-03 15:32:40.740506882 +0100 -+++ calico.yaml 2020-07-03 15:27:47.651499841 +0100 -@@ -10,13 +10,13 @@ - # Typha is disabled. - typha_service_name: "none" - # Configure the backend to use. -- calico_backend: "bird" -+ calico_backend: "vxlan" - # Configure the MTU to use for workload interfaces and tunnels. - # - If Wireguard is enabled, set to your network MTU - 60 - # - Otherwise, if VXLAN or BPF mode is enabled, set to your network MTU - 50 - # - Otherwise, if IPIP is enabled, set to your network MTU - 20 - # - Otherwise, if not using any encapsulation, set to your network MTU. -- veth_mtu: "1410" -+ veth_mtu: "8941" - - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. -@@ -3451,29 +3451,6 @@ - terminationGracePeriodSeconds: 0 - priorityClassName: system-node-critical - initContainers: -- # This container performs upgrade from host-local IPAM to calico-ipam. -- # It can be deleted if this is a fresh installation, or if you have already -- # upgraded to use calico-ipam. -- - name: upgrade-ipam -- image: calico/cni:v3.15.0 -- command: ["/opt/cni/bin/calico-ipam", "-upgrade"] -- env: -- - name: KUBERNETES_NODE_NAME -- valueFrom: -- fieldRef: -- fieldPath: spec.nodeName -- - name: CALICO_NETWORKING_BACKEND -- valueFrom: -- configMapKeyRef: -- name: calico-config -- key: calico_backend -- volumeMounts: -- - mountPath: /var/lib/cni/networks -- name: host-local-net-dir -- - mountPath: /host/opt/cni/bin -- name: cni-bin-dir -- securityContext: -- privileged: true - # This container installs the CNI binaries - # and CNI network config file on each node. - - name: install-cni -@@ -3545,7 +3522,7 @@ - key: calico_backend - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE -- value: "k8s,bgp" -+ value: "k8s,kubeadm" - # Auto-detect the BGP IP address. - - name: IP - value: "autodetect" -@@ -3554,7 +3531,7 @@ - value: "Never" - # Enable or Disable VXLAN on the default IP pool. - - name: CALICO_IPV4POOL_VXLAN -- value: "CrossSubnet" -+ value: "Always" - # Set MTU for tunnel device used if ipip is enabled - - name: FELIX_IPINIPMTU - valueFrom: -@@ -3595,9 +3572,17 @@ - value: "false" - # Set Felix logging to "info" - - name: FELIX_LOGSEVERITYSCREEN -- value: "info" -+ value: "Warning" -+ - name: FELIX_LOGSEVERITYFILE -+ value: "Warning" -+ - name: FELIX_LOGSEVERITYSYS -+ value: "" - - name: FELIX_HEALTHENABLED - value: "true" -+ - name: FELIX_PROMETHEUSGOMETRICSENABLED -+ value: "false" -+ - name: FELIX_PROMETHEUSMETRICSENABLED -+ value: "true" - securityContext: - privileged: true - resources: -@@ -3608,7 +3593,6 @@ - command: - - /bin/calico-node - - -felix-live -- - -bird-live - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 -@@ -3617,7 +3601,6 @@ - command: - - /bin/calico-node - - -felix-ready -- - -bird-ready - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules diff --git a/artifacts/kubezero-calico/aws/calico.yaml b/artifacts/kubezero-calico/aws/calico.yaml deleted file mode 100644 index de2d34e..0000000 --- a/artifacts/kubezero-calico/aws/calico.yaml +++ /dev/null @@ -1,3733 +0,0 @@ ---- -# Source: calico/templates/calico-config.yaml -# This ConfigMap is used to configure a self-hosted Calico installation. -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-config - namespace: kube-system -data: - # Typha is disabled. - typha_service_name: "none" - # Configure the backend to use. - calico_backend: "vxlan" - # Configure the MTU to use for workload interfaces and tunnels. - # - If Wireguard is enabled, set to your network MTU - 60 - # - Otherwise, if VXLAN or BPF mode is enabled, set to your network MTU - 50 - # - Otherwise, if IPIP is enabled, set to your network MTU - 20 - # - Otherwise, if not using any encapsulation, set to your network MTU. - veth_mtu: "8941" - - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": { - "type": "calico-ipam" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - }, - { - "type": "bandwidth", - "capabilities": {"bandwidth": true} - } - ] - } - ---- -# Source: calico/templates/kdd-crds.yaml - - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: bgpconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPConfiguration - listKind: BGPConfigurationList - plural: bgpconfigurations - singular: bgpconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: BGPConfiguration contains the configuration for any BGP routing. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPConfigurationSpec contains the values of the BGP configuration. - properties: - asNumber: - description: 'ASNumber is the default AS number used by a node. [Default: - 64512]' - format: int32 - type: integer - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: INFO]' - type: string - nodeToNodeMeshEnabled: - description: 'NodeToNodeMeshEnabled sets whether full node to node - BGP mesh is enabled. [Default: true]' - type: boolean - serviceClusterIPs: - description: ServiceClusterIPs are the CIDR blocks from which service - cluster IPs are allocated. If specified, Calico will advertise these - blocks, as well as any cluster IPs within them. - items: - description: ServiceClusterIPBlock represents a single whitelisted - CIDR block for ClusterIPs. - properties: - cidr: - type: string - type: object - type: array - serviceExternalIPs: - description: ServiceExternalIPs are the CIDR blocks for Kubernetes - Service External IPs. Kubernetes Service ExternalIPs will only be - advertised if they are within one of these blocks. - items: - description: ServiceExternalIPBlock represents a single whitelisted - CIDR External IP block. - properties: - cidr: - type: string - type: object - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: bgppeers.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPPeer - listKind: BGPPeerList - plural: bgppeers - singular: bgppeer - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPPeerSpec contains the specification for a BGPPeer resource. - properties: - asNumber: - description: The AS Number of the peer. - format: int32 - type: integer - node: - description: The node name identifying the Calico node instance that - is peering with this peer. If this is not set, this represents a - global peer, i.e. a peer that peers with every node in the deployment. - type: string - nodeSelector: - description: Selector for the nodes that should have this peering. When - this is set, the Node field must be empty. - type: string - peerIP: - description: The IP address of the peer. - type: string - peerSelector: - description: Selector for the remote nodes to peer with. When this - is set, the PeerIP and ASNumber fields must be empty. For each - peering between the local node and selected remote nodes, we configure - an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, - and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The - remote AS number comes from the remote node’s NodeBGPSpec.ASNumber, - or the global default if that is not set. - type: string - required: - - asNumber - - peerIP - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: blockaffinities.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BlockAffinity - listKind: BlockAffinityList - plural: blockaffinities - singular: blockaffinity - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BlockAffinitySpec contains the specification for a BlockAffinity - resource. - properties: - cidr: - type: string - deleted: - description: Deleted indicates that this block affinity is being deleted. - This field is a string for compatibility with older releases that - mistakenly treat this field as a string. - type: string - node: - type: string - state: - type: string - required: - - cidr - - deleted - - node - - state - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: clusterinformations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: ClusterInformation - listKind: ClusterInformationList - plural: clusterinformations - singular: clusterinformation - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: ClusterInformation contains the cluster specific information. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ClusterInformationSpec contains the values of describing - the cluster. - properties: - calicoVersion: - description: CalicoVersion is the version of Calico that the cluster - is running - type: string - clusterGUID: - description: ClusterGUID is the GUID of the cluster - type: string - clusterType: - description: ClusterType describes the type of the cluster - type: string - datastoreReady: - description: DatastoreReady is used during significant datastore migrations - to signal to components such as Felix that it should wait before - accessing the datastore. - type: boolean - variant: - description: Variant declares which variant of Calico should be active. - type: string - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: felixconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: FelixConfiguration - listKind: FelixConfigurationList - plural: felixconfigurations - singular: felixconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: Felix Configuration contains the configuration for Felix. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: FelixConfigurationSpec contains the values of the Felix configuration. - properties: - bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. [Default: - true]' - type: boolean - bpfDataIfacePattern: - description: 'BPFDataIfacePattern is a regular expression that controls - which interfaces Felix should attach BPF programs to in order to - catch traffic to/from the network. This needs to match the interfaces - that Calico workload traffic flows over as well as any interfaces - that handle incoming traffic to nodeports and services from outside - the cluster. It should not match the workload interfaces (usually - named cali...). [Default: ^(en.*|eth.*|tunl0$)]' - type: string - bpfDisableUnprivileged: - description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled - sysctl to disable unprivileged use of BPF. This ensures that unprivileged - users cannot access Calico''s BPF maps and cannot insert their own - BPF programs to interfere with Calico''s. [Default: true]' - type: boolean - bpfEnabled: - description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. - [Default: false]' - type: boolean - bpfExternalServiceMode: - description: 'BPFExternalServiceMode in BPF mode, controls how connections - from outside the cluster to services (node ports and cluster IPs) - are forwarded to remote workloads. If set to "Tunnel" then both - request and response traffic is tunneled to the remote node. If - set to "DSR", the request traffic is tunneled but the response traffic - is sent directly from the remote node. In "DSR" mode, the remote - node appears to use the IP of the ingress node; this requires a - permissive L2 network. [Default: Tunnel]' - type: string - bpfKubeProxyEndpointSlicesEnabled: - description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls - whether Felix's embedded kube-proxy accepts EndpointSlices or not. - type: boolean - bpfKubeProxyIptablesCleanupEnabled: - description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF - mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s - iptables chains. Should only be enabled if kube-proxy is not running. [Default: - true]' - type: boolean - bpfKubeProxyMinSyncPeriod: - description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the - minimum time between updates to the dataplane for Felix''s embedded - kube-proxy. Lower values give reduced set-up latency. Higher values - reduce Felix CPU usage by batching up more work. [Default: 1s]' - type: string - bpfLogLevel: - description: 'BPFLogLevel controls the log level of the BPF programs - when in BPF dataplane mode. One of "Off", "Info", or "Debug". The - logs are emitted to the BPF trace pipe, accessible with the command - `tc exec bpf debug`. [Default: Off].' - type: string - chainInsertMode: - description: 'ChainInsertMode controls whether Felix hooks the kernel’s - top-level iptables chains by inserting a rule at the top of the - chain or by appending a rule at the bottom. insert is the safe default - since it prevents Calico’s rules from being bypassed. If you switch - to append mode, be sure that the other rules in the chains signal - acceptance by falling through to the Calico rules, otherwise the - Calico policy will be bypassed. [Default: insert]' - type: string - dataplaneDriver: - type: string - debugDisableLogDropping: - type: boolean - debugMemoryProfilePath: - type: string - debugSimulateCalcGraphHangAfter: - type: string - debugSimulateDataplaneHangAfter: - type: string - defaultEndpointToHostAction: - description: 'DefaultEndpointToHostAction controls what happens to - traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables “DROP” action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables “INPUT” chain; - Calico will insert its rules at the top of that chain, then “RETURN” - packets to the “INPUT” chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' - type: string - deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. - type: integer - deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. - type: string - disableConntrackInvalidCheck: - type: boolean - endpointReportingDelay: - type: string - endpointReportingEnabled: - type: boolean - externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. - items: - type: string - type: array - failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a comma-delimited list of - UDP/TCP ports that Felix will allow incoming traffic to host endpoints - on irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. Each - port should be specified as tcp: or udp:. - For back-compatibility, if the protocol is not specified, it defaults - to “tcp”. To disable all inbound host ports, use the value none. - The default value allows ssh access and DHCP. [Default: tcp:22, - udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' - items: - description: ProtoPort is combination of protocol and port, both - must be specified. - properties: - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a comma-delimited list - of UDP/TCP ports that Felix will allow outgoing traffic from host - endpoints to irrespective of the security policy. This is useful - to avoid accidentally cutting off a host with incorrect configuration. - Each port should be specified as tcp: or udp:. - For back-compatibility, if the protocol is not specified, it defaults - to “tcp”. To disable all outbound host ports, use the value none. - The default value opens etcd’s standard ports to ensure that Felix - does not get cut off from etcd as well as allowing DHCP and DNS. - [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667, - udp:53, udp:67]' - items: - description: ProtoPort is combination of protocol and port, both - must be specified. - properties: - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - genericXDPEnabled: - description: 'GenericXDPEnabled enables Generic XDP so network cards - that don''t support XDP offload or driver modes can use XDP. This - is not recommended since it doesn''t provide better performance - than iptables. [Default: false]' - type: boolean - healthEnabled: - type: boolean - healthHost: - type: string - healthPort: - type: integer - interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' - type: string - interfacePrefix: - description: 'InterfacePrefix is the interface name prefix that identifies - workload endpoints and so distinguishes them from host endpoint - interfaces. Note: in environments other than bare metal, the orchestrators - configure this appropriately. For example our Kubernetes and Docker - integrations set the ‘cali’ value, and our OpenStack integration - sets the ‘tap’ value. [Default: cali]' - type: string - ipipEnabled: - type: boolean - ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' - type: integer - ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico’s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' - type: string - iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is legacy. - type: string - iptablesFilterAllowAction: - type: string - iptablesLockFilePath: - description: 'IptablesLockFilePath is the location of the iptables - lock file. You may need to change this if the lock file is not in - its standard location (for example if you have mapped it into Felix’s - container at a different path). [Default: /run/xtables.lock]' - type: string - iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' - type: string - iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' - type: string - iptablesMangleAllowAction: - type: string - iptablesMarkMask: - description: 'IptablesMarkMask is the mask that Felix selects its - IPTables Mark bits from. Should be a 32 bit hexadecimal number with - at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' - format: int32 - type: integer - iptablesNATOutgoingInterfaceFilter: - type: string - iptablesPostWriteCheckInterval: - description: 'IptablesPostWriteCheckInterval is the period after Felix - has done a write to the dataplane that it schedules an extra read - back in order to check the write was not clobbered by another process. - This should only occur if another application on the system doesn’t - respect the iptables lock. [Default: 1s]' - type: string - iptablesRefreshInterval: - description: 'IptablesRefreshInterval is the period at which Felix - re-checks the IP sets in the dataplane to ensure that no other process - has accidentally broken Calico’s rules. Set to 0 to disable IP sets - refresh. Note: the default for this value is lower than the other - refresh intervals as a workaround for a Linux kernel bug that was - fixed in kernel version 4.11. If you are using v4.11 or greater - you may want to set this to, a higher value to reduce Felix CPU - usage. [Default: 10s]' - type: string - ipv6Support: - type: boolean - kubeNodePortRanges: - description: 'KubeNodePortRanges holds list of port ranges used for - service node ports. Only used if felix detects kube-proxy running - in ipvs mode. Felix uses these ranges to separate host and workload - traffic. [Default: 30000:32767].' - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - logFilePath: - description: 'LogFilePath is the full path to the Felix log. Set to - none to disable file logging. [Default: /var/log/calico/felix.log]' - type: string - logPrefix: - description: 'LogPrefix is the log prefix that Felix uses when rendering - LOG rules. [Default: calico-packet]' - type: string - logSeverityFile: - description: 'LogSeverityFile is the log severity above which logs - are sent to the log file. [Default: Info]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - logSeveritySys: - description: 'LogSeveritySys is the log severity above which logs - are sent to the syslog. Set to None for no logging to syslog. [Default: - Info]' - type: string - maxIpsetSize: - type: integer - metadataAddr: - description: 'MetadataAddr is the IP address or domain name of the - server that can answer VM queries for cloud-init metadata. In OpenStack, - this corresponds to the machine running nova-api (or in Ubuntu, - nova-api-metadata). A value of none (case insensitive) means that - Felix should not set up any NAT rule for the metadata path. [Default: - 127.0.0.1]' - type: string - metadataPort: - description: 'MetadataPort is the port of the metadata server. This, - combined with global.MetadataAddr (if not ‘None’), is used to set - up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. - In most cases this should not need to be changed [Default: 8775].' - type: integer - natOutgoingAddress: - description: NATOutgoingAddress specifies an address to use when performing - source NAT for traffic in a natOutgoing pool that is leaving the - network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) - type: string - natPortRange: - anyOf: - - type: integer - - type: string - description: NATPortRange specifies the range of ports that is used - for port mapping when doing outgoing NAT. When unset the default - behavior of the network stack is used. - pattern: ^.* - x-kubernetes-int-or-string: true - netlinkTimeout: - type: string - openstackRegion: - description: 'OpenstackRegion is the name of the region that a particular - Felix belongs to. In a multi-region Calico/OpenStack deployment, - this must be configured somehow for each Felix (here in the datamodel, - or in felix.cfg or the environment on each compute node), and must - match the [calico] openstack_region value configured in neutron.conf - on each node. [Default: Empty]' - type: string - policySyncPathPrefix: - description: 'PolicySyncPathPrefix is used to by Felix to communicate - policy changes to external services, like Application layer policy. - [Default: Empty]' - type: string - prometheusGoMetricsEnabled: - description: 'PrometheusGoMetricsEnabled disables Go runtime metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - prometheusMetricsEnabled: - description: 'PrometheusMetricsEnabled enables the Prometheus metrics - server in Felix if set to true. [Default: false]' - type: boolean - prometheusMetricsHost: - description: 'PrometheusMetricsHost is the host that the Prometheus - metrics server should bind to. [Default: empty]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. [Default: 9091]' - type: integer - prometheusProcessMetricsEnabled: - description: 'PrometheusProcessMetricsEnabled disables process metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. - type: boolean - reportingInterval: - description: 'ReportingInterval is the interval at which Felix reports - its status into the datastore or 0 to disable. Must be non-zero - in OpenStack deployments. [Default: 30s]' - type: string - reportingTTL: - description: 'ReportingTTL is the time-to-live setting for process-wide - status reports. [Default: 90s]' - type: string - routeRefreshInterval: - description: 'RouterefreshInterval is the period at which Felix re-checks - the routes in the dataplane to ensure that no other process has - accidentally broken Calico’s rules. Set to 0 to disable route refresh. - [Default: 90s]' - type: string - routeSource: - description: 'RouteSource configures where Felix gets its routing - information. - WorkloadIPs: use workload endpoints to construct - routes. - CalicoIPAM: the default - use IPAM data to construct routes.' - type: string - routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - sidecarAccelerationEnabled: - description: 'SidecarAccelerationEnabled enables experimental sidecar - acceleration [Default: false]' - type: boolean - usageReportingEnabled: - description: 'UsageReportingEnabled reports anonymous Calico version - number and cluster size to projectcalico.org. Logs warnings returned - by the usage server. For example, if a significant security vulnerability - has been discovered in the version of Calico being used. [Default: - true]' - type: boolean - usageReportingInitialDelay: - description: 'UsageReportingInitialDelay controls the minimum delay - before Felix makes a report. [Default: 300s]' - type: string - usageReportingInterval: - description: 'UsageReportingInterval controls the interval at which - Felix makes reports. [Default: 86400s]' - type: string - useInternalDataplaneDriver: - type: boolean - vxlanEnabled: - type: boolean - vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' - type: integer - vxlanPort: - type: integer - vxlanVNI: - type: integer - wireguardEnabled: - description: 'WireguardEnabled controls whether Wireguard is enabled. - [Default: false]' - type: boolean - wireguardInterfaceName: - description: 'WireguardInterfaceName specifies the name to use for - the Wireguard interface. [Default: wg.calico]' - type: string - wireguardListeningPort: - description: 'WireguardListeningPort controls the listening port used - by Wireguard. [Default: 51820]' - type: integer - wireguardMTU: - description: 'WireguardMTU controls the MTU on the Wireguard interface. - See Configuring MTU [Default: 1420]' - type: integer - wireguardRoutingRulePriority: - description: 'WireguardRoutingRulePriority controls the priority value - to use for the Wireguard routing rule. [Default: 99]' - type: integer - xdpEnabled: - description: 'XDPEnabled enables XDP acceleration for suitable untracked - incoming deny rules. [Default: true]' - type: boolean - xdpRefreshInterval: - description: 'XDPRefreshInterval is the period at which Felix re-checks - all XDP state to ensure that no other process has accidentally broken - Calico''s BPF maps or attached programs. Set to 0 to disable XDP - refresh. [Default: 90s]' - type: string - required: - - bpfLogLevel - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: globalnetworkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkPolicy - listKind: GlobalNetworkPolicyList - plural: globalnetworkpolicies - singular: globalnetworkpolicy - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - applyOnForward: - description: ApplyOnForward indicates to apply the rules in this policy - on forward traffic. - type: boolean - doNotTrack: - description: DoNotTrack indicates whether packets matched by the rules - in this policy should go through the data plane's connection tracking, - such as Linux conntrack. If True, the rules in this policy are - applied before any data plane connection tracking, and packets allowed - by this policy are marked as not to be tracked. - type: boolean - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - required: - - action - type: object - type: array - namespaceSelector: - description: NamespaceSelector is an optional field for an expression - used to select a pod based on namespaces. - type: string - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - preDNAT: - description: PreDNAT indicates to apply the rules in this policy before - any DNAT. - type: boolean - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress rules are present in the policy. The - default is: \n - [ PolicyTypeIngress ], if there are no Egress rules - (including the case where there are also no Ingress rules) \n - - [ PolicyTypeEgress ], if there are Egress rules but no Ingress - rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are - both Ingress and Egress rules. \n When the policy is read back again, - Types will always be one of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: globalnetworksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkSet - listKind: GlobalNetworkSetList - plural: globalnetworksets - singular: globalnetworkset - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs - that share labels to allow rules to refer to them via selectors. The labels - of GlobalNetworkSet are not namespaced. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: GlobalNetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: hostendpoints.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: HostEndpoint - listKind: HostEndpointList - plural: hostendpoints - singular: hostendpoint - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: HostEndpointSpec contains the specification for a HostEndpoint - resource. - properties: - expectedIPs: - description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. - If \"InterfaceName\" is not present, Calico will look for an interface - matching any of the IPs in the list and apply policy to that. Note: - \tWhen using the selector match criteria in an ingress or egress - security Policy \tor Profile, Calico converts the selector into - a set of IP addresses. For host \tendpoints, the ExpectedIPs field - is used for that purpose. (If only the interface \tname is specified, - Calico does not learn the IPs of the interface for use in match - \tcriteria.)" - items: - type: string - type: array - interfaceName: - description: "Either \"*\", or the name of a specific Linux interface - to apply policy to; or empty. \"*\" indicates that this HostEndpoint - governs all traffic to, from or through the default network namespace - of the host named by the \"Node\" field; entering and leaving that - namespace via any interface, including those from/to non-host-networked - local workloads. \n If InterfaceName is not \"*\", this HostEndpoint - only governs traffic that enters or leaves the host through the - specific interface named by InterfaceName, or - when InterfaceName - is empty - through the specific interface that has one of the IPs - in ExpectedIPs. Therefore, when InterfaceName is empty, at least - one expected IP must be specified. Only external interfaces (such - as “eth0”) are supported here; it isn't possible for a HostEndpoint - to protect traffic through a specific local workload interface. - \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; - initially just pre-DNAT policy. Please check Calico documentation - for the latest position." - type: string - node: - description: The node name identifying the Calico node instance. - type: string - ports: - description: Ports contains the endpoint's named ports, which may - be referenced in security policy rules. - items: - properties: - name: - type: string - port: - type: integer - protocol: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - required: - - name - - port - - protocol - type: object - type: array - profiles: - description: A list of identifiers of security Profile objects that - apply to this endpoint. Each profile is applied in the order that - they appear in this list. Profile rules are applied after the selector-based - security policy. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipamblocks.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMBlock - listKind: IPAMBlockList - plural: ipamblocks - singular: ipamblock - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMBlockSpec contains the specification for an IPAMBlock - resource. - properties: - affinity: - type: string - allocations: - items: - type: integer - # TODO: This nullable is manually added in. We should update controller-gen - # to handle []*int properly itself. - nullable: true - type: array - attributes: - items: - properties: - handle_id: - type: string - secondary: - additionalProperties: - type: string - type: object - type: object - type: array - cidr: - type: string - deleted: - type: boolean - strictAffinity: - type: boolean - unallocated: - items: - type: integer - type: array - required: - - allocations - - attributes - - cidr - - deleted - - strictAffinity - - unallocated - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipamconfigs.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMConfig - listKind: IPAMConfigList - plural: ipamconfigs - singular: ipamconfig - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMConfigSpec contains the specification for an IPAMConfig - resource. - properties: - autoAllocateBlocks: - type: boolean - strictAffinity: - type: boolean - required: - - autoAllocateBlocks - - strictAffinity - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipamhandles.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMHandle - listKind: IPAMHandleList - plural: ipamhandles - singular: ipamhandle - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMHandleSpec contains the specification for an IPAMHandle - resource. - properties: - block: - additionalProperties: - type: integer - type: object - handleID: - type: string - required: - - block - - handleID - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ippools.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPPool - listKind: IPPoolList - plural: ippools - singular: ippool - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPPoolSpec contains the specification for an IPPool resource. - properties: - blockSize: - description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. - type: integer - cidr: - description: The pool CIDR. - type: string - disabled: - description: When disabled is true, Calico IPAM will not assign addresses - from this pool. - type: boolean - ipip: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - properties: - enabled: - description: When enabled is true, ipip tunneling will be used - to deliver packets to destinations within this pool. - type: boolean - mode: - description: The IPIP mode. This can be one of "always" or "cross-subnet". A - mode of "always" will also use IPIP tunneling for routing to - destination IP addresses within this pool. A mode of "cross-subnet" - will only use IPIP tunneling when the destination node is on - a different subnet to the originating node. The default value - (if not specified) is "always". - type: string - type: object - ipipMode: - description: Contains configuration for IPIP tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. IPIP tunelling - is disabled). - type: string - nat-outgoing: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - type: boolean - natOutgoing: - description: When nat-outgoing is true, packets sent from Calico networked - containers in this pool to destinations outside of this pool will - be masqueraded. - type: boolean - nodeSelector: - description: Allows IPPool to allocate for a specific node by label - selector. - type: string - vxlanMode: - description: Contains configuration for VXLAN tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. VXLAN - tunelling is disabled). - type: string - required: - - cidr - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: kubecontrollersconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: KubeControllersConfiguration - listKind: KubeControllersConfigurationList - plural: kubecontrollersconfigurations - singular: kubecontrollersconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KubeControllersConfigurationSpec contains the values of the - Kubernetes controllers configuration. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host endpoints. - Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation of - host endpoints for every node. [Default: Disabled]' - type: string - type: object - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - type: object - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - required: - - controllers - type: object - status: - description: KubeControllersConfigurationStatus represents the status - of the configuration. It's useful for admins to be able to see the actual - config that was applied, which can be modified by environment variables - on the kube-controllers process. - properties: - environmentVars: - additionalProperties: - type: string - description: EnvironmentVars contains the environment variables on - the kube-controllers that influenced the RunningConfig. - type: object - runningConfig: - description: RunningConfig contains the effective config that is running - in the kube-controllers pod, after merging the API resource with - any environment variables. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace - controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host - endpoints. Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation - of host endpoints for every node. [Default: Disabled]' - type: string - type: object - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - type: object - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which - logs are sent to the stdout. [Default: Info]' - type: string - required: - - controllers - type: object - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: networkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkPolicy - listKind: NetworkPolicyList - plural: networkpolicies - singular: networkpolicy - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - required: - - action - type: object - type: array - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress are present in the policy. The default - is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including - the case where there are also no Ingress rules) \n - [ PolicyTypeEgress - ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, - PolicyTypeEgress ], if there are both Ingress and Egress rules. - \n When the policy is read back again, Types will always be one - of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: networksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkSet - listKind: NetworkSetList - plural: networksets - singular: networkset - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- ---- -# Source: calico/templates/calico-kube-controllers-rbac.yaml - -# Include a clusterrole for the kube-controllers component, -# and bind it to the calico-kube-controllers serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -rules: - # Nodes are watched to monitor for deletions. - - apiGroups: [""] - resources: - - nodes - verbs: - - watch - - list - - get - # Pods are queried to check for existence. - - apiGroups: [""] - resources: - - pods - verbs: - - get - # IPAM resources are manipulated when nodes are deleted. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - verbs: - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - # kube-controllers manages hostendpoints. - - apiGroups: ["crd.projectcalico.org"] - resources: - - hostendpoints - verbs: - - get - - list - - create - - update - - delete - # Needs access to update clusterinformations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - clusterinformations - verbs: - - get - - create - - update - # KubeControllersConfiguration is where it gets its config - - apiGroups: ["crd.projectcalico.org"] - resources: - - kubecontrollersconfigurations - verbs: - # read its own config - - get - # create a default if none exists - - create - # update status - - update - # watch for changes - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers -subjects: -- kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system ---- - ---- -# Source: calico/templates/calico-node-rbac.yaml -# Include a clusterrole for the calico-node DaemonSet, -# and bind it to the calico-node serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-node -rules: - # The CNI plugin needs to get pods, nodes, and namespaces. - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - - services - verbs: - # Used to discover service IPs for advertisement. - - watch - - list - # Used to discover Typhas. - - get - # Pod CIDR auto-detection on kubeadm needs access to config maps. - - apiGroups: [""] - resources: - - configmaps - verbs: - - get - - apiGroups: [""] - resources: - - nodes/status - verbs: - # Needed for clearing NodeNetworkUnavailable flag. - - patch - # Calico stores some configuration information in node annotations. - - update - # Watch for changes to Kubernetes NetworkPolicies. - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - # Used by Calico for policy information. - - apiGroups: [""] - resources: - - pods - - namespaces - - serviceaccounts - verbs: - - list - - watch - # The CNI plugin patches pods/status. - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - # Calico monitors various CRDs for config. - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - ipamblocks - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - networksets - - clusterinformations - - hostendpoints - - blockaffinities - verbs: - - get - - list - - watch - # Calico must create and update some CRDs on startup. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - - felixconfigurations - - clusterinformations - verbs: - - create - - update - # Calico stores some configuration information on the node. - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - watch - # These permissions are only required for upgrade from v2.6, and can - # be removed after upgrade or on fresh installations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - bgpconfigurations - - bgppeers - verbs: - - create - - update - # These permissions are required for Calico CNI to perform IPAM allocations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - - apiGroups: ["crd.projectcalico.org"] - resources: - - ipamconfigs - verbs: - - get - # Block affinities must also be watchable by confd for route aggregation. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - verbs: - - watch - # The Calico IPAM migration needs to get daemonsets. These permissions can be - # removed if not upgrading from an installation using host-local IPAM. - - apiGroups: ["apps"] - resources: - - daemonsets - verbs: - - get - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system - ---- -# Source: calico/templates/calico-node.yaml -# This manifest installs the calico-node container, as well -# as the CNI plugins and network config on -# each master and worker node in a Kubernetes cluster. -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: calico-node - namespace: kube-system - labels: - k8s-app: calico-node -spec: - selector: - matchLabels: - k8s-app: calico-node - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: calico-node - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Make sure calico-node gets scheduled on all nodes. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - serviceAccountName: calico-node - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - priorityClassName: system-node-critical - initContainers: - # This container installs the CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: calico/cni:v3.15.0 - command: ["/install-cni.sh"] - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Prevents the container from sleeping forever. - - name: SLEEP - value: "false" - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - securityContext: - privileged: true - # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes - # to communicate with Felix over the Policy Sync API. - - name: flexvol-driver - image: calico/pod2daemon-flexvol:v3.15.0 - volumeMounts: - - name: flexvol-driver-host - mountPath: /host/driver - securityContext: - privileged: true - containers: - # Runs calico-node container on each Kubernetes node. This - # container programs network policy and routes on each - # host. - - name: calico-node - image: calico/node:v3.15.0 - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Choose the backend to use. - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s,kubeadm" - # Auto-detect the BGP IP address. - - name: IP - value: "autodetect" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "Never" - # Enable or Disable VXLAN on the default IP pool. - - name: CALICO_IPV4POOL_VXLAN - value: "Always" - # Set MTU for tunnel device used if ipip is enabled - - name: FELIX_IPINIPMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the VXLAN tunnel device. - - name: FELIX_VXLANMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the Wireguard tunnel device. - - name: FELIX_WIREGUARDMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # The default IPv4 pool to create on startup if none exists. Pod IPs will be - # chosen from this range. Changing this value after installation will have - # no effect. This should fall within `--cluster-cidr`. - # - name: CALICO_IPV4POOL_CIDR - # value: "192.168.0.0/16" - # Set MTU for the Wireguard tunnel device. - - name: FELIX_WIREGUARDMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Set Felix logging to "info" - - name: FELIX_LOGSEVERITYSCREEN - value: "Warning" - - name: FELIX_LOGSEVERITYFILE - value: "Warning" - - name: FELIX_LOGSEVERITYSYS - value: "" - - name: FELIX_HEALTHENABLED - value: "true" - - name: FELIX_PROMETHEUSGOMETRICSENABLED - value: "false" - - name: FELIX_PROMETHEUSMETRICSENABLED - value: "true" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - exec: - command: - - /bin/calico-node - - -felix-live - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - exec: - command: - - /bin/calico-node - - -felix-ready - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - - name: policysync - mountPath: /var/run/nodeagent - volumes: - # Used by calico-node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - # Mount in the directory for host-local IPAM allocations. This is - # used when upgrading from host-local to calico-ipam, and can be removed - # if not using the upgrade-ipam init container. - - name: host-local-net-dir - hostPath: - path: /var/lib/cni/networks - # Used to create per-pod Unix Domain Sockets - - name: policysync - hostPath: - type: DirectoryOrCreate - path: /var/run/nodeagent - # Used to install Flex Volume Driver - - name: flexvol-driver-host - hostPath: - type: DirectoryOrCreate - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - ---- -# Source: calico/templates/calico-kube-controllers.yaml -# See https://github.com/projectcalico/kube-controllers -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - # The controllers can only have a single active instance. - replicas: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - strategy: - type: Recreate - template: - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: - nodeSelector: - kubernetes.io/os: linux - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - serviceAccountName: calico-kube-controllers - priorityClassName: system-cluster-critical - containers: - - name: calico-kube-controllers - image: calico/kube-controllers:v3.15.0 - env: - # Choose which controllers to run. - - name: ENABLED_CONTROLLERS - value: node - - name: DATASTORE_TYPE - value: kubernetes - readinessProbe: - exec: - command: - - /usr/bin/check-status - - -r - ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-kube-controllers - namespace: kube-system - ---- -# Source: calico/templates/calico-etcd-secrets.yaml - ---- -# Source: calico/templates/calico-typha.yaml - ---- -# Source: calico/templates/configure-canal.yaml - - diff --git a/artifacts/kubezero-calico/canal.yaml b/artifacts/kubezero-calico/canal.yaml deleted file mode 100644 index ac806f5..0000000 --- a/artifacts/kubezero-calico/canal.yaml +++ /dev/null @@ -1,3749 +0,0 @@ ---- -# Source: calico/templates/calico-config.yaml -# This ConfigMap is used to configure a self-hosted Canal installation. -kind: ConfigMap -apiVersion: v1 -metadata: - name: canal-config -data: - # Typha is disabled. - typha_service_name: "none" - # The interface used by canal for host <-> host communication. - # If left blank, then the interface is chosen using the node's - # default route. - canal_iface: "" - - # Whether or not to masquerade traffic to destinations not within - # the pod network. - masquerade: "true" - - # Configure the MTU to use - veth_mtu: "1450" - - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": { - "type": "host-local", - "subnet": "usePodCidr" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - }, - { - "type": "bandwidth", - "capabilities": {"bandwidth": true} - } - ] - } - - # Flannel network configuration. Mounted into the flannel container. - net-conf.json: | - { - "Network": "10.244.0.0/16", - "Backend": { - "Type": "vxlan" - } - } - ---- -# Source: calico/templates/kdd-crds.yaml - - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: bgpconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPConfiguration - listKind: BGPConfigurationList - plural: bgpconfigurations - singular: bgpconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: BGPConfiguration contains the configuration for any BGP routing. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPConfigurationSpec contains the values of the BGP configuration. - properties: - asNumber: - description: 'ASNumber is the default AS number used by a node. [Default: - 64512]' - format: int32 - type: integer - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: INFO]' - type: string - nodeToNodeMeshEnabled: - description: 'NodeToNodeMeshEnabled sets whether full node to node - BGP mesh is enabled. [Default: true]' - type: boolean - serviceClusterIPs: - description: ServiceClusterIPs are the CIDR blocks from which service - cluster IPs are allocated. If specified, Calico will advertise these - blocks, as well as any cluster IPs within them. - items: - description: ServiceClusterIPBlock represents a single whitelisted - CIDR block for ClusterIPs. - properties: - cidr: - type: string - type: object - type: array - serviceExternalIPs: - description: ServiceExternalIPs are the CIDR blocks for Kubernetes - Service External IPs. Kubernetes Service ExternalIPs will only be - advertised if they are within one of these blocks. - items: - description: ServiceExternalIPBlock represents a single whitelisted - CIDR External IP block. - properties: - cidr: - type: string - type: object - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: bgppeers.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPPeer - listKind: BGPPeerList - plural: bgppeers - singular: bgppeer - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPPeerSpec contains the specification for a BGPPeer resource. - properties: - asNumber: - description: The AS Number of the peer. - format: int32 - type: integer - node: - description: The node name identifying the Calico node instance that - is peering with this peer. If this is not set, this represents a - global peer, i.e. a peer that peers with every node in the deployment. - type: string - nodeSelector: - description: Selector for the nodes that should have this peering. When - this is set, the Node field must be empty. - type: string - peerIP: - description: The IP address of the peer. - type: string - peerSelector: - description: Selector for the remote nodes to peer with. When this - is set, the PeerIP and ASNumber fields must be empty. For each - peering between the local node and selected remote nodes, we configure - an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, - and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The - remote AS number comes from the remote node’s NodeBGPSpec.ASNumber, - or the global default if that is not set. - type: string - required: - - asNumber - - peerIP - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: blockaffinities.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BlockAffinity - listKind: BlockAffinityList - plural: blockaffinities - singular: blockaffinity - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BlockAffinitySpec contains the specification for a BlockAffinity - resource. - properties: - cidr: - type: string - deleted: - description: Deleted indicates that this block affinity is being deleted. - This field is a string for compatibility with older releases that - mistakenly treat this field as a string. - type: string - node: - type: string - state: - type: string - required: - - cidr - - deleted - - node - - state - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: clusterinformations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: ClusterInformation - listKind: ClusterInformationList - plural: clusterinformations - singular: clusterinformation - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: ClusterInformation contains the cluster specific information. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ClusterInformationSpec contains the values of describing - the cluster. - properties: - calicoVersion: - description: CalicoVersion is the version of Calico that the cluster - is running - type: string - clusterGUID: - description: ClusterGUID is the GUID of the cluster - type: string - clusterType: - description: ClusterType describes the type of the cluster - type: string - datastoreReady: - description: DatastoreReady is used during significant datastore migrations - to signal to components such as Felix that it should wait before - accessing the datastore. - type: boolean - variant: - description: Variant declares which variant of Calico should be active. - type: string - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: felixconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: FelixConfiguration - listKind: FelixConfigurationList - plural: felixconfigurations - singular: felixconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: Felix Configuration contains the configuration for Felix. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: FelixConfigurationSpec contains the values of the Felix configuration. - properties: - bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. [Default: - true]' - type: boolean - bpfDataIfacePattern: - description: 'BPFDataIfacePattern is a regular expression that controls - which interfaces Felix should attach BPF programs to in order to - catch traffic to/from the network. This needs to match the interfaces - that Calico workload traffic flows over as well as any interfaces - that handle incoming traffic to nodeports and services from outside - the cluster. It should not match the workload interfaces (usually - named cali...). [Default: ^(en.*|eth.*|tunl0$)]' - type: string - bpfDisableUnprivileged: - description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled - sysctl to disable unprivileged use of BPF. This ensures that unprivileged - users cannot access Calico''s BPF maps and cannot insert their own - BPF programs to interfere with Calico''s. [Default: true]' - type: boolean - bpfEnabled: - description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. - [Default: false]' - type: boolean - bpfExternalServiceMode: - description: 'BPFExternalServiceMode in BPF mode, controls how connections - from outside the cluster to services (node ports and cluster IPs) - are forwarded to remote workloads. If set to "Tunnel" then both - request and response traffic is tunneled to the remote node. If - set to "DSR", the request traffic is tunneled but the response traffic - is sent directly from the remote node. In "DSR" mode, the remote - node appears to use the IP of the ingress node; this requires a - permissive L2 network. [Default: Tunnel]' - type: string - bpfKubeProxyEndpointSlicesEnabled: - description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls - whether Felix's embedded kube-proxy accepts EndpointSlices or not. - type: boolean - bpfKubeProxyIptablesCleanupEnabled: - description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF - mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s - iptables chains. Should only be enabled if kube-proxy is not running. [Default: - true]' - type: boolean - bpfKubeProxyMinSyncPeriod: - description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the - minimum time between updates to the dataplane for Felix''s embedded - kube-proxy. Lower values give reduced set-up latency. Higher values - reduce Felix CPU usage by batching up more work. [Default: 1s]' - type: string - bpfLogLevel: - description: 'BPFLogLevel controls the log level of the BPF programs - when in BPF dataplane mode. One of "Off", "Info", or "Debug". The - logs are emitted to the BPF trace pipe, accessible with the command - `tc exec bpf debug`. [Default: Off].' - type: string - chainInsertMode: - description: 'ChainInsertMode controls whether Felix hooks the kernel’s - top-level iptables chains by inserting a rule at the top of the - chain or by appending a rule at the bottom. insert is the safe default - since it prevents Calico’s rules from being bypassed. If you switch - to append mode, be sure that the other rules in the chains signal - acceptance by falling through to the Calico rules, otherwise the - Calico policy will be bypassed. [Default: insert]' - type: string - dataplaneDriver: - type: string - debugDisableLogDropping: - type: boolean - debugMemoryProfilePath: - type: string - debugSimulateCalcGraphHangAfter: - type: string - debugSimulateDataplaneHangAfter: - type: string - defaultEndpointToHostAction: - description: 'DefaultEndpointToHostAction controls what happens to - traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables “DROP” action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables “INPUT” chain; - Calico will insert its rules at the top of that chain, then “RETURN” - packets to the “INPUT” chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' - type: string - deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. - type: integer - deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. - type: string - disableConntrackInvalidCheck: - type: boolean - endpointReportingDelay: - type: string - endpointReportingEnabled: - type: boolean - externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. - items: - type: string - type: array - failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a comma-delimited list of - UDP/TCP ports that Felix will allow incoming traffic to host endpoints - on irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. Each - port should be specified as tcp: or udp:. - For back-compatibility, if the protocol is not specified, it defaults - to “tcp”. To disable all inbound host ports, use the value none. - The default value allows ssh access and DHCP. [Default: tcp:22, - udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' - items: - description: ProtoPort is combination of protocol and port, both - must be specified. - properties: - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a comma-delimited list - of UDP/TCP ports that Felix will allow outgoing traffic from host - endpoints to irrespective of the security policy. This is useful - to avoid accidentally cutting off a host with incorrect configuration. - Each port should be specified as tcp: or udp:. - For back-compatibility, if the protocol is not specified, it defaults - to “tcp”. To disable all outbound host ports, use the value none. - The default value opens etcd’s standard ports to ensure that Felix - does not get cut off from etcd as well as allowing DHCP and DNS. - [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667, - udp:53, udp:67]' - items: - description: ProtoPort is combination of protocol and port, both - must be specified. - properties: - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - genericXDPEnabled: - description: 'GenericXDPEnabled enables Generic XDP so network cards - that don''t support XDP offload or driver modes can use XDP. This - is not recommended since it doesn''t provide better performance - than iptables. [Default: false]' - type: boolean - healthEnabled: - type: boolean - healthHost: - type: string - healthPort: - type: integer - interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' - type: string - interfacePrefix: - description: 'InterfacePrefix is the interface name prefix that identifies - workload endpoints and so distinguishes them from host endpoint - interfaces. Note: in environments other than bare metal, the orchestrators - configure this appropriately. For example our Kubernetes and Docker - integrations set the ‘cali’ value, and our OpenStack integration - sets the ‘tap’ value. [Default: cali]' - type: string - ipipEnabled: - type: boolean - ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' - type: integer - ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico’s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' - type: string - iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is legacy. - type: string - iptablesFilterAllowAction: - type: string - iptablesLockFilePath: - description: 'IptablesLockFilePath is the location of the iptables - lock file. You may need to change this if the lock file is not in - its standard location (for example if you have mapped it into Felix’s - container at a different path). [Default: /run/xtables.lock]' - type: string - iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' - type: string - iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' - type: string - iptablesMangleAllowAction: - type: string - iptablesMarkMask: - description: 'IptablesMarkMask is the mask that Felix selects its - IPTables Mark bits from. Should be a 32 bit hexadecimal number with - at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' - format: int32 - type: integer - iptablesNATOutgoingInterfaceFilter: - type: string - iptablesPostWriteCheckInterval: - description: 'IptablesPostWriteCheckInterval is the period after Felix - has done a write to the dataplane that it schedules an extra read - back in order to check the write was not clobbered by another process. - This should only occur if another application on the system doesn’t - respect the iptables lock. [Default: 1s]' - type: string - iptablesRefreshInterval: - description: 'IptablesRefreshInterval is the period at which Felix - re-checks the IP sets in the dataplane to ensure that no other process - has accidentally broken Calico’s rules. Set to 0 to disable IP sets - refresh. Note: the default for this value is lower than the other - refresh intervals as a workaround for a Linux kernel bug that was - fixed in kernel version 4.11. If you are using v4.11 or greater - you may want to set this to, a higher value to reduce Felix CPU - usage. [Default: 10s]' - type: string - ipv6Support: - type: boolean - kubeNodePortRanges: - description: 'KubeNodePortRanges holds list of port ranges used for - service node ports. Only used if felix detects kube-proxy running - in ipvs mode. Felix uses these ranges to separate host and workload - traffic. [Default: 30000:32767].' - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - logFilePath: - description: 'LogFilePath is the full path to the Felix log. Set to - none to disable file logging. [Default: /var/log/calico/felix.log]' - type: string - logPrefix: - description: 'LogPrefix is the log prefix that Felix uses when rendering - LOG rules. [Default: calico-packet]' - type: string - logSeverityFile: - description: 'LogSeverityFile is the log severity above which logs - are sent to the log file. [Default: Info]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - logSeveritySys: - description: 'LogSeveritySys is the log severity above which logs - are sent to the syslog. Set to None for no logging to syslog. [Default: - Info]' - type: string - maxIpsetSize: - type: integer - metadataAddr: - description: 'MetadataAddr is the IP address or domain name of the - server that can answer VM queries for cloud-init metadata. In OpenStack, - this corresponds to the machine running nova-api (or in Ubuntu, - nova-api-metadata). A value of none (case insensitive) means that - Felix should not set up any NAT rule for the metadata path. [Default: - 127.0.0.1]' - type: string - metadataPort: - description: 'MetadataPort is the port of the metadata server. This, - combined with global.MetadataAddr (if not ‘None’), is used to set - up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. - In most cases this should not need to be changed [Default: 8775].' - type: integer - natOutgoingAddress: - description: NATOutgoingAddress specifies an address to use when performing - source NAT for traffic in a natOutgoing pool that is leaving the - network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) - type: string - natPortRange: - anyOf: - - type: integer - - type: string - description: NATPortRange specifies the range of ports that is used - for port mapping when doing outgoing NAT. When unset the default - behavior of the network stack is used. - pattern: ^.* - x-kubernetes-int-or-string: true - netlinkTimeout: - type: string - openstackRegion: - description: 'OpenstackRegion is the name of the region that a particular - Felix belongs to. In a multi-region Calico/OpenStack deployment, - this must be configured somehow for each Felix (here in the datamodel, - or in felix.cfg or the environment on each compute node), and must - match the [calico] openstack_region value configured in neutron.conf - on each node. [Default: Empty]' - type: string - policySyncPathPrefix: - description: 'PolicySyncPathPrefix is used to by Felix to communicate - policy changes to external services, like Application layer policy. - [Default: Empty]' - type: string - prometheusGoMetricsEnabled: - description: 'PrometheusGoMetricsEnabled disables Go runtime metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - prometheusMetricsEnabled: - description: 'PrometheusMetricsEnabled enables the Prometheus metrics - server in Felix if set to true. [Default: false]' - type: boolean - prometheusMetricsHost: - description: 'PrometheusMetricsHost is the host that the Prometheus - metrics server should bind to. [Default: empty]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. [Default: 9091]' - type: integer - prometheusProcessMetricsEnabled: - description: 'PrometheusProcessMetricsEnabled disables process metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. - type: boolean - reportingInterval: - description: 'ReportingInterval is the interval at which Felix reports - its status into the datastore or 0 to disable. Must be non-zero - in OpenStack deployments. [Default: 30s]' - type: string - reportingTTL: - description: 'ReportingTTL is the time-to-live setting for process-wide - status reports. [Default: 90s]' - type: string - routeRefreshInterval: - description: 'RouterefreshInterval is the period at which Felix re-checks - the routes in the dataplane to ensure that no other process has - accidentally broken Calico’s rules. Set to 0 to disable route refresh. - [Default: 90s]' - type: string - routeSource: - description: 'RouteSource configures where Felix gets its routing - information. - WorkloadIPs: use workload endpoints to construct - routes. - CalicoIPAM: the default - use IPAM data to construct routes.' - type: string - routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - sidecarAccelerationEnabled: - description: 'SidecarAccelerationEnabled enables experimental sidecar - acceleration [Default: false]' - type: boolean - usageReportingEnabled: - description: 'UsageReportingEnabled reports anonymous Calico version - number and cluster size to projectcalico.org. Logs warnings returned - by the usage server. For example, if a significant security vulnerability - has been discovered in the version of Calico being used. [Default: - true]' - type: boolean - usageReportingInitialDelay: - description: 'UsageReportingInitialDelay controls the minimum delay - before Felix makes a report. [Default: 300s]' - type: string - usageReportingInterval: - description: 'UsageReportingInterval controls the interval at which - Felix makes reports. [Default: 86400s]' - type: string - useInternalDataplaneDriver: - type: boolean - vxlanEnabled: - type: boolean - vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' - type: integer - vxlanPort: - type: integer - vxlanVNI: - type: integer - wireguardEnabled: - description: 'WireguardEnabled controls whether Wireguard is enabled. - [Default: false]' - type: boolean - wireguardInterfaceName: - description: 'WireguardInterfaceName specifies the name to use for - the Wireguard interface. [Default: wg.calico]' - type: string - wireguardListeningPort: - description: 'WireguardListeningPort controls the listening port used - by Wireguard. [Default: 51820]' - type: integer - wireguardMTU: - description: 'WireguardMTU controls the MTU on the Wireguard interface. - See Configuring MTU [Default: 1420]' - type: integer - wireguardRoutingRulePriority: - description: 'WireguardRoutingRulePriority controls the priority value - to use for the Wireguard routing rule. [Default: 99]' - type: integer - xdpEnabled: - description: 'XDPEnabled enables XDP acceleration for suitable untracked - incoming deny rules. [Default: true]' - type: boolean - xdpRefreshInterval: - description: 'XDPRefreshInterval is the period at which Felix re-checks - all XDP state to ensure that no other process has accidentally broken - Calico''s BPF maps or attached programs. Set to 0 to disable XDP - refresh. [Default: 90s]' - type: string - required: - - bpfLogLevel - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: globalnetworkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkPolicy - listKind: GlobalNetworkPolicyList - plural: globalnetworkpolicies - singular: globalnetworkpolicy - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - applyOnForward: - description: ApplyOnForward indicates to apply the rules in this policy - on forward traffic. - type: boolean - doNotTrack: - description: DoNotTrack indicates whether packets matched by the rules - in this policy should go through the data plane's connection tracking, - such as Linux conntrack. If True, the rules in this policy are - applied before any data plane connection tracking, and packets allowed - by this policy are marked as not to be tracked. - type: boolean - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - required: - - action - type: object - type: array - namespaceSelector: - description: NamespaceSelector is an optional field for an expression - used to select a pod based on namespaces. - type: string - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - preDNAT: - description: PreDNAT indicates to apply the rules in this policy before - any DNAT. - type: boolean - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress rules are present in the policy. The - default is: \n - [ PolicyTypeIngress ], if there are no Egress rules - (including the case where there are also no Ingress rules) \n - - [ PolicyTypeEgress ], if there are Egress rules but no Ingress - rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are - both Ingress and Egress rules. \n When the policy is read back again, - Types will always be one of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: globalnetworksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkSet - listKind: GlobalNetworkSetList - plural: globalnetworksets - singular: globalnetworkset - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs - that share labels to allow rules to refer to them via selectors. The labels - of GlobalNetworkSet are not namespaced. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: GlobalNetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: hostendpoints.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: HostEndpoint - listKind: HostEndpointList - plural: hostendpoints - singular: hostendpoint - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: HostEndpointSpec contains the specification for a HostEndpoint - resource. - properties: - expectedIPs: - description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. - If \"InterfaceName\" is not present, Calico will look for an interface - matching any of the IPs in the list and apply policy to that. Note: - \tWhen using the selector match criteria in an ingress or egress - security Policy \tor Profile, Calico converts the selector into - a set of IP addresses. For host \tendpoints, the ExpectedIPs field - is used for that purpose. (If only the interface \tname is specified, - Calico does not learn the IPs of the interface for use in match - \tcriteria.)" - items: - type: string - type: array - interfaceName: - description: "Either \"*\", or the name of a specific Linux interface - to apply policy to; or empty. \"*\" indicates that this HostEndpoint - governs all traffic to, from or through the default network namespace - of the host named by the \"Node\" field; entering and leaving that - namespace via any interface, including those from/to non-host-networked - local workloads. \n If InterfaceName is not \"*\", this HostEndpoint - only governs traffic that enters or leaves the host through the - specific interface named by InterfaceName, or - when InterfaceName - is empty - through the specific interface that has one of the IPs - in ExpectedIPs. Therefore, when InterfaceName is empty, at least - one expected IP must be specified. Only external interfaces (such - as “eth0”) are supported here; it isn't possible for a HostEndpoint - to protect traffic through a specific local workload interface. - \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; - initially just pre-DNAT policy. Please check Calico documentation - for the latest position." - type: string - node: - description: The node name identifying the Calico node instance. - type: string - ports: - description: Ports contains the endpoint's named ports, which may - be referenced in security policy rules. - items: - properties: - name: - type: string - port: - type: integer - protocol: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - required: - - name - - port - - protocol - type: object - type: array - profiles: - description: A list of identifiers of security Profile objects that - apply to this endpoint. Each profile is applied in the order that - they appear in this list. Profile rules are applied after the selector-based - security policy. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipamblocks.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMBlock - listKind: IPAMBlockList - plural: ipamblocks - singular: ipamblock - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMBlockSpec contains the specification for an IPAMBlock - resource. - properties: - affinity: - type: string - allocations: - items: - type: integer - # TODO: This nullable is manually added in. We should update controller-gen - # to handle []*int properly itself. - nullable: true - type: array - attributes: - items: - properties: - handle_id: - type: string - secondary: - additionalProperties: - type: string - type: object - type: object - type: array - cidr: - type: string - deleted: - type: boolean - strictAffinity: - type: boolean - unallocated: - items: - type: integer - type: array - required: - - allocations - - attributes - - cidr - - deleted - - strictAffinity - - unallocated - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipamconfigs.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMConfig - listKind: IPAMConfigList - plural: ipamconfigs - singular: ipamconfig - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMConfigSpec contains the specification for an IPAMConfig - resource. - properties: - autoAllocateBlocks: - type: boolean - strictAffinity: - type: boolean - required: - - autoAllocateBlocks - - strictAffinity - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipamhandles.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMHandle - listKind: IPAMHandleList - plural: ipamhandles - singular: ipamhandle - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMHandleSpec contains the specification for an IPAMHandle - resource. - properties: - block: - additionalProperties: - type: integer - type: object - handleID: - type: string - required: - - block - - handleID - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ippools.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPPool - listKind: IPPoolList - plural: ippools - singular: ippool - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPPoolSpec contains the specification for an IPPool resource. - properties: - blockSize: - description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. - type: integer - cidr: - description: The pool CIDR. - type: string - disabled: - description: When disabled is true, Calico IPAM will not assign addresses - from this pool. - type: boolean - ipip: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - properties: - enabled: - description: When enabled is true, ipip tunneling will be used - to deliver packets to destinations within this pool. - type: boolean - mode: - description: The IPIP mode. This can be one of "always" or "cross-subnet". A - mode of "always" will also use IPIP tunneling for routing to - destination IP addresses within this pool. A mode of "cross-subnet" - will only use IPIP tunneling when the destination node is on - a different subnet to the originating node. The default value - (if not specified) is "always". - type: string - type: object - ipipMode: - description: Contains configuration for IPIP tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. IPIP tunelling - is disabled). - type: string - nat-outgoing: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - type: boolean - natOutgoing: - description: When nat-outgoing is true, packets sent from Calico networked - containers in this pool to destinations outside of this pool will - be masqueraded. - type: boolean - nodeSelector: - description: Allows IPPool to allocate for a specific node by label - selector. - type: string - vxlanMode: - description: Contains configuration for VXLAN tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. VXLAN - tunelling is disabled). - type: string - required: - - cidr - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: kubecontrollersconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: KubeControllersConfiguration - listKind: KubeControllersConfigurationList - plural: kubecontrollersconfigurations - singular: kubecontrollersconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KubeControllersConfigurationSpec contains the values of the - Kubernetes controllers configuration. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host endpoints. - Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation of - host endpoints for every node. [Default: Disabled]' - type: string - type: object - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - type: object - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - required: - - controllers - type: object - status: - description: KubeControllersConfigurationStatus represents the status - of the configuration. It's useful for admins to be able to see the actual - config that was applied, which can be modified by environment variables - on the kube-controllers process. - properties: - environmentVars: - additionalProperties: - type: string - description: EnvironmentVars contains the environment variables on - the kube-controllers that influenced the RunningConfig. - type: object - runningConfig: - description: RunningConfig contains the effective config that is running - in the kube-controllers pod, after merging the API resource with - any environment variables. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace - controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host - endpoints. Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation - of host endpoints for every node. [Default: Disabled]' - type: string - type: object - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - type: object - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which - logs are sent to the stdout. [Default: Info]' - type: string - required: - - controllers - type: object - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: networkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkPolicy - listKind: NetworkPolicyList - plural: networkpolicies - singular: networkpolicy - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - required: - - action - type: object - type: array - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress are present in the policy. The default - is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including - the case where there are also no Ingress rules) \n - [ PolicyTypeEgress - ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, - PolicyTypeEgress ], if there are both Ingress and Egress rules. - \n When the policy is read back again, Types will always be one - of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: networksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkSet - listKind: NetworkSetList - plural: networksets - singular: networkset - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- ---- -# Source: calico/templates/calico-kube-controllers-rbac.yaml - -# Include a clusterrole for the kube-controllers component, -# and bind it to the calico-kube-controllers serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -rules: - # Nodes are watched to monitor for deletions. - - apiGroups: [""] - resources: - - nodes - verbs: - - watch - - list - - get - # Pods are queried to check for existence. - - apiGroups: [""] - resources: - - pods - verbs: - - get - # IPAM resources are manipulated when nodes are deleted. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - verbs: - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - # kube-controllers manages hostendpoints. - - apiGroups: ["crd.projectcalico.org"] - resources: - - hostendpoints - verbs: - - get - - list - - create - - update - - delete - # Needs access to update clusterinformations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - clusterinformations - verbs: - - get - - create - - update - # KubeControllersConfiguration is where it gets its config - - apiGroups: ["crd.projectcalico.org"] - resources: - - kubecontrollersconfigurations - verbs: - # read its own config - - get - # create a default if none exists - - create - # update status - - update - # watch for changes - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers -subjects: -- kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system ---- - ---- -# Source: calico/templates/calico-node-rbac.yaml -# Include a clusterrole for the calico-node DaemonSet, -# and bind it to the calico-node serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-node -rules: - # The CNI plugin needs to get pods, nodes, and namespaces. - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - - services - verbs: - # Used to discover service IPs for advertisement. - - watch - - list - # Used to discover Typhas. - - get - # Pod CIDR auto-detection on kubeadm needs access to config maps. - - apiGroups: [""] - resources: - - configmaps - verbs: - - get - - apiGroups: [""] - resources: - - nodes/status - verbs: - # Needed for clearing NodeNetworkUnavailable flag. - - patch - # Calico stores some configuration information in node annotations. - - update - # Watch for changes to Kubernetes NetworkPolicies. - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - # Used by Calico for policy information. - - apiGroups: [""] - resources: - - pods - - namespaces - - serviceaccounts - verbs: - - list - - watch - # The CNI plugin patches pods/status. - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - # Calico monitors various CRDs for config. - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - ipamblocks - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - networksets - - clusterinformations - - hostendpoints - - blockaffinities - verbs: - - get - - list - - watch - # Calico must create and update some CRDs on startup. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - - felixconfigurations - - clusterinformations - verbs: - - create - - update - # Calico stores some configuration information on the node. - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - watch - # These permissions are only required for upgrade from v2.6, and can - # be removed after upgrade or on fresh installations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - bgpconfigurations - - bgppeers - verbs: - - create - - update - ---- -# Flannel ClusterRole -# Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: flannel -rules: - - apiGroups: [""] - resources: - - pods - verbs: - - get - - apiGroups: [""] - resources: - - nodes - verbs: - - list - - watch - - apiGroups: [""] - resources: - - nodes/status - verbs: - - patch ---- -# Bind the flannel ClusterRole to the canal ServiceAccount. -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: canal-flannel -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flannel -subjects: -- kind: ServiceAccount - name: canal - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: canal-calico -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: canal - namespace: kube-system - ---- -# Source: calico/templates/calico-node.yaml -# This manifest installs the canal container, as well -# as the CNI plugins and network config on -# each master and worker node in a Kubernetes cluster. -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: canal - labels: - k8s-app: canal -spec: - selector: - matchLabels: - k8s-app: canal - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: canal - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Make sure canal gets scheduled on all nodes. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - serviceAccountName: canal - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - priorityClassName: system-node-critical - initContainers: - # This container installs the CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: calico/cni:v3.15.0 - command: ["/install-cni.sh"] - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-canal.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: canal-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: canal-config - key: veth_mtu - # Prevents the container from sleeping forever. - - name: SLEEP - value: "false" - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - securityContext: - privileged: true - # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes - # to communicate with Felix over the Policy Sync API. - - name: flexvol-driver - image: calico/pod2daemon-flexvol:v3.15.0 - volumeMounts: - - name: flexvol-driver-host - mountPath: /host/driver - securityContext: - privileged: true - containers: - # Runs canal container on each Kubernetes node. This - # container programs network policy and routes on each - # host. - - name: calico-node - image: calico/node:v3.15.0 - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Configure route aggregation based on pod CIDR. - - name: USE_POD_CIDR - value: "true" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Don't enable BGP. - - name: CALICO_NETWORKING_BACKEND - value: "none" - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s,canal" - # Period, in seconds, at which felix re-applies all iptables state - - name: FELIX_IPTABLESREFRESHINTERVAL - value: "60" - # No IP address needed. - - name: IP - value: "" - # The default IPv4 pool to create on startup if none exists. Pod IPs will be - # chosen from this range. Changing this value after installation will have - # no effect. This should fall within `--cluster-cidr`. - # - name: CALICO_IPV4POOL_CIDR - # value: "192.168.0.0/16" - # Set MTU for the Wireguard tunnel device. - - name: FELIX_WIREGUARDMTU - valueFrom: - configMapKeyRef: - name: canal-config - key: veth_mtu - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Set Felix logging to "info" - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - - name: FELIX_HEALTHENABLED - value: "true" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - exec: - command: - - /bin/calico-node - - -felix-live - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - httpGet: - path: /readiness - port: 9099 - host: localhost - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - - name: policysync - mountPath: /var/run/nodeagent - # This container runs flannel using the kube-subnet-mgr backend - # for allocating subnets. - - name: kube-flannel - image: quay.io/coreos/flannel:v0.11.0 - command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] - securityContext: - privileged: true - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: FLANNELD_IFACE - valueFrom: - configMapKeyRef: - name: canal-config - key: canal_iface - - name: FLANNELD_IP_MASQ - valueFrom: - configMapKeyRef: - name: canal-config - key: masquerade - volumeMounts: - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - # Used by canal. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - # Used by flannel. - - name: flannel-cfg - configMap: - name: canal-config - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - # Used to create per-pod Unix Domain Sockets - - name: policysync - hostPath: - type: DirectoryOrCreate - path: /var/run/nodeagent - # Used to install Flex Volume Driver - - name: flexvol-driver-host - hostPath: - type: DirectoryOrCreate - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: canal - ---- -# Source: calico/templates/calico-kube-controllers.yaml -# See https://github.com/projectcalico/kube-controllers -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-kube-controllers - labels: - k8s-app: calico-kube-controllers -spec: - # The controllers can only have a single active instance. - replicas: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - strategy: - type: Recreate - template: - metadata: - name: calico-kube-controllers - labels: - k8s-app: calico-kube-controllers - spec: - nodeSelector: - kubernetes.io/os: linux - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - serviceAccountName: calico-kube-controllers - priorityClassName: system-cluster-critical - containers: - - name: calico-kube-controllers - image: calico/kube-controllers:v3.15.0 - env: - # Choose which controllers to run. - - name: ENABLED_CONTROLLERS - value: node - - name: DATASTORE_TYPE - value: kubernetes - readinessProbe: - exec: - command: - - /usr/bin/check-status - - -r - ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-kube-controllers - ---- -# Source: calico/templates/calico-etcd-secrets.yaml - ---- -# Source: calico/templates/calico-typha.yaml - ---- -# Source: calico/templates/configure-canal.yaml - - diff --git a/artifacts/kubezero-calico/kustomization.yaml b/artifacts/kubezero-calico/kustomization.yaml deleted file mode 100644 index d91e517..0000000 --- a/artifacts/kubezero-calico/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -namespace: kube-system - -resources: -- canal.yaml - -patchesStrategicMerge: -- logging.yaml -- prometheus.yaml diff --git a/artifacts/kubezero-calico/logging.yaml b/artifacts/kubezero-calico/logging.yaml deleted file mode 100644 index aa4cfb8..0000000 --- a/artifacts/kubezero-calico/logging.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: canal -spec: - template: - spec: - containers: - - name: calico-node - env: - - name: FELIX_LOGSEVERITYSCREEN - value: "Warning" - - name: FELIX_LOGSEVERITYFILE - value: "Warning" - - name: FELIX_LOGSEVERITYSYS - value: "" diff --git a/artifacts/kubezero-calico/prometheus.yaml b/artifacts/kubezero-calico/prometheus.yaml deleted file mode 100644 index 1b183e2..0000000 --- a/artifacts/kubezero-calico/prometheus.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: canal -spec: - template: - spec: - containers: - - name: calico-node - env: - - name: FELIX_PROMETHEUSGOMETRICSENABLED - value: "false" - - name: FELIX_PROMETHEUSMETRICSENABLED - value: "true" diff --git a/artifacts/kubezero-calico/remove-namespace.patch b/artifacts/kubezero-calico/remove-namespace.patch deleted file mode 100644 index 65343b7..0000000 --- a/artifacts/kubezero-calico/remove-namespace.patch +++ /dev/null @@ -1,50 +0,0 @@ ---- canal.yaml.orig 2020-07-02 16:56:37.279169481 +0100 -+++ canal.yaml 2020-07-02 16:56:37.285169542 +0100 -@@ -5,7 +5,6 @@ - apiVersion: v1 - metadata: - name: canal-config -- namespace: kube-system - data: - # Typha is disabled. - typha_service_name: "none" -@@ -3438,7 +3437,6 @@ - apiVersion: apps/v1 - metadata: - name: canal -- namespace: kube-system - labels: - k8s-app: canal - spec: -@@ -3683,7 +3681,6 @@ - kind: ServiceAccount - metadata: - name: canal -- namespace: kube-system - - --- - # Source: calico/templates/calico-kube-controllers.yaml -@@ -3692,7 +3689,6 @@ - kind: Deployment - metadata: - name: calico-kube-controllers -- namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: -@@ -3706,7 +3702,6 @@ - template: - metadata: - name: calico-kube-controllers -- namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: -@@ -3741,7 +3736,6 @@ - kind: ServiceAccount - metadata: - name: calico-kube-controllers -- namespace: kube-system - - --- - # Source: calico/templates/calico-etcd-secrets.yaml diff --git a/charts/kubezero-kiam/README.md b/charts/kubezero-kiam/README.md index 050558b..9d4ca94 100644 --- a/charts/kubezero-kiam/README.md +++ b/charts/kubezero-kiam/README.md @@ -25,7 +25,8 @@ The required certificates for Kiam server and agents are provided by a local cer [KubeZero cert-manager](../kubezero-cert-manager/README.md) ## Metadata restrictions -Required for the *csi ebs plugin* and most likely various others assuming basic AWS information. +Some services require access to some basic AWS information. One example is the `aws-ebs-csi` controller. +By default all access to the meta-data service is blocked, expect for: - `/latest/meta-data/instance-id` - `/latest/dynamic/instance-identity/document` @@ -76,3 +77,5 @@ Required for the *csi ebs plugin* and most likely various others assuming basic ## Resources - https://github.com/uswitch/kiam - https://www.bluematador.com/blog/iam-access-in-kubernetes-kube2iam-vs-kiam + +![Kiam overview](./kiam_architecure.png) diff --git a/charts/kubezero-kiam/README.md.gotmpl b/charts/kubezero-kiam/README.md.gotmpl index 9a441b2..6382e4e 100644 --- a/charts/kubezero-kiam/README.md.gotmpl +++ b/charts/kubezero-kiam/README.md.gotmpl @@ -19,7 +19,8 @@ The required certificates for Kiam server and agents are provided by a local cer [KubeZero cert-manager](../kubezero-cert-manager/README.md) ## Metadata restrictions -Required for the *csi ebs plugin* and most likely various others assuming basic AWS information. +Some services require access to some basic AWS information. One example is the `aws-ebs-csi` controller. +By default all access to the meta-data service is blocked, expect for: - `/latest/meta-data/instance-id` - `/latest/dynamic/instance-identity/document` @@ -34,3 +35,5 @@ Required for the *csi ebs plugin* and most likely various others assuming basic ## Resources - https://github.com/uswitch/kiam - https://www.bluematador.com/blog/iam-access-in-kubernetes-kube2iam-vs-kiam + +![Kiam overview](./kiam_architecure.png) diff --git a/charts/kubezero-kiam/kiam_architecure.png b/charts/kubezero-kiam/kiam_architecure.png new file mode 100644 index 0000000000000000000000000000000000000000..0ef4bde70b5273e7918275ebfbbd3194d0416f90 GIT binary patch literal 43992 zcmce-byQqUv?tm)1OfyJB)BvXoZu3i5L|-0Yp_NeI!H)x?M8xo2p-&>KpJ;(Yi7N>)_b$w`KM29t9^Fus`~Y=b0Rg>6(2u({s;g7JXU%muMGfT!2tlw zcw7t=WfLu&0sufS*HqO}czAgD{YeN#%+dr-PEC1dN*Wp(uC1*V6&0yia=wQ^C@CrX z`uc8A*L2g}@lS4U?(_5WKi?o1NgA^t-e13dg@lC2%ge)Hu*u0uS65dJ4UPKx`unqk z-(jNJdW?v&u)E#G%RHOkQIfyoUn7@CuDjE+w1Bs(@T;cii*&=wukP1_UpK>*>+0%` zj*ccKCf2p(X1#beHaC%bTX%Q2ov|JgN-w_Zv7UYqq#(eY93L<02+?%lUhb|LU2bW! z;^3sfYqNXtGgNZjpYKl<uR$7YpR~OfV#gU)Ae>icj?M~h<^`)n$PyHA`UE2`; z?VX!a60B@N~;j@IYbCV#Yy<}?PHulVo{4GqoIbj?FMeA;d0gY1YaK?XNV5!^3Q#3 zA0bLaMAXtz+t^!NT%9tPnNr#cgJ=p4WQGPudLE`*zB5p+bdieFezGTR?E`T?s`A`rq6C zcbA3FA!WTK<)f|>ISKJF5n>|8#Ys6b*(&-ID7?0qm`zlgTN4x_yPN(3XV+wRqI8~C znZA-bN^|cagm|73d>wB8cTRO357A))PQrh_<i|Y-{L&OHIO0!I zn~~7fYSV~eiZUFHo{<*Fj5PHpgUA zquI>%ly*{njt=56=RsvhgGVgKfTi-4y8Umw&E&EM^zWTPJbU&Z7nyoys~M}|&%&-U z{7#=0B|Yd>g5H;QC##nhsA{{u80xe~PHFh?gZm=#3-KiFCe6G~+I)|qJ{pVX`Q4OUB4$MA4vB~NXEid!UTH12YTO7 zG$*9L03i@-il;y${7FefHPcIY|9TWBj%D(dHH*}o6N6+4)uo1;GxB1ORHwRuV38P0 zq?)_ciXl>Ux)#nwofe_#rW0Hk>7z#FIKA+PVJ+7#0I%uROoO2J7Q-f9A4Zol)cLY` zdi;sz3}Th>ku=xV;ggrqD!;;R)uF9~9 znq)51WMW?f=S(YBDNA+@&r(o;}@cP`)RWE>@3z$n#Vwykp2cJdij(3iE ztkVduf=1Q#Ns{(cIX=;a#F~*#vri5R$6qcA6IftFDN3biKH>5zoT@$2v{9T9uLZui z0{94O?b8P`;6y_|N%SDt-zqCUx+Z=t{JPgF@jo|2w1!(@YB?#=f{ zM#9@9MoSqc`Fwh?OLsIoN07qf(D~Ig)gy+ld4jRLb%g_(t2Oz|dav0Ge_+(YpKY7x z`a^_J4`h;`?-XVmCtnouG^bi~O+JStlA0{`go}wm;0@#KSshUBW6@(4H*YxArLVCD zZMwOj3173~WXNoKCZp4${pT};ZJ4yT;gUXXDj=^ zy)@uSPoN*A$UI3PF(PYm%h$j%-@dG?X#Uwl_Wm{HB6<0>9XQ0MsY zft{rom%v$V)r6C4nF4nC#wBh}W+xC)Sb$fdvP1a1^3PHto>(Gg)C~qIUcz?qV=Pjf zsf`j~dG;#U9wsxI^Ab+p{s+mA=?ppQyEI5#_$GfZ3#<{{$Yo+soh3)AG_-<6;mdqS&=5mk-}=vu;TARsuT1~ZiIr9)Y4((C=Au~_N^7fN9*fqyFrS(2d4iga z7zQR%rL&2z+Bz+O9TSfHG`FVaI<0%Qv%US`z}2@FfzF6pn&?h2;-$p0!MB0! zLph{pReJSzv3~Fp@~|r@1FAT>9MLOK?4ZVzvjNq>&4rn=Ms;e(wr2m|3wVLoRK2`X zw(mtc5ECjwSLU(s95dsJ%2!-pBt5+ATl_!dY{7Yl$F7t_BwKG;GH%`M^wpP*Dk|6R zZU?e(%4Vnn8Tos8GyXV=bg0&RgFjeh-wry}x1^d^nQz$d^Apr2(na5X53kw+p^Rsq zww*<0iQBm_5@3kjyBAhakbDcJVa~~85LqTZCbikMVt}cnK7Hv$JZwwC=<&paU_)AD zyjq0<@8c%)#j~2aP+1wNsN=Ly(z+_&Zj-gtn}-yW9@{%2p0Ts+Mk7-1A1})_L4aoE zj@qgCw)zZ?Cs!vwh2VaOmZY}vh2)FEBH7hSlmJ+za+s0ldN#T5ls$_7NE`V&jfy;;& z;9m#H7S2Grs&L}u5dU9VAb{_Fxe3r7KDDH;j=8jOdRtGK6vsr!CJGoy=4e(P&9~(n+bfN~tvUh5s4Dp_Jx3zU1# z20r!4!k7>()de*bBBb}UkehHlzpl%0P5A?IMZ-DydaGI%3CzQ8(!^W+)XLqas$N7qlT=v>(c zCF?~Dnh-QY@g&uiM|sY>>-Y|HvOE<7iPG{4zY)VKMLq6W8=(~Fn2QMZ7gD$^Z zXu0R_sls%`np(@G7o5?8?ci6(gI6OuJL-$38MLmL0Jf{~FQqDb?$*~$PXW0IIpaN( z7Lzoy4t#(nGQ=>pXo$j^(hN-+u6XDMe6UHs_~huz3Xq0l5Evp$9Or*tb=Ua=QSBSb z4n#)ykH3$W1d%{j=@5=5G-u)|MkUcz1ZFbiEP|;cbkwNMX@DbN` zKGtMNzFyp9VT(?Ax-Ny(QE3Bs)~u<2Bw0n6!dREe-i3$_;Gc&W+hB2R+UikRsu8@)3Gr7FJ|!ITcRRJqS+ zueGm!9?^5Mp}-oF7LsOV2ThRj*e5^0q~Z|AcRs}spHTkxt@^y|n>O*Z9zOm=&c+y@ zy`B3B2X^9qJBZe&*4Gtj=homSl$ohoDd86wGZrWRJ$5PPohBtw7b3nn^e2%jEiae( z&$E6I^JvMjsyd#flDu@9o+-Q~tOk)FSW)jXAYoC;gRLRpNdHqe3!7yKyyQ&7m3(JK zFGGCsqXllbIp8Oj{lfJOW0Hl5%Hr3LvGI)^mg5UMWSPER`V)P%(F^s3zE`jN+N|!o z5riQG!~wA-Y0Ch)4}|ZzJKS6DDo6<4+lIYt58a8NrLJSe5+1o?v)^46`9ZEJGmpW+ z@<7MDs%qA*K$ai9Q8ohcVjtQr`zcXc8nCjnA0YO8Xp67Zp%(6PynU;;e!eDr^<9i& zGuo5mtwTlnvTUu&4Kjz-X~^~Rm;DU+K)5r^mBn6o>ErH3geQ$4U4CtzJVFHM=XF(ltx@rw*a@Tt1z*2ge(&z!5I9AiCU0E2Xp(Kz+rAHPZqXr7%u~-sA4SO z?(X;6vnUBNaxw|7w6;`NhCC)2;=s_)5DZPV3G6|b{%#E%0}2$c-feNB7V?gM{?)1P z8LS>J&VN_B88k8DrI?{ar5bquVoYgN>Ypx~>Us69$S)HmpCQ5V-##9fa>OX)U#km(gY%-s{a%zc^l=`k8A;AOQI?eeFQ#_|B*r~$Bsr{mc2#OZl<>SgB z^7sNhg_CC7Ym0bgw1NpUr$ZV?6YW5twp2q75=={}@{HS;o)<^^*6)CBEDuKH&!Uf< zURmB6V)dm`R2F~yv8t$F#?hVPiA+Co5U=HU^{ec&P{^#))L;udKz`7#$qM-_F3=Km zyiPXWxSG7aPL1QY+p&_ZFDp)Om{PAE@o3RkP<8}S5T+8$@b4T;6TXZ;Yj#txn5}*- zm$IxGP>4ftFwa0NNJe%u%la8)+>wmN=E2+H!HF1c!< z___J!1N}j=H;muDGE=;t^sIDDdv-?iHJYT#Aa{XiDh{p#Ye_H_39;c7B^enn$X$R(pR7RjwhBA z-_lm6nnt(Wd_{nI(Ig@$k@gKrz~htymeYK4f4=wd1$gS_{-bvBUO|?%h+IZQOwVqd z1a>@zV6Cb&@AL$fB2TY>>$4)yg=L8Q5aGS3!AceOX{F-P;A5ThZQW-@QfvI>;Pid@ zH3-jIIoj_1`MsUBbwv@(SWAy zf3~cjJ&VWGX1K!1vTVdHnGFd3mQaC?|3diVr+0~rv8Gr;g=(ZQxxxj;Zd2EFc8Vzb z*A_@?%Md1sp&N?=OTrVzzTzkD^U%)Q=RZq06m7?#Id1*>Pjl8uR@S3oM^6CW9~C#s(%vS>Ve@$VkJ{ z`XZn>Xh0ldCS_X80aK(W8bEnfh7MSz(eg+B?{>cWtfFv4*MG7b1BfFH56~~Oo7`ME z@o{e{4&OLY1q4GcIp2;@m&>*h;nh^NL^OvEc<`#dZb$HUqXFMC zRaA~$^uoviF0*AWaC;t105QWH5?_oEpk-Xey(0TyBu{Ikx|v9ulj!P-9kw-iMw z0BqDFsgQ*`b14)~CZQ@$1hVkA`llxR=Ez~zl{gX<2<5{C(mB4+c0N?$e#`N4A{`|} z27#-Lz?hW;iGUj3jLCK=D4M^nV&=F$+aEPwa)W|O5g$0fc7h(*M$M1)FN|3KR?OCw zvz>t1GsCHEI?kU^MyO5TiW!~4vx})FAey%*mA?6;Pie}U=}xr_j3qDRl}Am{{-c;! zSPziaVENFi;#r?o$0jMc@b*dvkh_wU^nla+>ubV_S(ev{5#ZM9cvZI$w}+SD0a7=E zrTchBMxSms+MPGkWWMjm{2R>E;53I)KMdNpPspsTCsaIt1sCm?S{GQ02NIPVwTjmO zk*;TMYD9N$FnG>e4K0%Bn)L3X%4FrwnV!Dh_01R`iZ}Th2p4Byybi%z_xsuSb{xqR zUlz4R&UkJ5Ja{X4{TndIAj@Z~@VY}0rS<;lx*t&F9o)rxFgcoHO=r#9EFz-A4PC?x z@7@W{(<#yD`3cCHuRtJT_JYt9Nu_(ft$=%_kdhhk^ch;5KD0OW7C98{A##6U;(q#Z zik#7?kLlM(LNnx0s#S-Zi5P{6Scw4$MWuiK5v31{K2^S5_)dl#B3DtDkB)hhId0Ad z_Gm2qE7OlbT}2k2Q)xrTfwG_S5-`B=Juhezd}jhGvVuoClsrNCD(VL>X!X#!|MHi@ zN95iHrEWvOZ~P!XyMT>i9Zx1a+);Y)Zk23Pyl}mgq_!|KFKe!m{#ZQ)mw>enpkGL^$l=>Ndnv_vUs=W`_`MP;nBd3(|rxy{m;@ zo`zJy+IAmu9+>4H=%wQlb(PhO#ts8p>;3IV!Bj{&B10rhe@J=Nx`%Sf80xAF-%91% zi5I9qKp&D`b1_zgmjHPrH1VHQOeLn6FlAgUR>;1DG>?aKT-CaNu$6&}tC}w6hiNGJ zs%Nn`pYO^$5wYF;^eW;HcIsnzy84*~68HkVo+rseT~0h9M&kmve*Z4sqZ2yExN?x2 z-sa$+E2;^1Gl>s>LXjjf-5L*^rQF!U*>+Pt8jr+k6Z)(S=i)u*&5G+yJ=t|r;pY9B zPEDOeS~261YVyt@RiNS6CDW2oI1RoZM+5B6C|%bi#vD?uo_lhGZ)sBv2(0*W3{k%G z7>!XakXFo2q_pXa2Rj#rs({s61=%aaUXcAl;(Yq*W}8R&Enrn<-?jPEY*lL8YoVeG zLWfuBJSvqy=W&%l4Q?vNsx(pDeZrW%bPdhEU?AXUbgk_&Ubk zEW`20({#@NG!t^LfnWs^T>^Qsxx_V&^7gwB_&bEPryx6ck>cWx!8YQxypf9nwt=d4 zlHcWtt>qskjr@9)#qeuYN~_;(Zw`_HR+c(mvw~oJ*i*3G{r>aAz`)>%Z7K&_{Za*> zTLKRhXSZ&Hma_9LC|0J>$3ZF;(#5%HxmDS_HCXrJ&L#HHGQBOqu8Uc|PF>Ly*-wA| zyd7bLp`9unWDJTg;RIZH*Dh@-RJt^W=trnXsEBJpyc~F^;js0ikz9IFgxTTl)&)*QRA-q;Ck`js!uX8+uKlrQdnQvXqZAlPvtayq^@3+RuSxT=p(l)f$e2wCBG|3nGegWluwfo zGvWz<)VNQ9yPmyv#>nw0vkXH!G0%}bH@Xb2hEjCklyh2#_{2lQVTg_K!L`n^3ONq> zb0Tphw2`m09_RM6PljIq&>WB`$!)0Tk9x|>8_9t)gthM7Ayjy)@9dNOLlYC%m7@!j zwm?PZPr4S0O}D}jvxwcCi=3oe*83f(ub?7_{DWAEga`!Q=;pT4(pgbM)acs2f}B~4 zhya?$zpsRwudTjw?X3K8STV9an^Lu0m~d-?+uzmD z0GYhdintW+e)UhIexVkzw3IP&$LibDP5uwbd!Y)Ex1sTNEJw-$9=S^@?i)j*M22b6 zTv`Z8{DoG!&q5X^q^la+7QT+0L5%G5QX29eeD z?<)Zoj$kOpZJ;t?LKYOm1_d7$UWiJg9RB~qkzeyBWsHBoP;fD6nba6ArOx;d=X*}JjgaPj!w5y==qy1DeaJ*aa5|6kns(U z&IQEk-g#)IXE$$5Cmnw?@$K2H1%mD+j+<~%@8qLp<-7g)p;owUIx-kwkOIcIw^zwxR^qz8maNn8~|z|zYr zW`~R}Y(Ni6g*WfJ%6+*h`J>AcRI&BM{FMdL^*Eo!XFfr2yn`1k%}8X!I$bI?- zp0fDB%@-SrlY_VJhZC|&GHzp*_1n33A z9OA;9RbfIONg2s@p^OEjI1rFYAf{I{aH(ZL6ntghG}0`Fw+G|fFv&hS(O7wQF8lKi zIdX+t5umoJshik~Fup7&+uS#o_-=kb_H54zIrUwfB=KYMW6eZoD^~8us+fJvZ%M|P z=~+k6cyQ3lwqv)H=Whhi>uexhNUd^&+D`{Lw`DA^?|>v3pAg!wq#5C&=B&e`7u4>xVOU4v zX;o*1yzxm~{_|sX0rBQBhGpG3*E^LsV8Dj?PnOK!-r?twq3$?@A?pqarJu*ul3p`m z3Y4m2s7qDkM$})}KV|s$k|7hlYc)E!e;WR?=aL_;IUeJp#Nq8;7 zmnEKY?!!bt$i`F1p1^xd_p+f?U>+sLfk*YcsgtQIJ?%yQmcK$F4ofN+&Pr#duVJ=+ z5T!9TF*6iBSJ8^AoNaB8Mine!t1{>5A&b!75p#= z%Vp6|!VtVl(vh>SkGV>W7~oAK1jHZrx-x(AZC2F-;d5oVMG2~5=z&u$S`jW1)!goSdErvXWrfH z=SHUYl7h{>JsFO*qKP*fX4)1B3TI^0QK)Bk)r3VC%F`kW<_DtSTMN3|VcnBC-vg)& z<5F=Qb@jv{PgD75-bp{%^&Y5r0=Tw@4PhE-*CEAVZmu>Yj2R}SI-_dYjQMEGTCGqf ztB+n;oJYUl9Mo0vf1<3`r_$h|1b7tr-E14~KT|x$cAx;`<+)B_V)Y3yw?K!2+<;?u!4`l|f%-;{3 z9a5Ox>cYY$HQ#6@*>5YU1!}wzHojsMD0*hwt6;z)7LUJ2_Zk0Gcr#czDqlkIArk(; zl|6`9aB@=6%BB$%JsVKF-`*M;x)Gk5$uT#R?3geQdP}CDcAwlX-e5_I*~7N{ZQOiK znQ}sQ=+WzUUC#}<7_<-7M?co(~?kXp3|I*{X>m#R5uJ*5<|0DHX7kNb^E779E|F?L<(+w;=j-=vWeU#1}*@E68EeHkI)Zu88DC8yv#Cd~yn!O}ydK8{3dJ6^sRsb1=Y38SbGiK0Tpit}a%6{1X3h+r6n zo~RK07ffQuu+dW0u%NhTm-lRGiIbcCota=92ox+O^lQO5Xl7EK zR&d5A3Dz5)C2r(Gjh)Xcks19gggJqmd71AL{`McjAVFl-kx!YTS> z;*MQ4j=+??XVK!1qw=@8Z{B{!G!oZ;oYWc_S-(0vAlGv>iOiW<{`^XmM;dUDplU^4 zjkw7L6`NSsnapKN;hykXc?*&>UKJVg#5j7Ya2t}j3c2`w9UECw&rvAplNC-qGSR3L zYnLC4Xk0_zh>D??d|Ueb>Mg7}m-#`_9G5YuL-He&B-N{Rj2XM0Ms7! z2W+q! z(0IuwUpQy6Yke{Y71v4JZ8_AC*jl`&54(Q%fIh&QEiW&evEeP)UYfp9@Q$P=lslU7 zhh&6el~NTjo)!-N(DxLmd64~k*FQl{9aJ$#Hi~DLU z7oAQ`9H}dQ>F=fkx>9Pyw5at3YVb`lj`I&hPg>4Sy{laUsW!E3uVmWNXPwQ4*1GpZ zER|E4jM}%XdcF%e!_5-~Fl(aTEyjb5P#DJh0E)s7Rjn5vVdSy$5xOzaK}6a}ft}I1p2fSo*ekCbd%J_oM)fWj*N*D4T|X0b)n#Uy z@9-TxsWf_8xRjp-Qpl`NRmf{H#tVSqV(q#Wn*>T+vwO!39kaQ>qW3uTV_qjBmPh*X;YT$z1>VX{=L5!&s&BCjnj@ zjqw(KD{w;h18cg05w7lP8TmjMA2n^PX#a4aV3Vik=zN0g{1C0IHe6pPbYql%Nhh(f zZ0D$yyZ85TRhp{>)0Z0wbX@FV3Guj_34)tUDM05?+OQ^m@@kR*vKq~N$ zPX1rZ)fk`p}(WBvZ95SJz}e^|m$!z_wk$iS6vh1r1Dq(2)umjFEoj-z>RuVD=a zuxfYNN+n9s%2!vHAqrZ?TYHc;2&M{K{QN#dnH#AOtm-FrAf%V_$%~t^u$e06c*sBZ z@l9PloVuB7>oi;IRF}UuIsdG@k43aPX6jar{wAQYW8Ya{UjAN+y*c8?FCEa6#F-KV zBV*ZHxPrW1F*G`&TpdX9#4t;_Bj4%NIxh!F&&YlSrM_$Y6mfg0gCu{k&X^CaI${TV z1DiioN0Ad*?_6M7@c5OGYwUS#eUr#OIx~p#em3^!P}vO4q@21K zB_3_6wh&uzaybWpTll|q!HggJ2KH0=`R9B)-QKj|~-TcKA2tY;pZOR&C#0)J$CW#dKhdCq0T(Zm6Ly32(9{&NbuVbX+v$?Vm}u<78*Tf)xXVR*rA&9HQ%x-0)K`xjS4R>sX19!!2nbg<&!(rW1SpTSk-` z^rvd@w`r|0*6P|lir;)MLb1;7B{bm*dPUGG2QM)fi5#UhsJKXWCmWd1``|h1PhWr? z1WVkW7QiSRPD!Z?O4KbuaHMBdhIn(bKcl1u-Nmg=PcgUnj;WU*soXQ=&|FxR=#VBwDEJP&>}LL$^mvx`Qa4txg^ zAID%9P?<;gT3~C}x!puUFzkl1M;`bGr)2 zL`Q0VSZ0`H^fB^u%lTkG6)jz$7Y$it!f^$~f*z5M{)kQU^{yd}uzr`m0xx&S6>1)( zZjL@l$Wro2B*WIJRk}yBcbH9YOGAj1Z3d@b{@_cuqr=*ogvPSW>BWV5Y2XPS|8mI* z0%RNlNzvSt36~LS2?c&w=t9R82f1&lEk=BhbEU`2H!1}2 z)o6dRX*%;Rw_WzL+6vsz!_d)}5n>*H#g0uRZF-l11480I=$e{Acg80_NC%%!nDt0UwstTzsK=`HMTWtqwM-kDJ}>&i_c@HLy~wXrN~1^TWfFb)YDf&7(Xl{ z2E`+Cq<7kDAvlS4u{bx6PgpLIkAgnSb)QMjo0(x`(EP-p^)q{TYVs%hH|V269$>U} z(adbBeDB2~)wcLimFHepZp-zytQd1@EV@_XpJA-du#sW9q)1HLTi9&u_bTJ zmPU4WgAG+CLnNufhsf9>s+Fjy1kGoI3!f>8moZcOY`^sm&m@p3Uwt)_PFmc)iwyjl zgu7S&@JmT!ED^f1+sS?fEA|!QxW_0P$tu>UF|tRR76;fKUj{tZth9yJCvL#p?twVzW?@ zpF=JnaPj^)!p{7S>Xsa^)_>TRRq4pPt{hEwopWD&+C_W;oC6q%<NN9PZ#Xr6o9i2oWj1-n)<6{36D%1`YZ;$arI_ zoXwCs4&@OI)JHbR2?Hz&!|iW75O-dV#ZtWT8cUO-cx$O`^^ZAMAX}6(sZ1PH`MTZn zXz4x#k2*4FtEiO;`si2}OCcCq;t;zI=U6459mCd+sz1_;lINvy6hrk7$Zsa``IK(?4vs`24N#_Y7Uu>gv|A6a$xwVw@ z@2EaQkKz*T-~z960;V4eW>f*^Jj-4=T}Y2D5}OvY{?0>Wxr~nkCeZ5{OkHA1lF^4D z4F1nov@u!=eun{!U!@%f0)bWipxZn;13F*ViD%V- z96XsQYwiqfaXmJy;Zf~|CXyKNy@T$RkyEe#W$GeW@n8l~qH&-Cf6MUi=nO+2+_NFL zpFg2ZQ=*0mMCl~Yu7DvYAS7=T4`GoRA;}+ijAMTpD9<^X`RYqCyxmH=72Fd0`>6RueS}Cnd;u~= zW!}ng5iBgX|E*vpH9CYY{Q2m#Sm_o0sKc}!<(*gTE6?i!ng3;+2iWn(C=-&p>}T-i zOzr845K6-vJ%xy$J?6fR$~_-c*A-!OP5f?tX1-Y~$xv3laC{gv6<<0HSS94x&PHS} zxtZn$H`lRE{C9eza-80nx!~0$RALRfRyC-79vH*~V*^!__u{Z+?%d9m9)(5|LXk12;u?P zn?XYdlCMHcDyPiswUGkP-d9!5?x&7RfDYLdA;gJxQo{d?S+Dl-`rcX+ z{wH2|%CXgIz%5fpd`N5DPW6Adr>A^B1p^MPI3SrW~UAx zKjdW`PQ{=WYl&DMIh0JMr+g|>_1PjQaA z`*&#krs4J6t7-(t?>;Eb$)i;-P%u*pgk)k;*aF_nsX`0q5ZT+s)#too&M$K7f_fUJ zNI>=7$F=bP6AAUz!F2R1lWGL)bIQfQKT;^tWH;fc0TTZ)ZBO?JIMSF1)6iDNW`F2M+uJ%Kq-))z`1n>YECdEP|BhB z6V5-XC~rBNHx+4qzat{z6jkZCfU1l$nxz0A_$vJGQSXYO-~mFiiBlyo9D1z#o1E4F zv?{K06gpIc@MRPADLp>vf}X7!g6K?AfSrnP<2xccPSN8m;AOY1Au<59u~wZtQG~lV zRVNg;M_nGVO+$T$xIGM|IGby~D#xveLr^E#HN=TK*!(2~Q7&=KE`tFN+Mj_uIX92_ zgt?s$kowIB-ki+^2Fmw1jkwJSp&`}UL@PsNPo~Su5BsN#2BauX!aU>ssCw4*dXHekjH{*rEW;IX1&e&0z)4NO4`VqSI z)k76+%RE)(VN5*`lii{($WVx4kG1wod2rjal={1x2IE=6p~Z^#!L}33f+CMk*6I>K-Mw9mp@6Y8g=zcT$xvBE39d4SIa`>cbZ|kHBOr;O!3LjKvWU1w5jKm%>JQR2An&l>zM$4^4wcEi!jrr zg?0U7&*CD(W??X+;f?N`+r>QMQK%s{Wf-Pb2nM!bmmxdlX9rSM^;dhKOgU{Cdgm`g z%?vE1H z;!e|Q!fbKnS|`%fTz3nkSi295DFL6w>8o}rko5Ms!=!aJ*qKsgLIp4Liz(~MTBU&E zKFi#&Ut#MdV%CyzkJw;7LxLQ3;|e0dHJKYhJjLh7;0*D5D-miZZPlM=X6fDlw>|_7 zG*Iy#OO34}tc67l)wr4M*_^(5k25nBcKEUf(0pyk!_@v#ozW)qT(~)en}rVbLCRxm zr58PfA-@B2-1J}UqRhwL+JWM!(z`}5(oBhQdt>Oc9g#-MANy#GYoC0PaAAiE)ucPE z5UwCNr0;fH(oMcGG+IboSo%GYevp9>9t`EmyUwCfwpaOU$o*LQ>jH1u(rH@h~({v#8H3BjY4zeC7c zziFPn+>R?N42B7~f>>-~81W$X$m?r}&yN@dNlog6x$2DDKWQe`UtdVSTwbmGh==6* z#=NrUrcdVN0dSkPquE|OxPVebp<86O!kxj_MetX`=SLj!TE^Xdk3SW4nmVS;s;M8& zyi3!HqLR5Xu6yjC$;7YqqW#E=+BYeA)l7^AL2QMfTb3^z7q9~?TZNoCT#E__fhlb$k5c3a;8DbQ*!ZY$S zZqfhR{qXqC8fIKuX(>SI!&4^vK_$^f6*B1& zE~Mifwr#?CWE8EHvh+``s|AK^o!G0?4J?UCBjv9}Iuqiyx`?(W4Y?h>rHw74g@OY!1XTnhv!?k=G?lmLaG zAJ=u?@Ao{%``7zlj&nzTGqamLcXno$F&Fo%SV30Dnl+L2W4bf^Da#p-MH%4?-zeg5 z-{m_22dL|4|gyDWR=J)OYlZ zgas998JI2eM#h}%Fj=N|{1>5n(tUmYH}Z4;XnLW8&|?z*sm6pl%HLfu@1%Pp3wfBD zf{5{=T3aA(xZ#3 zw3eHqk$`fl#0Rc1sboTg#Ye5we6Ri_r*t~(Y>r+h8%$utrQT$(e$8wDlNYvNOGL7( z2D$%+E8e(ZaKmT+R3t z;cu+NuZL+7HO};)D`coKw8%saGz#E%x1l)Xu@`c+K@WDtR;FdqTfG+yar&V9C0yO6 zyw*T4r|0dvzP%?|srIl2HHa{le~04G{#DB!#?{-+e*C5+e;or(=-$H!uxk$6oort~ z*Bhp~ty>{c_->RD+a@Hc%t<1Gn_Y9JP2|gY|g`G;rGP;4EW7=;aghYYS>k%S#r^@fyB5UL8+P zBC%0jB}*QVw95zcLEWedrCs^)x;wC`3WyQowACr6YANTPJtBz z&tr;H#|94r-)f+TT4J^kiCn>YKq#z6_dd>(d#1ekV0fQh2?qMKpv-aa;^%hW>;~9d z41mhQ=j^WlEu-OPIVcaTE9ZR`OFF#CA<2u&Kc%tZKWoG@QloPjp3AXpKH&e#=zOEk z82qr}3Ca^^AB0~HenA=ZDIp_4i^JE*G}=$J&?e=EqBgrumik}_!@DKmlmbXAndE#- zPh52lZ&kToE17L-feqi`f!m)|pV9h=)C%-Qvh<_|zUz@!nJY;Bt@PdPOYsJh7Ie9_ zb0W!T{3b*QBp;KpuVG;MxVB+?zFnku+7TJsqm-bb9F`ljd}imWz_6K&X7bTB6AL9- z_(eNCV-`dE2l+3tVr#|gr!g%|j<{wOpnN_>9SgQ<6lM(~zkRX=5yVUNg+=3M>r?Dc z0Nwej@=Yr>%FTO(1nCZKZ}yh#(>~5IVbTIxaEKW)ZRAB+Si`FDnTMB=->xU#lZI0g z-wy^7z9{dx0Zf|Ui`oqxXyJV&m*2g1+cH7_R>WE%ZS-IaF5)L}T;m~DY^o)J^y@mQ zxxFm1o48O_^5VfHxY&4hfD5e;`jKFkI8u(%;6u=qAL$o`U-JPrVKDa+4eY@^1(QRD zmejV=Yw_$uR~qWIwxtwU82jR;W`ZK_YQFl+k1uH@{O)H>dbX!NgWSbj1QuELR^@Tl z3kUoPI6{29hXd+GmfUO>`1NB*Gf95V@9%#Uy#xSTg4+iFy&|<7n0!#PA_e6@MW+hR zy(<-{R;eMiS6YjA;LFw zR8c56B-BI)GHAFyXi$RR{DR@DGI2lTUBBh6I*lS7c@-m#@p_rUDde4vY_jCQ?!U-L zz63UBG6M4eaF*w01$)UW9_TJa_gm*{w*sYR^PBQJ_LU?faduf~|2799_-?rD6Q`B2 z6TmKKgr9}8l}B2%;-}onDBepypw`wXpxZU!C7%O|moXGk_8XG6H`md#s33#7I`DrC zL}WdTeC3aaRn7hggOFn#zps_%k^?C$5$~1CpY}$b$waYEz=rq%YN#!3lI6+Y(ZP44 z+^tO4&0Z6*MJ%XAq=^hjK*)=ei4?rT8E2^sSxA5GhdDL(X6)X}SZ&1x@Sk$eZ&sjV z{E*1$q!S1;ygVcWw_hu96W#)HMAUZ*BgXMPaof_dYgeZi-UPEYwABIUe{KOf4nRb`X@*E%$YT<0{%_H4L^8MI zIx0eQTGP`7J!T*&oBqo!J(N2~HuAGd`>b7SM?3xten5g26`TCVkOukmxiR?cO0*~n zkaOF`+Ar-(lT@iB2a599U}{Y#j{C&s*QZiD=O1tXA=kvBg|RymRHs zLT;mo{dFdh?~dZ^$M>diA#f$;FA{~Wj`N({9;FJv*?9mw2glnnIeTM<^K%;PO(|C1 z^u4Y!xdKpLz!&5R`j+^F51v@5(3=A-v)W*Dh{a8G#jK^=gTR&K>HOIZT30yGN^bkp zbe`^R*Lx?MpQgl9xh6~`IUbFdxA{t2XClyqb2uGKk4AW%;X1^B-h|}BhE?%+k!19W zV{#WuKDUFeb=RyNKt2m%2NxqqHp!oj(ACqeWRM&4;L`*>UxnX8U<}@Wj{pe_ z=B@ts&{rzjK>&d=4ab3O+(i2`W3Y`JHFbpGW=1r^0i2$;+d)+g*!!|*+A2a}{W&XZ z$?|SBfr`Q+S1YfyV+RN1OU}aH`ky~}%^-FtYBJI659D)4!J(@_Nb{ckV2T-&=ZujXMLL-SO1G%5!?SZ$e9zY+@A0D^Qgp8+s{yF-@c&nul z4}17*MGam&y|%dwf>7SnjBl$P%r7xDCi(xoU6Ie-Q88mcKyJ$cPZ6CBOP=)M1%J+8 zex!Ym-;2VWlr62n#r?dS#Oc0(WqyBw153bW%F zaz)ZGN8(jg6C8id#rt&;nVsjA@KRN(SC0G~P<#`l_BUb+0Yn7hr7^Zj$zvDb0bDg( zDpfWk$t^39AmwylGfD64Jb1I_&O(t)gYnqMWA zqx}m&Ce_QFn!bdxL7NKv_>ja)ln9u}MzB1Oz@*|Rs}vfsH0%gWAoCoFt$G^D!hX^S!8h*L%pOwF}v(@H|wT(&sL2e zWuQ$Y&TG#^Jl&Y^F%0_GV%=04L85J#Arz9@H5>J_S_;%t3$!1mCMH3H*5)qj6&#q?9YNMvtu>J@ZYa^4CX0^}0R_BKXk<+XP zR_Q+1^;ZDie;9wQUfZw`NzvB)$^HmH0k+AC;4L&emQb>_q+8Ti!0+4c^~%+smRVkN_3DUpbz_55I+3bP*-PS$D~gN-}| zb+dpKdQ1xo*hf{BKr_FBXPrbG%lA&9hLbV@nGNp}943Joj&a>|KA{7<9^vlw*pE-o zb8qN;F@mCy#Mp$27+Vj(s0OBgT}#^(sy7J5cAlSyfrvZrnr0g)LTyEg4a4pv#6(Nq9lg0rQfI0;<}zCX`Z5d{Xo+^>#nfA^*cVe zX{Bp;e^R#jsJUnMHPq9EgY%3T?S+{3P<0T)(-Xw_Ocm83*&W$V!MRO(1v!6$$)&*BL3zDHV3a1{KXc6kARLAvrk=P^5dq-rpS%2 ziizs4$mlj1P&ohEo7Uc65o}jRU>oY9q_Fg9ychC3e^?p#d8I(OpQ!3gX2?k2@nK*c zVkxICx5f!tajp<$f!!Y>{t z7CtKzrZ!Jj8HBNmk}PrOngJYxC|*VqK#N{i%^LJg57ydCG9Uk~=m56rQW-G%@r6(N z<|p_Qw4pir56~KxHOq^n(0QB-&#`CfdV=GEjw1#}6NR1CQ@wQZjQbuO0}-cl$F)Ke z)hjScfe~3fP~B3VPEcjUzDM9jca^DGClesgEn4Zi@xiM{V?A_1lZ1rdy5+(`GE~?x z_K^{4WjW!-iqn7v7L9duq*?)hEy$PjDM$6iIKA*Yr-y+67i7enIZk3)KmN2J=em;@ zMphfJ-IYGum!D8emI{;bQ*S33Re6=`z_`|+&Og8Bl^2}19yn^63yavVbA0X zubw`2W)L#RCEp~&-}U_D;nz_l^av^yF)C_1bMQjX<6E)sf+!J)cf_hKi`lAK*oT$J zLO7lxkg5g#E8j@CvD2BYCKxN8{{r!9kJLrAfd~&vw_!)LjkCS~&M?5p$$X}Oy3okD zR~#PwQvJ>LGFnevEHjF(9s`tN_-B3$6RZ9}S%a+t_Gs`oMWdn$kK7W};p#q3vkeWY z;;)KP3+2_TbC)w)NkB%Q8e^kHiluJvnI|f*<)D9@b++$GES}#bjd+B4^9Vp^a$b+8 zwNo6o1QULD{aE9g6JUtEIKXe&Nmi%~k95!>l#${?%7`@9C*L|($ypV&=ah$Qb54ts zQLQG{hFgJv%?y)L>|<=kf*-w@KS|t@R6sWVQ65rac(&r3H6}=|L4`1hi9uYY6G2k1 zNq2LAr*@Lyv%^oZVyj9x(~WV6gryU=v@yJ07djv~W>vE~QA^3k?RPjV#EC1|DTV7B z`0bq8fbZPskcn|o2%52@>ukQq{8W)8Cm|-fb zD&`+&ngNTOH~3B?Z*m_B`&K@H6jJhbnMR>{NeQ*f%`yTk8+DJi$Y|*CU47CRKLb`& zl81wP>3Adpxfe)LcqDt66Kow$_gLycqC)aBG@ep=$yxSIxBfl}rU$nyCQIZ&T6v+A zQvC$sg3A39`~9VPKOn;W9ps+-_732(k9@RW@DX{J4a%@<$eBAXyK}6j(6!&;p{E;8 zT$X8t(#%858WPR9m1PA)r2MPfDgySG3IK`5^ zEBW>A$wPc2Rr~#%`vbbxuF#$J#Ih{j*R7=x#!Y&>b_K|b6}#;|lzt*6OdJ{%v^S$S z0}_)&s_GU9y)up;kwU!evmc9R6~ms4Eum5Kr3w0Wz25D)r5YEZF-rl1zG_22y{)@( zptpa0b^0ptA`FKTJ?5-^5ij=`SPQ)nHN#3!^a6^l3N5e2e(9{yrTXV6$CkTqJ&&B9 zus&aF%nuj%buIQ?1o3|kZ6Y;q&Ad@dRXL>|&$>ry-pNY;(kcgwSl~v8I{xgEAz9mo z)(l+{*=Ir_5G7~?FjG4qHO%L9f;>)o^CNE)AG6A5rkrxp!gEF@YJ3o zJuZ)eYkf~_s3lauof16>AoNOl4;gYoj7?fV9G+{@@b**!u>o$z;|hWty9tY#)+ysg zMi?Ugmti_ED6FbZa0$u!PYW9gUu#$`lrHuCK~`pm#y9TLQpp57w!2bcIpFm&9L<}< zDnX@-`d=c$DCIX<86DxDWv$*tW?m_NO{$Hi%(z!A*?$Y%2=E=$qR_VfD zhULKNuensV$uc^k4rPEVupi0o5-clNqEY zl+WjV(u*rQD{6|2dvtpJnr!;RITpG?t1Cqy@cIzE8JI~*p0O%-dBtrxee%pyY6fu` z@T}l79@;@N`}5Yy%p63M^j+^i=>Aa_1w~chco~e&ydn$4eIxboI>W zSxm-1zt%@-pcG(+E+KNR>rJ2q1EiNrBlwZ1UZN$zv;IlniB+yqQH*RX{(joX&opWs2t`5phx!v^Oi3hpbtb{U`!?Rn^gy zkJBfTK=R>rO5@2Jj4a`a4mJp?hxL26zuSvPtPUeIDtVv7BP~6rm2D!M)Cq}`=4$Ys z?P}isf$r2g>`@;;0MRS2QnZ= z_EE3K@)Nx8divb0{11T%{xG2k2M2z&o8o+epU#bESF(T0P5s5((lhWssE;f4 zOjcktf^usig7g?yZT>$L+<2IUv!_)l3K6Dqg7V)Cv#@qe#(oK~o)a#khnFSnv2 zc2cOagtuNwakQRqhEOI7W!~f0SJ3lX@2=NO{i{3N2_Cwj1)CiB51Y}D!>p^!M(16h zvx?{*_E>Mv9BeO9hp3#B+3|32U%d+YJMEVz*_B}*cwILQ=pD!n_ zg%=@U)OCzhqcO7Jeb!ZL2uahaA8M81q)b&{g=oywpBJ_LS^mWpyl_2x2xapR9wgP^-MOnyePsmx>jeT7V*I zZ70N*{k3yi z2KPyJMq|lF2I_E+a+EN%U|X+VzT^@LRpw#a(cQzKX ztae5mz^y+DOB}IK`~8Yh3Fz`>jiVKjaVkmR16UUPLJpDpLVME_xbq(d2)aPY^HE&psAFDGR~w@#eu%&-(Qvj zAAx$1hJ>*MgIU*~6Md?Ed2R@sOD1vPnaeFVm)P|ke}<5v5Z^zS-s z0$GCR<~5)kFncGWPnLW)?P_u*6+P|moV;z6# zSCNZx8JIUcUiAwoMqLQ~8*6#>rtOyh(a50Y+0uVcdK9_z7ZUT6X9Srg%}cP2KbBHL z`n-})eT3`(Lj*(MI-3P)(*WvVQ%OFQy`h{rpPQFqB)R!e9PRp0203b(WsC;`tIk>wgg)_L?}4)_GIbm!1GQ*0alU!tGjN{{Luya_$l_mEEa1-F03 zSY~J~62}++kgso*4@8YldV9>C$|$EnPiRQr%PWFgOZ7hZGyV%UBDA#7RB$*`$d0Z0 zcH)Z@Uu&;1|Fx1u{fVBRtWS%_bJ1>IhJJhgyH?76PYo(VCOE9TBpdW|xo2%?q&YZl8S9`FO zSB+!+JZ!B$(}*ze_6Dl4qIUEjSd~fH$6gCJI7x%(6Iu05vWyNYfoKjO%$9pN+tuCf z-=&zJke^3GE6nFr%%FeakQURM4R47`Dk=&xsXNO}B(2@i>Va0(3tOt5=7QB|qq zSY|P{E9>sMCrfEc8x=v*rYO+@N@Z@}zMZ)-ucxqsre-y&>dwG<2ERkwC76_{^*omz zpkvqt0g?+@H&;^~(G&N`cFZe$jeiB|g_DE_Yd;=w`WVn|t;QNB2CuY}MR8WKEk#;Z zJNS@W%)_Yiv4Azh77zLuT$r7ryrLrSr>9gVu=4{}&)50VFR-kxsV1nz@4lWoPLuR$ zIDt&2Q?QK_M?PzwIs0?!+tMh`OBBkJ6*wb)lQsbPsV`JaT8vGWvf@mwR3N`^vY;Q3 zGGZeMq!_G*C|lf`&?>h4d{qwmls zn#B#ypU?MdLAtU};&Nq|FieRJED6EhSVw}Kk@ykzJY-CpaiUD&;v7#2?2w4a${%%I zg_d@z^I8T&aPwGwNEXhWPAL7R|^C7r47Abs{Iu?o&5;8@O!QMyFW&kpGpBX%G7 zgj7+`=VFO68kf~9z~_69MuWVZlM8GdTMR@ zEeqG^sjMH;S@om$LvSbuV(kob2h(|5Wm=(H(j=!LdwrfpXR*oxIZ+Z={IF@FZ6pu8 zSdJ*eqdO@jzDrI6;C?VM3eBGK#=j?KP@OklXVR=VDf{{tW6dQNA(o&`uh+E z?GV#+8}ArWFzmxnE{m-h4PH{S1af0(@L9L)p9BnYnwoxV-QFXYVAUY4q=TBb$YP>w zP*W;_BQ&>kO}mdjAq{ghLM%Rjo^RK1PaAHNPTZ)=G}!8?@R`a(ATIF@{QI383okKIFYMTqoy&;bmLFS^xrQ_piDnK?E=RLj)HalR6AnLOkMN z^MYUe$vtF+-cfc3D1w8$_Z^}7E^ZpsOfPbChzg4fs>XFOPq8_v@MV%^wd*p*YbB)q zrl(Qk&obB!{q=|macc8O?IQ{WCfOTfJhCY`7p&2Mg&*hK%JM|SnMw@}H{+)l)A+#~ zmmPp7YM-D^xgQv%fQh(BR5gG%2JBPn>(V;x)j*{cW7bg7mFEFvR!2K9l%%mQdEn^5x70(0!n<5ZJMysl#9D1w768@xB2ONtn#uxRZ2S8elDjPp;{|YE0l_&g`S>X}w zwyQzlH6cNkjIaW~umB5|=OhQ#@m!9kEd-^*XECz+QIo(NP?(cT*+cRITmOkUn3s;h zLNO~;p7BF!MIF^39iH7tu9$07;#Tn+y&mFTfe$04zMt?X*2LC7#FEJ^YN?p)PcBhF zU3^WzsLny15j~EHk=S9miDQazw z@_W&2H;1t&XO$*B9EX)t%gg~;iwq2!p-S>FsUkn09!kinQgM=I58Dh%cTxu_0iJLQ z@)uhP5j@7rgS85?VWeOITAv{jH3d-5so+s@S(ru+QA-a+;EtmNWya@pG%wQ3QW;7j zugI)}Sc#j?2kh&_d@OuUVg1kgZz>|@{XL?7B= zXl!m>e-rh7db^V@U%e)&BXTmb74wgos$iNLZ}f|Hz>)7+acW%u?e>^5f4%j99mm1=MczYAPsF~|v@_}pQ$*wOL$WDvH?5Pa>xz^1 z6a&BbV3Ja->en4bIap$cudEm^snM0a%CwIcw@}UBl7L)Xb=BH@te`~?QUkSr|Co7s z#0w_j93+&cO7>t4EJQz&_?)e;PUIwz1M!#W$gBEt_4o@i4;!s+__SGymb3Xn|2HOZ zP!j8VW^o-DYSZP-CqL=8CC?X^1@%Kp*j%(88k654jmG)fk=jw6xGbUz+ zgob8d9NPrIQN-A+LkcHV(dz_TV&W%8 zeUkn2ocI~Ih+kL(v$EK(lGzFp*I(yjX83oas}RL^6@11Nefcp#mCTt@pg?mh9*B&H zggJzrX*3nFO1e|YGkzY)EWW(`Y}B%(sd(+jtvYi1Ldz_#9jNzEd-h#K!nLx%p4QN`^=G=;H8MMYDq&`!z1YBdxi0S2YwJuSfU-!b~?2f%&%hM>FgKZ zD9KZx@5E&V$eSwTIRm%=Tx5^JeDFqBA9ms_k_m_&q(SJ2?z!jM7Nz$SHLFvTHY>aO zXKy<1wmf`^r7p+&wS3)D>Al2NFHf(KbNS)Kb!y-Cm=N9^LbMI34CQSN%9d-lHux(_ zmhIHGltyYZ8+HRTyniiJ4x+4_b~Mpy(evG2Wr8p=f1r|^qM;~gzmY`Olo8L5gT>O? zc-pr$tL6TzX<|voKNNtn9WpC%E+rklhN(1LkV=$kaOWb~%H`buY3yzXU*}w{=*17S zOOLCL1LN(@405ja#!Ke(HCvq(oQ(fwSEW7Yt7PAyalGjte_{MyKC#x;ak|w>+K{1I z97zrfP%B{yKyqV57`*Q|Vanx7D0(b+Bjr4L>i&u(zFN@mqyX-|CYxe}inX=3`$=3L zSlu&ucvB1J94Foz(aczV@m{{uc?eQ?WWYPf(?pqk0@xnY*=movz;aYvvc&PiH*%!n zUVL}dRo#B8^fQ9BR$%vc?s5f~kwu`fbNB2zA(?E^V{N{|m&0&MFB;0#AM4cDF!}-h zC!N2?H_7~}Hp%XVS=wbt)nnhucHuk8WHW$a<7WR5t20Cz4BVxv_lCgTDHGxge;t6{m8Ix`dFv+ zZcnnqs&n&bP%UW7NBI8XLcVaAitQ>}$;PYXsogvg!Fsi2P`;5?IG%)G)&+HhTjPTB!kuCAKheS~u!;HImI<6VS0$P9$j)g!4sOy3ru|*|= zQB1M@eh8z)55d8bJe|I70)a8T`fN>ra1}_O{$P8XE;<2*c~-F}84;BcX0kPoG@yv5 zvDA(&sb3(AYjL-+VDBeYN`g2Ikz`If^ylVLR0Lj9?h$sfm^G|6ykj`ODI>-oaLblj zR(3;~LAssuRt4w{xdy?Rr;=>r#0}mA=?l9Ro^bP) zf}&xa>`B5T`2AkR_EV%qbyN#5=MUBOhJ@`e&9q7M&!*lbG5Cb_!Gs7~k^N#`*=iBL zWeoLUOSgIGns4|`Y)s?&S`%s855#r1!lR zHFL64FrC#=2rM)^bCSHgmu|r*W~4$b=`Jo^3OpM@1M5)jWS>n94G0pDiP)@LSTFTB`IsCvc<&- z&p8E{-@toGwfs$&Uo1ZOP71r3OETxeMc$bwxLT9}k;_lsqy`c>C8)gnrLWBW@=epe zRT4HP%ZIWlr+CZJWmEeDcbM4$ zy>)W9;;|fz+URA3qD8Ab&>CQ=7c>dyJ#2tkS3MVfO<<$Z*W<8M<2TxJcGup z9^(Rp$n8L~1NQWoXx`tTE0;&UfEDgs>KXZ}of6+@AuU!}6eifCfJZCal{gECl?u~b zZHO_a+K0ps)_C}Wz*%EP^sVm_cXc8p9Us*%Tx0kqc*oGdEsNY=ovNc3dg6X*&pq5n z+34-E8Em|2*Rw)i+k1OQ+l<@y?$c3)%+rk`XirE8Jc<_7-XjgJd^lp-^+<*h*zcOU zMFYpj zNu)%&+D>g_D8)zYc#}RiFTH#-P+n5UQlI2Pm!}J|2jw?c3F%bszs~avh)p3eaOKSN zbf)hdVY=7Sm%FaI>lyWZU|H!)1f|y~cFFW@T#q$9xU%2-?|4414=qOjk^lfTiqSMw zR_66b%t(G0C>uRSFJ(o`;}Z>>lR^>QK3dXy#&J)8%}3y%T5F4ud-XS z?s|XO^%3N41gZN1EbH4&N&rx8iMt*wZD->B(|V=qT&ih-qWPzDCI3asJ_$5DFuv7q zso0mF%IlkqcZo`0*ST^`auxp3x5XMURY|?Yy7lhkmUk-nRN|bDXWX&9FvTWbPk$Jh zHNI6(TTWO+Qch}&d2q$UyY_OMFG21|KeCeD;aFE)<{k&PTGROIX&9bpn};Y$&3C(_@RZ<+%Sq7Ps!&AyI03 zDAG(-9xvs&Ati0^g`#ke{NVedLu|pva58wo2?Ta0q_3tzqju0DC-dmc|3GNB$IyT6 zb`58;B&zITH8gLnjr%Yyg^lWLW|))zGKYVNds7KyQr&wMIVLyo_C|tU`qF~-ume$2 zZ(CcBe0z(sEb;6)0?0Kju*C*c2z^hKUewmbzhSu`KJd8u)rkvl_^9Jq3Cq~jwy3cywY>> zV^3k`F|0fIf3KW({9h|i$2pww$Fz9TmG8xo+!#XM^rheBowcJs<*d&+b`2UMACiw~ z|3y)<=&JmoraLO{4Cr=gN-}aNwr4K10FB5$&r&w&C4>9}$Uf(oJlRym>{+|!!h z=KeQp_ABE&VPrvQMahw5^I)hmRc`8<0|}#eNxppUPreL+sM1I}_7J{5Yfq^zfFQeP z>s#j)=Mv};8aX=(pgJ7;R|MUtGTAsWY-Zl?=*En+0b*(Kf(gvqLg23m4^W+z8n+Ha1!*r{Ee4MZ$djS@+8$hGzDjU(00fVjXLEzd=kO`2Cy@+QHm z9`}-P4E|DD)d(gig^E*+(G(z0EY}|L++?qazPJu~E;>~GH!_D2iNdrp^h`(SU@a-Y z`N}~dPRtr@`;-lRYMd10UfW_R70>lk;6+Y-8aKLN74U!J_Z5~j1=@Cx%X_XIk5DU< z7W$jtXKsgn_l5K8C^POoEh0Wyq)65DsXaJVO15zKGNDswP=YkPfWy(07ImGa35v4) zxK{)!v&2L18Gm|7OFu>NF6a>H9eA%w7M`v+7b9QIR9&Bt75M-*N&rsBD|CJR)phBz zd%*vD?{>Wq_UKJ7c6{W}(3Ka`z0l4m2OpS|r`9U>C4%ZyRoX22W9Qvh1-Zl@lyGM` zT?}~gQ1jib3R4pWwS~f>DwBjuG_M4~1b`<SWN}oeLA0Y z`)C8c_iSFzJboreRXEKMnD!BX>;p4yEsNXTj)un`r!!s~YnrQ0sKIaYERW)#B^m#a zDkWwYyo&Eq>M@m)rRgRQjYo?UMQbScV5I@=W&N#6?}{qBs_^sMKzrA(mzFyBRzoBI z%dh)czy{5!QT=a@B{X}tw}LPIdt>>5N?abf;V1&T)+D9#7pJDl&*HNV3jqA|;E>%{ z(3Qio$wI%6ZCNmslwXg6SuEO!2>D!@0i=iJ>BBMN=D4ne0+cPNLYESsDV>ig6&43v z9>YTnT369{9c(3RMzE`&%%4U_kuId}a0zAn)yS%%cS`B^$ zA0X=aEe-qG=Y{N$rpLB9b!$)3BovL0WGNR;8r!j(&-(Uqbci%Mq zEjlZIzjSciUWT5Z&N$sRPJdqAr6mp!QzaOD#fch*O~}MFj?-|~j{9B$#N`URlW!j6 zUD_Pes7KQf!x$!aFZ#&F0hA?sPZ-1v{ne|J?Oo!R<8yGtr1I6P-c`tf5ne^EP^t6n z?rN>Q_)XQ3_VxUK@q#+9y=uW+C`mnnBuJt+T^B=j!-Napj4@KtSE_59msuR zUgJ(fcBB~nGf#cZupjk?yJg=rS9F>t*FnFu{=Dd*nTWJGo(9xLB857C0(9)0m!DX6 z;WT>(KqzGucv#nrR(j{f-!l}=-bV}V(#_})j}-b^PCPOpzKqZ?wQ~3HGL+-N%SpZk zru5M6i=%l=cY3cvO@!*5+*D6LWpnRK7fpTwLks~t8{pA*#M=Eos5MZv^FzOAAB)Vw zKkv!3AN@|c@+ta=dXY=dtN2p#S4ZnL!NjqC&;unAC6wJD-B-7Wp@NO<>o+M(w1Dpg zmMRa@swRKXzd0m-|JkPsf>W>EpxtyxYz#^i#AhPEvbNu}`K8f#fd2P52rO21k7#EB zZK8hgX~a_O{NO9QP4*%hCHB}Sb>AbIoDNspRp~DjdrzNn#)zX(W1?SK@(d`khwU4Y z$=i>0bZ=!o$4j2J;c7Ms>wFoO*!V}XgT*%9yEs`!Z=D|s`K!rUKw!*9|Tnx*gKzFt=@L5o)C)D?&TSsPc(RQ2*fEWoI{&=4U!i6-pR=(R0h1v@( zaq|yvcX_-gSvO{`aY){m&dU3w3o)>J=VoHUd^K(rVGy-Jv)J&22j$Nq$!d-`doA?c zr05^Zk%C9+Si|);o`L8%Kxcd+4Xbg_@w#5xHSH|^7p?9%2^en0v)%A&TF z#}U|j(LuG1fIhoU-#B=h{JOZ39f;68-1;ab<%dIhx4?+jkmjGw)MH4@kM{!Rcvab5 zT{t?{DnyF;n|09^?y8`NpO=tx)pbQOwldRI6*ji9w9mM;*Ex`^slEMgtYyDVoPX;_ zwEaC$qKxyGvza*j%460mih$YS4`~;A&ahHq*qu~wHm*Q-295UsdXu1lM@!6jzVZ*U#cJ_M&uf3n*D zAQ;U4q2Y2sOFAlUp^?wZW=78jD=jd!N?r2&U6UvNE-M zNbf+OkD7P)vv*b7^;$}n4VIVsr>dftIvDn+OXOuQY7V~66nYm+${7f+Y+L8Zqsz^hWIi=Wv*rD^accPn)@!JNoBq``O+nzR`eOhyu+^Q&!I;vpUfw)}d14sbO+a4Eg=8TMRWefe5?0^M%i$@T-)PI6Yr`th^{D>g-jjSBdHK#3=J@w5Ek zVq58hZzse~wQ=Knpt$9%+OX?*zbO;+3)#iOm2np423eI#x~21YI&DZ`Q&`EnMe}I0 zuU(@G?A>uNzckw~il0XE*A*?B1b+M5eZ3N`K#P+<{Y?F~B@%Z3((dbX?lYzuH>}ml z`FmX>9j{U`?)3vnW8)Nom|m;FTY}%Y*n=PJ+LuQZg(;kKjaA|5`r#am-b;a?y!%YH zn{eYZWS()Z7M6m;NSHQUxcRf?#5w^K(orz;;!HZ?yQs*_Upr5FL8q6p{BH>OrhVNL z+vLAq(($?DbZwZ9PY^P<#(y`$`W_)N= z#ONn3wYZL-8-f`TF3B|U*ZFIJLmn$Xin7^NQJms>r-t9lnr&`L+ErQ^LTy4f76CEa zIn*aL%ZCboPhX`q=|W~gjCq~sWq9HOhURGPC+Sj_&J#|SgcuNi>q{PH7W-h-?0XEA z4%^8&@kXK-LIM4MD02s+?yab@CQy9CZ?uj-f7_#C$o(ee{PE;kh`MFZu0?OBN0!<> zv2`MAE&9h5TaK<6{iOx`wEym!Ml}947T?E5B&b+t=;ypq2DS$mh(CKm<7KxL9Q#d< z4d&s>3~iboKD1@bRX3xq@6Q$A2=|eG{0g^AG6%fX$f=chlmeO%do*#J-R-=>ffo@( z*$AR-&_t36O(d*2Asp}tbFn#(C2b)grsolQ2aHY;*=|^} zwja3!v;Wj&i!iBbCRd z4+DF6|6g@q6%|*|t$BbXNN@>(;O?%$J-EBOYjF2Ka2j_>a0woQ25;Qm0*!kI*Xi8* zzubA5rPA^1FmB;* z-cyL+S=Y9~?tkf6f%0)t-CLF*dxGw^<9<9o6}hk!E&4@E0?*$9f>r3pH5Q-=*Zpye zb!Qh`-qwa_PfwHuFD6d3Snox5_uFZYwUd>=i}n$k7x4O0gB+Tq>8%iI**7G<&seG) zi5NYnyG3z~p=9sgwX6&$Fn2^PM&nl>OTJ}(JZ7qD#C`iasGy@MXdnl4y#CXPiv7cy z;V!+n8%Or43*Q6{s8t}_&(0dku;PCB;s_;n<3eaMcxoxe^vVJA-po24K9zIcy~$n9 z^jGgDw@xr>_r7!pdqLHc04bM~XrD3fcdfG);NcrswQx1WDFkK`jfsNeIE}!N+n_t= zr;%KR)Ablf`wjt+v%OU=SQ{n$!o0TU=^{T+%$`{B8A?32ec18(X`xM=F=ga9&8j&G z%sQzDaIGWf6RNDtL)a-VvJ8Kn>h$zIL?mpBJXSa=({#C%%l8;MAC}BMG**L4SsGR{ zmuOb~jVl_4&_41#+muGNyfboi@VKbw(M2&IAA@L$@emaQ-NUX{#oZ@pAkd|J~WN&#%#} zpCHJC63=fTCI_n!z1MsnC_Z9 zi`LbV=vCS)GOoK$L;knAMwpw{$SqA`1=SZ=sz z({hyH?bd1$%DrFO2!8${r0RO~h_KL^oqLd;eZcV`sphNA`#I`lX_cL6Fuftd@ig%% zdr9)9^{Z5x70-b(wX|ECBbJr|OD;XZbSBub#*G0k8H##tDL}>_?qKcU{WU2$EpyA= z)~#2cXQqz(@c?&f6@qmpcWa;ttZq8tLUPXS_^6@RGz-^GtHQn~Ne;sP%8ZccLknp3 z@LH>*G%O^X|AmA;EDU4uO1pxvJ*xnnb)x+oOsjAW&tmLsWhzy6euC;R2)7Q_=wEuV zQd7nde^B1`lcUti>!UH-o3_47rCAs%qniGRR=7vgA~s~ld--;V|GrJncqq=0_5;^QQ{1rDss{9qT9}`TE7P zyG-ECyYQn@Wn)CP&StaJE=`m*h0e3X0EEM*Ry$|6t3FH5IF`0Z+6r1w<=(7_>rP(# ztZm5e8Yb0fW_BMn1tRUN99d7S=V^c7Ab_5a%ZKZwV#tEo+;CzW8wtB;tpbemjS}|W zF~qOWi=uk#O@)7lrlBytYL#kSh<9P&!PDwSvk~kYR5454xbe4n&qlfn8V0n#^-d|R-&FY7#>TU>TdN7@)_Nh9;T{P#n3Xt`cASmdX# zasG%AEG3bDp>tc|9ZoR;i!eM&DgzVCbU9e_N!|^1KM~z8t2YBQSp4N2XQh7kPq21FEZ@hNJ;B~d0d3wb{fj^mC zp8}hDNPSSf*Ud*e1Tms+#FC%;Up4gCKKX>w&AoPPo&Nrlc5@5L6h56h{)!hjO{r)T zMVwAdeBm7BfK^TS+c&xFbY!G=G5zn~C4_}M2?0UX*4B^YBUR3(Lt;^1M27!R-<)*> zYU*A58M`Wx4MbGe%!;PH^GlOG-Bv)-UENYhN!+|&BMIs{5;Iyl@qCLgq1lW%qEyW! zlJg-}PAN-d@1PS?RmdlKA!=yky$InZk%e3I=k4aB8w#eC*}+y49lK2f2!npOWi~~Q zy(kGo8Q2r7duBI7M_Z(PF_VgqL28fL6?x07@2y{?72@C`Mt%A4(im{fE>(KdhNR*L zFRxq%uBkkJPVcB3+d>Y0UBs8xY^gJ90vBS67PG1lx z*-r_)DFV#Nbx_!O;@h{F>e|;~_Ov`5G}1OGSuSMyZ*d9{E zmej_A4Q9)dn;+cxjy!%<$n#m!jQCD*h}dJfj*Tuj&?!lY$%B|=^D%p@wK>y%0<0!a zUu4xw0XRE@6*9>gooLS_p63Ip7@MX^v>)?2jyq-gDcmF~T;`tXh4pDWyt2ku5iSYi z%S#zh0|Xs7JaDROa=HMfQ`-k{Rj&U9?aX=aqB-CBhz_#O`dQxFbH%SHNQuzw$R_Eu!{G#$ za;`)a*e9=~6mIt$Jry%@n)(A(Q0JY~Bi{U_B}BU`7T=P%rfuO-@s#I$`2&;opjZf| zNFwgUujr8#<{#rvK>*j z;>&2|7uLny>ToiyjATq6K0$Brjz~7D@wCI@GCaEo(?j36|2V4X9l>+RYpQH~l4?KJ z_a}NjM|0L4cn!oX1N7Q9aB;HY#U5vI<7>!@rLe9%yXW|J4w*!YOAYqc^>f>B(u9wy zGfvJ!`Ggsrvl0q8v_8Sit6aAKDWpf1Nq`}gaMn$=2?(3?CW#wVY}Wje_UA&#K9{T- z!}?f=U3f2$H3;j*AI~>=q<7N!XMaKPepA(x!OLWN9#e0t)WDNj%1%^#i&HF$uOv-LMAds zU7wSAUoFu>Fv&zuTOrIox~qy!QhtI0=7ZUa0}LkC)UBn%y=v*Ua@SAax>Bc%pC1oO z%I``)te=%VttbZts%~Hz=NTF0<=;g7JqypzhY@?_J@<2Nekmn2e)jFGJ|S(|I3rV( zetC|I3%tMDDcwsL8xvkT4^%$c+sii!3{ZO^@b~fc^)(g5SfoAi`u#Twf57x-*OB}! z=@;7bz19umgj<=D8w~l+CU47rBX^O!(R<3i{X=%UeR>T0E#NoTD?;+-9co!2U-cNH~N93Cbpd1w`p_7s6wcvJgVPYNnJJ7Q71A`UyF~7SI&Gb(^p5$RWuVA zE6XxDu<0GO7R6D6Lm^@$7iFBz(3Q9wu>9!dsA&!Q)$F7yB)F~!{SCEQIAvjD=&idu z9CgbDq57>x8hQx~puFZPGEvUb5ia}X)JjgChND)FoDoUo^;s>J&tBGzx8=6%3pzU35KqdV*yi=9RGJHmn zTAaQ0?t2YT#A!ulL{1_JUYy~rM?Yajjua&pdnj)u&r2Hl3BlrmevKUT>kqv=+-*TH z+A2I*L%!$xYt&E@#uSi8)@ihPykcsYL1=jRWs-*7=qp-TVg35MA0Id`toJD|!kQm6 zWre|ge8lO)>#453#WMO_i2>9%u1os?%N^RH+{v!TtyFG zIDR?C$`G3n1a?x!1F`B3=VV@{L)7`aX71e#_@XS+GGRu=4@tgtMX`9uzo}yi1eNVXu5EyK>!3hvGJNacy*uelstN$1w!`HCt zO}zPl>zih}uoiDwRR%Qnf15s(70d-`e^iL#`3W(ZgP9>Q4N!Lbve{lo79p#ib);m# zyk-9Hs+?T&zMi;yyrvtULEf|-H5WUsEg(?rf9+AG@>Ud`k6XRtw>uB*1f3BF)%RE* z%2Yz9r956Z{sDzHxCSpczB25KSW9K7J{;WAj**YC^2*1mx8#{PWUha~-_Jw2G~QGE z9iST|y5i!%Ja(noMF__H=S<$@E&uy{O-iynn-O*Q3|+rLZf!w&j-wUR;Jl(E@gew| zVYe7QjZw+WSY07oHrk31@y=fTU_awe1f!$B*+ZTWt`2Z*=9BT>g()i8eE>6R)Y=WD%^;CH*(nC!( zdX>X9I7q?%40ttDI7{K#o>wQUKOuN7Eme>)o1KuZ5~s^_sZ#fe!UBTu5PL97q^||# zE6fPRj?e&l_Y~_oLZrfmOWsNO{q`@I$k6c{lu0&V>PmzV?2ba4Jv-l|y<^p7!FwIt zUT3Wgjb`YXf(G~FDj0o2?jb7RSM63jF~Mzct4~i|6trp!A_F&=p0D70glaf^ly{pb zR47?Fe!RF3=dohRi2O=5J`i|{yWdcmsRPb#;@9UQU$Yxpucx`2MvH4w9u`H3h(fqD z*Yn$$!wZu_JCAW6{h`G2G$3zCeVuE8RRR{rz{6>;T3{bf7W6h|W6b>JN`iEd20^1Y z+of8NLDR%9`fK<>#b6>*JsdjY@Ad6PkOy!XVadTbD26T!oJYLUeu%Y4U<;?B#$wNz4 zebdQ}^{(cfzNz)TBOK0t(?B?^Uu8a7M|+oB<9)wAsVlE-tmMX0A@xp?voRGc_R^mrsbGcYLfda17g-HwQXvA z(k+|JI(_!~px&;&3`&w!Y9;=Ws&$yH{a23^x-Q<3j|CR#omLZLt@rrK8u@UYaXKg` z5p~mtHuTuWJb8vaD#p3Y3L8@y|7V;=at@9#)H21EyD{B9IO?Vi^DGVG|0L$h%G|g; zn=MsIGux#^Hi(q4jhz30J!>$Cm7MUN-oBS$--T zD4+Zgyqgf9%>SU3Jn{qIhcPAUfC@|u?2Od2%uqkgKS44*mPJ3(w6-A0?A-CulT{}dr$vrz5sm&t{xf#y5ir`tBRTI(Jwx2_ux$yN7UHwE2UUI8J zF?i*DXMj8_cf1v*=QpAJ#`YAJZj7>&H;*{oPsQqz=S5i+&spX4yQidD9FAseNW-)U zj$!9MV!~TG1(wLV`l*G=D;TtrQ^oV`LSF1D@D$&}eO6Kg!9y8ZDMBzSVq)LG^BTvI zH4BrK#^;BoV{e{nQQff#h+?LHOZ3NDI7-3gc_cv`@?n?1lnw-)0ZCo*=IR<`Cl!HwLIF{-(U0#8TD``(21KoOh(l}F9KO7`Nm_$R2hFldHYtr(_ zxrkTJXkSoJfB5WMV~Ptnm_>B{{KYX+D+#O-rNeb--o~Z0adQl2K#uInt-`*Nd>W)u z_MeubsB18mpwO=xGv^#Cb(1|+0NX7r96AqQ`Qrk zznwx#nyL7&;D(I_Y&-u(p4g)^XcFKh8+UtW+J1?Q*Tg zkl!!AL18q!V1M>^SHE14vhh9cmwwrcGjQ8I!gHVHV83uPx^D&K`VYpxbVt8rOgsPK ze2RN$Sw3C7@bE9ed13#)(PMr8;f89XZu{s~f%e&4t}9~VlJ``$bKCj))TYVxeH8q@ z3#oKyov7~*CW4n2-s3$-s@6((iO>Ue>WRXXa!R+l_a%uuN%syV16|ga0-S5bN^akU zv&%2e2%g3jZpuIg@2_M0!|A8eeT9*0E%GS(U zb80B>lFjo!~sC(*mZnP^{ggLlrbe{G-&sV;wcefQ7_WMccRK`X}z!^ z^_N}A04ZmdF>Pz_?w|c!%>CRzQTZNEQ`ewj-4@smuJ1Kj(8j^pkpj6d7odW&yBML*aCX-7qGOX_nXP{?h^b$4|tS z8$NssC8zMR@T9(`v7ml=C2SzDiV2QJs3sN<1f#D7DW3PGgfM2rI>&LE{}_v*Ap{y5(06$4-M#G= zYb0yhz@g?a`Fzq~^1Vv0E;oUcN1>r3%$m_-P9aXjq%q=qx@;#Uj-_PLm46vNx&sMM z6$DG0!EUOR=ymrgU1Pt8J7o@MK)(?sC${FBDM6$mWQPn}KbZ-w!-`D8p!R*9f}<7| z;2O@Qv0LrJAP3bFj_UQHR0|2byo~XqN6n=ia;vl+k%n$g6gpfxT7$#c`I4L=xB7Mh zeg|b*=1Fzo_?Ekh^tY|n%|JT|C6W(UTe^`y5gr!I3Jet>5oP)8#(&L%hN*q~bep|y z|9cJwW2|zRJV=%Vna(A9t_mpr*0Sx`?Fo+Fn60U|!Jh7e0vRY5jAjq)CeB}WnM{1^ zQ7%28#>xLZ`(;y`JZdj~BYSUVl)h}_qwFO`S4n;x#xu)k*r!~ZmVgK0WHkRIH5}2I z7*bRa_(H-48LoJx?$)o^)PeAyVYB19eO$5b*D2G(q<_nUQQ{nN(=9$8h7;yDRRv_g zfhFbjbM&SCCR>K?H)*0IwYuST>5=#xe|&a4>;BKx6ONUSIvSO^jLtUN*}kaW8vICT z?VRt~q`ch1MGyQDzr4k~lZ;iqY_AaGe(f*U;-Js5Lq8MwH0kx`Deb?9;s1PZ1TB># zU<1JWS0+#TBEQiJ^x?21mCxRclz)Jvf08oQx1#1OYtc&{bdz%m1%GSpB>lve^JB)yuytxP=sNCoKSJ zQ797bWF3;>fLIL^JCdenw52YcSr~0DXCJfC-~UT&(4ki%rSN1E&BlRmgGFxwTk(Mq zr2)KP_Z6c6Tp-D_V;K)fiZnudEm|VOO7dE?EZ6ll+3IABJzS#Q@sbYEm*vZY2h`R4 zgB1T0%|2-Y;e0tk@egBWL64mQ^T$h1OaL~KQ~Frj$lazbJXs&9pI22S9j=tAcP)N! zpA~~_4nO%)9E8_7VRKnGfCQ>dkrLqp^p0)Xp&2%=#g%!uUSU z?KNOUmf$`!vElp%Kr%SEj5e10i~zhkuAUaUS_fbRQT#%iCP1)gs;`0_5KNYet%?qO z){>Lt2S#yoLzqahq-6fwh0C+RH@+FyrMWDo>aawxO7V zDa^pR+V{>Pt`$;7pZ*Xw#NdVUD?AU~gXEYYH^4gy^k!?Z<~2}TH>E<9F9x@X=eE`( z`4$-M6+>`)ti%KaDQ4Zit6nVO05qj-IxtPK7;d(EJm4qeChVlA_kf8SY!@COU?LZ> zK9z#d9gBy3g#wJ~6&B9D0e(_K$ot+Mk(Mb)UBv|$He!*~+QThcAzv_GC*eDYG7{q$~XdR=NovO^64@CL08 z@xCzqGW00#Y#L~edAJ&MwcM>Hf{l?n!|Vv+-}@p$T?0w~ih&{_E6)S16iy>_fRe9b z0po9S<Z)gYV5tAa&qD^t zGI$5JlXjb>Jz3@)yrT0gKH5baR!!jbugKatUCRMLp&%!TI&pBqrJ?p(nC^g=+8QPo z9dF>Zq@nKU`{l5#tADx96wPw;G8WzU>qVa3;^_@Ggm)T53ezQ8vT^ z0K88}V^~HCCAFjLvY-noV29-=5T(SUE$mK}{u3$xkX0H? zsKM3`ArU|Ub4vuqI7E;b39}8I&dajrpyf`5^3AB{ zATx0D@-rtUSl-)@V}BwKiw#e!YN00+t+Q!8c?2v#VWJ9=oA)5d&IM9iY&uukx zrUj~|pZmsrup@InLzMu4@yh$9uVx$>#|b>4C%&AqFoW2M{IDl1umtu3NKW z0*?rOrY_Mqz_UQLw7GhX{>nQHp|?QU%u^YCE8TJqakc!ZlDj_ScL$wUEqP24_r+ub z5KPzv{jp9*;V?lminmqyLh2nut}okMy5u?^aW!Gg(5$(7XmM6Hs_*=c>^9&Fk;#a4NxCqEeh`Tjnmk{Jv2sg7)MI|l@ z6K!%PTE6D$pZ!zIHc+kxSuYJ{)8~#%g*XxM@tE7Dj)8<{CVub-gM(FUftfgif1aPY zYY}A=u+`sQH?~o8P)~o1cTxzHLFlod_ClZl6WVm=escQfnE*Kpkx{=a{wvRQh=}k2tcX6>#uZfjkJ3Xzg!E4Y+;#DU{twCj p{$=_ujkrnZ{{lxEA6EbX literal 0 HcmV?d00001 -- 2.40.1 From 8f334fdb10ad1ba31c969a7985e69f076dba0b20 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 29 Jul 2020 15:12:06 +0100 Subject: [PATCH 03/96] Apparently no patch levels in requirements --- charts/kubezero/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index 7d00c53..285c51d 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -15,4 +15,4 @@ dependencies: - name: kubezero-lib version: ">= 0.1.1" repository: https://zero-down-time.github.io/kubezero/ -kubeVersion: ">= 1.16.12" +kubeVersion: ">= 1.16.0" -- 2.40.1 From 19b1fd61c93365d0f56543d63b196830d314c0d1 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 29 Jul 2020 18:02:18 +0100 Subject: [PATCH 04/96] Convert argo-cd ACL to DENY policy --- charts/kubezero-argo-cd/Chart.yaml | 2 +- .../templates/istio-authorization-policy.yaml | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/charts/kubezero-argo-cd/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml index cb87309..4d5129a 100644 --- a/charts/kubezero-argo-cd/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application name: kubezero-argo-cd -version: 0.3.5 +version: 0.3.6 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-argo-cd/templates/istio-authorization-policy.yaml b/charts/kubezero-argo-cd/templates/istio-authorization-policy.yaml index e0a21d4..e9522f1 100644 --- a/charts/kubezero-argo-cd/templates/istio-authorization-policy.yaml +++ b/charts/kubezero-argo-cd/templates/istio-authorization-policy.yaml @@ -1,25 +1,24 @@ {{- if index .Values "argo-cd" "istio" "enabled" }} +{{- if index .Values "argo-cd" "istio" "ipBlocks" }} apiVersion: security.istio.io/v1beta1 kind: AuthorizationPolicy metadata: - name: argocd-allow-only + name: argocd-deny-not-in-ipblocks namespace: istio-system spec: selector: matchLabels: app: istio-ingressgateway + action: DENY rules: - {{- if index .Values "argo-cd" "istio" "ipBlocks" }} - from: - source: - ipBlocks: + notIpBlocks: {{- with index .Values "argo-cd" "istio" "ipBlocks" }} {{- . | toYaml | nindent 8 }} {{- end }} to: - operation: hosts: ["{{ index .Values "argo-cd" "server" "config" "url" }}"] - {{- else }} - - {} - {{- end }} +{{- end }} {{- end }} -- 2.40.1 From 5aed60f21a410bd60b1ccc205465e08aeade6474 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 30 Jul 2020 17:19:04 +0100 Subject: [PATCH 05/96] minor bootstrap fix --- charts/kubezero-metrics/.helmignore | 23 ++++++++++++ charts/kubezero-metrics/Chart.yaml | 21 +++++++++++ charts/kubezero-metrics/README.md | 47 ++++++++++++++++++++++++ charts/kubezero-metrics/README.md.gotmpl | 10 +++++ charts/kubezero-metrics/values.yaml | 43 ++++++++++++++++++++++ deploy/templates/values.yaml | 2 +- 6 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 charts/kubezero-metrics/.helmignore create mode 100644 charts/kubezero-metrics/Chart.yaml create mode 100644 charts/kubezero-metrics/README.md create mode 100644 charts/kubezero-metrics/README.md.gotmpl create mode 100644 charts/kubezero-metrics/values.yaml diff --git a/charts/kubezero-metrics/.helmignore b/charts/kubezero-metrics/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/kubezero-metrics/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml new file mode 100644 index 0000000..2dff6e1 --- /dev/null +++ b/charts/kubezero-metrics/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: kubezero-metrics +description: KubeZero Umbrella Chart for prometheus-operator +type: application +version: 0.0.1 +home: https://kubezero.com +icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png +keywords: + - kubezero + - prometheus-operator + - grafana +maintainers: + - name: Quarky9 +dependencies: + - name: kubezero-lib + version: ">= 0.1.1" + repository: https://zero-down-time.github.io/kubezero/ + - name: prometheus-operator + version: 9.3.0 + repository: https://kubernetes-charts.storage.googleapis.com/ +kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-metrics/README.md b/charts/kubezero-metrics/README.md new file mode 100644 index 0000000..e341dfd --- /dev/null +++ b/charts/kubezero-metrics/README.md @@ -0,0 +1,47 @@ +kubezero-cert-manager +===================== +KubeZero Umbrella Chart for cert-manager + +Current chart version is `0.3.5` + +Source code can be found [here](https://kubezero.com) + +## Chart Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.jetstack.io | cert-manager | 0.15.1 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | + +## AWS - IAM Role +If you use kiam or kube2iam and restrict access on nodes running cert-manager please adjust: +``` +cert-manager.podAnnotations: + iam.amazonaws.com/role: +``` + +## Resolver Secrets +If your resolvers need additional sercrets like CloudFlare API tokens etc. make sure to provide these secrets separatly matching your defined issuers. + +## Chart Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| cert-manager.cainjector.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| cert-manager.cainjector.tolerations[0].effect | string | `"NoSchedule"` | | +| cert-manager.cainjector.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| cert-manager.extraArgs[0] | string | `"--dns01-recursive-nameservers-only"` | | +| cert-manager.ingressShim.defaultIssuerKind | string | `"ClusterIssuer"` | | +| cert-manager.ingressShim.defaultIssuerName | string | `"letsencrypt-dns-prod"` | | +| cert-manager.installCRDs | bool | `true` | | +| cert-manager.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| cert-manager.podAnnotations | object | `{}` | "iam.amazonaws.com/roleIAM:" role ARN the cert-manager might use via kiam eg."arn:aws:iam::123456789012:role/certManagerRoleArn" | +| cert-manager.prometheus.servicemonitor.enabled | bool | `false` | | +| cert-manager.tolerations[0].effect | string | `"NoSchedule"` | | +| cert-manager.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| cert-manager.webhook.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| cert-manager.webhook.tolerations[0].effect | string | `"NoSchedule"` | | +| cert-manager.webhook.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| clusterIssuer | object | `{}` | | +| localCA.enabled | bool | `true` | | +| localCA.selfsigning | bool | `true` | | diff --git a/charts/kubezero-metrics/README.md.gotmpl b/charts/kubezero-metrics/README.md.gotmpl new file mode 100644 index 0000000..1e4d03b --- /dev/null +++ b/charts/kubezero-metrics/README.md.gotmpl @@ -0,0 +1,10 @@ +{{ template "chart.header" . }} +{{ template "chart.description" . }} + +{{ template "chart.versionLine" . }} + +{{ template "chart.sourceLinkLine" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml new file mode 100644 index 0000000..bc87c3f --- /dev/null +++ b/charts/kubezero-metrics/values.yaml @@ -0,0 +1,43 @@ +prometheus-operator: + alertmanager: + enabled: false + coreDns: + enabled: false + kubeApiServer: + enabled: false + kubeControllerManager: + enabled: false + kubeDns: + enabled: false + kubeEtcd: + enabled: false + kubeProxy: + enabled: false + kubeScheduler: + enabled: false + kubeStateMetrics: + enabled: false + kubelet: + enabled: false + nodeExporter: + enabled: false + grafana: + enabled: false + prometheus: + enabled: false + defaultRules: + create: false + # Default configuration of prometheus operator will create CRDs in the cluster idempotently + prometheusOperator: + enabled: true + serviceMonitor: + selfMonitor: false + createCustomResource: false + tlsProxy: + enabled: false + admissionWebhooks: + enabled: false + namespaces: + releaseNamespace: true + additional: + - kube-system diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 8d00320..2a71673 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -96,7 +96,7 @@ argo-cd: configs: {{- toYaml . | nindent 4 }} {{- end }} - {{- if and ( not .Values.bootstrap ) ( index .Values "argo-cd" "istio" ) }} + {{- if and ( not .Values.bootstrap ) ( index .Values "argo-cd" "istio" "enabled" ) .Values.istio.enabled }} istio: {{- with index .Values "argo-cd" "istio" }} {{- toYaml . | nindent 4 }} -- 2.40.1 From a84fcf0a6394974f4426947b4dcb2ec2c97c8368 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 30 Jul 2020 17:19:48 +0100 Subject: [PATCH 06/96] Initial metrics chart --- charts/kubezero-argo-cd/README.md | 2 +- charts/kubezero-metrics/README.md | 59 ++++++++++++++----------------- 2 files changed, 27 insertions(+), 34 deletions(-) diff --git a/charts/kubezero-argo-cd/README.md b/charts/kubezero-argo-cd/README.md index de55733..8a94d64 100644 --- a/charts/kubezero-argo-cd/README.md +++ b/charts/kubezero-argo-cd/README.md @@ -2,7 +2,7 @@ kubezero-argo-cd ================ KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application -Current chart version is `0.3.5` +Current chart version is `0.3.6` Source code can be found [here](https://kubezero.com) diff --git a/charts/kubezero-metrics/README.md b/charts/kubezero-metrics/README.md index e341dfd..55028c9 100644 --- a/charts/kubezero-metrics/README.md +++ b/charts/kubezero-metrics/README.md @@ -1,8 +1,8 @@ -kubezero-cert-manager -===================== -KubeZero Umbrella Chart for cert-manager +kubezero-metrics +================ +KubeZero Umbrella Chart for prometheus-operator -Current chart version is `0.3.5` +Current chart version is `0.0.1` Source code can be found [here](https://kubezero.com) @@ -10,38 +10,31 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| -| https://charts.jetstack.io | cert-manager | 0.15.1 | +| https://kubernetes-charts.storage.googleapis.com/ | prometheus-operator | 9.3.0 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | -## AWS - IAM Role -If you use kiam or kube2iam and restrict access on nodes running cert-manager please adjust: -``` -cert-manager.podAnnotations: - iam.amazonaws.com/role: -``` - -## Resolver Secrets -If your resolvers need additional sercrets like CloudFlare API tokens etc. make sure to provide these secrets separatly matching your defined issuers. - ## Chart Values | Key | Type | Default | Description | |-----|------|---------|-------------| -| cert-manager.cainjector.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | -| cert-manager.cainjector.tolerations[0].effect | string | `"NoSchedule"` | | -| cert-manager.cainjector.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | -| cert-manager.extraArgs[0] | string | `"--dns01-recursive-nameservers-only"` | | -| cert-manager.ingressShim.defaultIssuerKind | string | `"ClusterIssuer"` | | -| cert-manager.ingressShim.defaultIssuerName | string | `"letsencrypt-dns-prod"` | | -| cert-manager.installCRDs | bool | `true` | | -| cert-manager.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | -| cert-manager.podAnnotations | object | `{}` | "iam.amazonaws.com/roleIAM:" role ARN the cert-manager might use via kiam eg."arn:aws:iam::123456789012:role/certManagerRoleArn" | -| cert-manager.prometheus.servicemonitor.enabled | bool | `false` | | -| cert-manager.tolerations[0].effect | string | `"NoSchedule"` | | -| cert-manager.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | -| cert-manager.webhook.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | -| cert-manager.webhook.tolerations[0].effect | string | `"NoSchedule"` | | -| cert-manager.webhook.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | -| clusterIssuer | object | `{}` | | -| localCA.enabled | bool | `true` | | -| localCA.selfsigning | bool | `true` | | +| prometheus-operator.alertmanager.enabled | bool | `false` | | +| prometheus-operator.coreDns.enabled | bool | `false` | | +| prometheus-operator.defaultRules.create | bool | `false` | | +| prometheus-operator.grafana.enabled | bool | `false` | | +| prometheus-operator.kubeApiServer.enabled | bool | `false` | | +| prometheus-operator.kubeControllerManager.enabled | bool | `false` | | +| prometheus-operator.kubeDns.enabled | bool | `false` | | +| prometheus-operator.kubeEtcd.enabled | bool | `false` | | +| prometheus-operator.kubeProxy.enabled | bool | `false` | | +| prometheus-operator.kubeScheduler.enabled | bool | `false` | | +| prometheus-operator.kubeStateMetrics.enabled | bool | `false` | | +| prometheus-operator.kubelet.enabled | bool | `false` | | +| prometheus-operator.nodeExporter.enabled | bool | `false` | | +| prometheus-operator.prometheus.enabled | bool | `false` | | +| prometheus-operator.prometheusOperator.admissionWebhooks.enabled | bool | `false` | | +| prometheus-operator.prometheusOperator.createCustomResource | bool | `false` | | +| prometheus-operator.prometheusOperator.enabled | bool | `true` | | +| prometheus-operator.prometheusOperator.namespaces.additional[0] | string | `"kube-system"` | | +| prometheus-operator.prometheusOperator.namespaces.releaseNamespace | bool | `true` | | +| prometheus-operator.prometheusOperator.serviceMonitor.selfMonitor | bool | `false` | | +| prometheus-operator.prometheusOperator.tlsProxy.enabled | bool | `false` | | -- 2.40.1 From db26111872d32e2234a1dc84a8d19fd6552c86e9 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 30 Jul 2020 18:18:32 +0100 Subject: [PATCH 07/96] Add draft metrics chart --- charts/kubezero/Chart.yaml | 2 +- charts/kubezero/README.md | 1 + charts/kubezero/templates/metrics.yaml | 6 ++++++ charts/kubezero/values.yaml | 3 +++ 4 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 charts/kubezero/templates/metrics.yaml diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index 285c51d..5c549c6 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero ArgoCD Application - Root App of Apps chart of KubeZero type: application -version: 0.3.11 +version: 0.4.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero/README.md b/charts/kubezero/README.md index 7fe23c6..eaca185 100644 --- a/charts/kubezero/README.md +++ b/charts/kubezero/README.md @@ -27,4 +27,5 @@ Source code can be found [here](https://kubezero.com) | istio.enabled | bool | `false` | | | kiam.enabled | bool | `false` | | | local-volume-provisioner.enabled | bool | `false` | | +| metrics.enabled | bool | `false` | | | platform | string | `"aws"` | | diff --git a/charts/kubezero/templates/metrics.yaml b/charts/kubezero/templates/metrics.yaml new file mode 100644 index 0000000..139a609 --- /dev/null +++ b/charts/kubezero/templates/metrics.yaml @@ -0,0 +1,6 @@ +{{- if index .Values "metrics" "enabled" }} +{{ template "kubezero-app.app" dict "root" . "name" "metrics" "type" "helm" }} + syncPolicy: + automated: + prune: true +{{- end }} diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index dba7782..2c486f1 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -35,3 +35,6 @@ aws-efs-csi-driver: istio: enabled: false + +metrics: + enabled: false -- 2.40.1 From 585c3770ef3d77b7cdef1ad389fd76bcde8cf025 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 30 Jul 2020 18:26:11 +0100 Subject: [PATCH 08/96] Move metrics to monitoring NS --- charts/kubezero/templates/metrics.yaml | 7 ++++++- deploy/templates/values.yaml | 2 ++ deploy/values.yaml | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/charts/kubezero/templates/metrics.yaml b/charts/kubezero/templates/metrics.yaml index 139a609..59c7ee3 100644 --- a/charts/kubezero/templates/metrics.yaml +++ b/charts/kubezero/templates/metrics.yaml @@ -1,6 +1,11 @@ {{- if index .Values "metrics" "enabled" }} -{{ template "kubezero-app.app" dict "root" . "name" "metrics" "type" "helm" }} +{{ template "kubezero-app.app" dict "root" . "name" "metrics" "type" "helm" "namespace" "monitoring"}} syncPolicy: automated: prune: true +--- +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring {{- end }} diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 2a71673..b462196 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -85,6 +85,8 @@ kubezero: {{- toYaml .Values.istio.ingress | nindent 8 }} {{- end }} + metrics: + enabled: {{ .Values.metrics.enabled }} {{- end }} argo-cd: diff --git a/deploy/values.yaml b/deploy/values.yaml index 0c00f1f..4572d31 100644 --- a/deploy/values.yaml +++ b/deploy/values.yaml @@ -25,6 +25,10 @@ kiam: istio: enabled: false +metrics: + enabled: false + +# Deprecated once metrics goes live prometheus: enabled: false -- 2.40.1 From a5142b523d3cfd1614e21cdd50802591f023890e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 30 Jul 2020 18:27:43 +0100 Subject: [PATCH 09/96] Add montoring and logging NS to kubezero argo project --- charts/kubezero-argo-cd/Chart.yaml | 2 +- charts/kubezero-argo-cd/templates/project.yaml | 4 ++++ charts/kubezero/Chart.yaml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-argo-cd/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml index 4d5129a..9c86450 100644 --- a/charts/kubezero-argo-cd/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application name: kubezero-argo-cd -version: 0.3.6 +version: 0.3.7 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-argo-cd/templates/project.yaml b/charts/kubezero-argo-cd/templates/project.yaml index 28e0e52..16faaba 100644 --- a/charts/kubezero-argo-cd/templates/project.yaml +++ b/charts/kubezero-argo-cd/templates/project.yaml @@ -24,6 +24,10 @@ spec: server: https://kubernetes.default.svc - namespace: istio-system server: https://kubernetes.default.svc + - namespace: monitoring + server: https://kubernetes.default.svc + - namespace: logging + server: https://kubernetes.default.svc clusterResourceWhitelist: - group: '*' diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index 5c549c6..1bd3f94 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero ArgoCD Application - Root App of Apps chart of KubeZero type: application -version: 0.4.0 +version: 0.4.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: -- 2.40.1 From 6faf757e304afbf4316ec174e26382bd613db58e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 30 Jul 2020 18:56:46 +0100 Subject: [PATCH 10/96] Let Argo take care of CRDs --- charts/kubezero-argo-cd/README.md | 2 +- charts/kubezero-metrics/values.yaml | 6 +++++- charts/kubezero/README.md | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-argo-cd/README.md b/charts/kubezero-argo-cd/README.md index 8a94d64..38acc14 100644 --- a/charts/kubezero-argo-cd/README.md +++ b/charts/kubezero-argo-cd/README.md @@ -2,7 +2,7 @@ kubezero-argo-cd ================ KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application -Current chart version is `0.3.6` +Current chart version is `0.3.7` Source code can be found [here](https://kubezero.com) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index bc87c3f..6347d55 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -30,9 +30,13 @@ prometheus-operator: # Default configuration of prometheus operator will create CRDs in the cluster idempotently prometheusOperator: enabled: true + + # We use Argo + manageCrds: false + createCustomResource: true + serviceMonitor: selfMonitor: false - createCustomResource: false tlsProxy: enabled: false admissionWebhooks: diff --git a/charts/kubezero/README.md b/charts/kubezero/README.md index eaca185..87b4b43 100644 --- a/charts/kubezero/README.md +++ b/charts/kubezero/README.md @@ -2,7 +2,7 @@ kubezero ======== KubeZero ArgoCD Application - Root App of Apps chart of KubeZero -Current chart version is `0.3.11` +Current chart version is `0.4.1` Source code can be found [here](https://kubezero.com) -- 2.40.1 From 794f249eeb892329c2390543945b25eb82e3af6f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 31 Jul 2020 01:18:07 +0100 Subject: [PATCH 11/96] Add webhooks and set tolerations --- charts/kubezero-metrics/values.yaml | 32 +++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 6347d55..a375b2f 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -31,17 +31,33 @@ prometheus-operator: prometheusOperator: enabled: true + # Run on controller nodes + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + nodeSelector: + node-role.kubernetes.io/master: "" + # We use Argo manageCrds: false createCustomResource: true - serviceMonitor: - selfMonitor: false + # Operator has TLS support starting 0.39, but chart does not support CAConfig and operator flags yet + # see: https://github.com/coreos/prometheus-operator/blob/master/Documentation/user-guides/webhook.md#deploying-the-admission-webhook + # Until then we use the patch job and tls proxy, to be reviewed later on to use the kubezero local cluster CA from cert-manager tlsProxy: - enabled: false + enabled: true admissionWebhooks: - enabled: false - namespaces: - releaseNamespace: true - additional: - - kube-system + enabled: true + patch: + enabled: true + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + nodeSelector: + node-role.kubernetes.io/master: "" + + #namespaces: + # releaseNamespace: true + # additional: + # - kube-system -- 2.40.1 From 0a0f54a316572011b46f9614790b0fdd455d0d4d Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 31 Jul 2020 01:32:44 +0100 Subject: [PATCH 12/96] Disable webhooks for now, latest operator --- charts/kubezero-metrics/values.yaml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index a375b2f..5527892 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -27,9 +27,11 @@ prometheus-operator: enabled: false defaultRules: create: false - # Default configuration of prometheus operator will create CRDs in the cluster idempotently + prometheusOperator: enabled: true + image: + tag: v0.41.0 # Run on controller nodes tolerations: @@ -38,24 +40,17 @@ prometheus-operator: nodeSelector: node-role.kubernetes.io/master: "" - # We use Argo + # Argo takes care of CRDs manageCrds: false createCustomResource: true # Operator has TLS support starting 0.39, but chart does not support CAConfig and operator flags yet # see: https://github.com/coreos/prometheus-operator/blob/master/Documentation/user-guides/webhook.md#deploying-the-admission-webhook - # Until then we use the patch job and tls proxy, to be reviewed later on to use the kubezero local cluster CA from cert-manager + # Until then we disable them as the patching interferes with Argo anyways tlsProxy: - enabled: true + enabled: false admissionWebhooks: - enabled: true - patch: - enabled: true - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - nodeSelector: - node-role.kubernetes.io/master: "" + enabled: false #namespaces: # releaseNamespace: true -- 2.40.1 From 9afa4828be038d8b33e5f3372039fa688a3bea84 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 13:06:07 +0100 Subject: [PATCH 13/96] Latest deploy, add prometheus adapter to metrics --- charts/kubezero-metrics/Chart.yaml | 7 ++++-- charts/kubezero-metrics/values.yaml | 37 +++++++++++++++++++++++++++++ deploy/deploy.sh | 23 +++++++++--------- deploy/templates/values.yaml | 32 +++++++++++++++---------- 4 files changed, 73 insertions(+), 26 deletions(-) diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 2dff6e1..d4a3d16 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,12 +2,12 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for prometheus-operator type: application -version: 0.0.1 +version: 0.0.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: - kubezero - - prometheus-operator + - prometheus - grafana maintainers: - name: Quarky9 @@ -18,4 +18,7 @@ dependencies: - name: prometheus-operator version: 9.3.0 repository: https://kubernetes-charts.storage.googleapis.com/ + - name: prometheus-adapter + version: 2.5.0 + repository: https://kubernetes-charts.storage.googleapis.com/ kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 5527892..ebdff4a 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -32,6 +32,8 @@ prometheus-operator: enabled: true image: tag: v0.41.0 + prometheusConfigReloaderImage: + tag: v0.41.0 # Run on controller nodes tolerations: @@ -56,3 +58,38 @@ prometheus-operator: # releaseNamespace: true # additional: # - kube-system + +# Metrics adapter +prometheus-adapter: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + nodeSelector: + node-role.kubernetes.io/master: "" + + resource: + cpu: + containerQuery: sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}[5m])) by (<<.GroupBy>>) + nodeQuery: sum(1 - irate(node_cpu_seconds_total{mode="idle"}[5m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}) by (<<.GroupBy>>) + nodeQuery: sum(node_memory_MemTotal_bytes{job="node-exporter",<<.LabelMatchers>>} - node_memory_MemAvailable_bytes{job="node-exporter",<<.LabelMatchers>>}) by (<<.GroupBy>>) + resources: + overrides: + instance: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 5m diff --git a/deploy/deploy.sh b/deploy/deploy.sh index aee17a8..d2b458a 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -40,30 +40,31 @@ else EOF fi - helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set bootstrap=true > generated-values.yaml - - # Deploy initial argo-cd + # Deploy initial argo-cad + helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set cert-manager.not_ready=true --set istio.enabled=false --set prometheus.enabled=false > generated-values.yaml helm install -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml - # Wait for argocd-server to be running kubectl rollout status deployment -n argocd kubezero-argocd-server - # Now wait for cert-manager to be bootstrapped + # Now wait for cert-manager and the local CA to be bootstrapped echo "Waiting for cert-manager to be deployed..." wait_for kubectl get deployment -n cert-manager cert-manager-webhook 2>/dev/null 1>&2 kubectl rollout status deployment -n cert-manager cert-manager-webhook + wait_for kubectl get Issuer -n kube-system kubezero-local-ca-issuer 2>/dev/null 1>&2 + kubectl wait --for=condition=Ready -n kube-system Issuer/kubezero-local-ca-issuer - # Now lets get kiam and cert-manager to work as they depend on each other, keep advanced options still disabled though - # - istio, prometheus - helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set istio.enabled=false --set prometheus.enabled=false > generated-values.yaml - helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd -f generated-values.yaml + # Now lets make sure kiam is working + helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set istio.enabled=false --set prometheus.enabled=false > generated-values.yaml + helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml + wait_for kubectl get daemonset -n kube-system kiam-agent 2>/dev/null 1>&2 + kubectl rollout status daemonset -n kube-system kiam-agent # Install Istio if enabled, but keep ArgoCD istio support disabled for now in case helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set argo-cd.istio.enabled=false > generated-values.yaml helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd -f generated-values.yaml + wait_for kubectl get deployment -n istio-operator istio-operator 2>/dev/null 1>&2 + kubectl rollout status deployment -n istio-operator istio-operator - echo "Install kube-prometheus and logging manually for now, before proceeding! " - read # Todo: Now we need to wait till all is synced and healthy ... argocd cli or kubectl ? # Wait for aws-ebs or kiam to be all ready, or all pods running ? diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index b462196..673a1d6 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -12,13 +12,14 @@ kubezero: prometheus: {{ .Values.prometheus.enabled }} cert-manager: enabled: {{ index .Values "cert-manager" "enabled" }} -{{- if not .Values.bootstrap }} values: {{- if eq .Values.platform "aws" }} cert-manager: podAnnotations: iam.amazonaws.com/role: "{{ index .Values "cert-manager" "IamArn" }}" {{- end }} + + {{- if not .Values.kiam.not_ready }} clusterIssuer: name: letsencrypt-dns-prod server: https://acme-v02.api.letsencrypt.org/directory @@ -38,19 +39,11 @@ kubezero: {{- . | toYaml | nindent 14 }} {{- end }} {{- end }} + {{- end }} + - # AWS only components {{- if eq .Values.platform "aws" }} - aws-ebs-csi-driver: - enabled: {{ index .Values "aws-ebs-csi-driver" "enabled" }} - values: - aws-ebs-csi-driver: - replicaCount: {{ ternary 2 1 .Values.HighAvailableControlplane }} - podAnnotations: - iam.amazonaws.com/role: "{{ index .Values "aws-ebs-csi-driver" "IamArn" }}" - extraVolumeTags: - Name: {{ .Values.ClusterName }} - + {{- if not ( index .Values "cert-manager" "not_ready" ) }} kiam: enabled: {{ .Values.kiam.enabled }} values: @@ -62,6 +55,20 @@ kubezero: prometheus: servicemonitor: enabled: {{ .Values.prometheus.enabled }} + {{- end }} + + {{- if not .Values.kiam.not_ready }} + # AWS only components + aws-ebs-csi-driver: + enabled: {{ index .Values "aws-ebs-csi-driver" "enabled" }} + values: + aws-ebs-csi-driver: + replicaCount: {{ ternary 2 1 .Values.HighAvailableControlplane }} + podAnnotations: + iam.amazonaws.com/role: "{{ index .Values "aws-ebs-csi-driver" "IamArn" }}" + extraVolumeTags: + Name: {{ .Values.ClusterName }} + {{- end }} aws-efs-csi-driver: enabled: {{ index .Values "aws-efs-csi-driver" "enabled" }} @@ -87,7 +94,6 @@ kubezero: metrics: enabled: {{ .Values.metrics.enabled }} -{{- end }} argo-cd: {{- with index .Values "argo-cd" "server" }} -- 2.40.1 From 91c98e36a4ed3e7771daa4a22e73ccd9cab0e189 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 13:16:48 +0100 Subject: [PATCH 14/96] Enable node_exporter --- charts/kubezero-metrics/values.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index ebdff4a..4f1768d 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -20,7 +20,7 @@ prometheus-operator: kubelet: enabled: false nodeExporter: - enabled: false + enabled: true grafana: enabled: false prometheus: @@ -66,7 +66,8 @@ prometheus-adapter: effect: NoSchedule nodeSelector: node-role.kubernetes.io/master: "" - + # Basic rules for HPA to work replacing heaptster, taken from kube-prometheus project + # https://github.com/coreos/kube-prometheus/blob/master/manifests/prometheus-adapter-configMap.yaml resource: cpu: containerQuery: sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}[5m])) by (<<.GroupBy>>) -- 2.40.1 From 546efb81fd348b66eff05d507258081f0e54ed96 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 13:26:00 +0100 Subject: [PATCH 15/96] Add basic Prometheus itself --- charts/kubezero-metrics/values.yaml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 4f1768d..9da3ee6 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -19,12 +19,8 @@ prometheus-operator: enabled: false kubelet: enabled: false - nodeExporter: - enabled: true grafana: enabled: false - prometheus: - enabled: false defaultRules: create: false @@ -59,6 +55,26 @@ prometheus-operator: # additional: # - kube-system + nodeExporter: + enabled: true + + prometheus: + enabled: true + retention: 10d + + resources: + requests: + memory: 512Mi + + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: ebs-sc-gp2-xfs + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 8Gi + # Metrics adapter prometheus-adapter: tolerations: -- 2.40.1 From 9af637cb2eb4d3355cf6bb553f5a314fd19f5083 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 13:50:32 +0100 Subject: [PATCH 16/96] Enable operator on release ns --- charts/kubezero-metrics/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 9da3ee6..f38e3ab 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -50,8 +50,8 @@ prometheus-operator: admissionWebhooks: enabled: false - #namespaces: - # releaseNamespace: true + namespaces: + releaseNamespace: true # additional: # - kube-system -- 2.40.1 From a90ee78922007e0b44434b1d078b88a2ab15fbfc Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 13:57:14 +0100 Subject: [PATCH 17/96] Enable operator on release ns --- charts/kubezero-metrics/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index f38e3ab..ec8d87a 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -52,8 +52,8 @@ prometheus-operator: namespaces: releaseNamespace: true - # additional: - # - kube-system + additional: + - kube-system nodeExporter: enabled: true -- 2.40.1 From 5002af7b585bbe73bfd7860c056c02739a4a523c Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 15:43:56 +0100 Subject: [PATCH 18/96] Revert to default Prometheus version --- charts/kubezero-metrics/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index ec8d87a..a07e0ef 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -26,10 +26,10 @@ prometheus-operator: prometheusOperator: enabled: true - image: - tag: v0.41.0 - prometheusConfigReloaderImage: - tag: v0.41.0 + #image: + # tag: v0.41.0 + #prometheusConfigReloaderImage: + # tag: v0.41.0 # Run on controller nodes tolerations: -- 2.40.1 From 94d05241e3958680512b3112418994d8280ea160 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 15:51:44 +0100 Subject: [PATCH 19/96] Fix scope of prometheus options --- charts/kubezero-metrics/values.yaml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index a07e0ef..331ee26 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -60,20 +60,21 @@ prometheus-operator: prometheus: enabled: true - retention: 10d - resources: - requests: - memory: 512Mi + prometheusSpec: + retention: 10d + resources: + requests: + memory: 512Mi - storageSpec: - volumeClaimTemplate: - spec: - storageClassName: ebs-sc-gp2-xfs - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 8Gi + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: ebs-sc-gp2-xfs + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 8Gi # Metrics adapter prometheus-adapter: -- 2.40.1 From 7348f6dfa6d035849c0ae3e599a8f1cd5fc24de0 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 16:08:16 +0100 Subject: [PATCH 20/96] Add Grafana --- charts/kubezero-metrics/values.yaml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 331ee26..9b76446 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -19,8 +19,7 @@ prometheus-operator: enabled: false kubelet: enabled: false - grafana: - enabled: false + defaultRules: create: false @@ -76,6 +75,23 @@ prometheus-operator: requests: storage: 8Gi + # Custom Grafana tweaks + # - persistence, plugins, auth + grafana: + enabled: true + persistence: + enabled: true + size: 4Gi + storageClassName: ebs-sc-gp2-xfs + plugins: + - grafana-piechart-panel + service: + portName: http-grafana + initChownData: + enabled: false + testFramework: + enabled: false + # Metrics adapter prometheus-adapter: tolerations: -- 2.40.1 From a49b01d7681277442ddc3c8f0cf756ed61078317 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 16:24:32 +0100 Subject: [PATCH 21/96] Add istio support for metrics grafana --- .../templates/istio-service.yaml | 16 ++++++++++++++++ charts/kubezero-metrics/values.yaml | 6 ++++++ deploy/templates/values.yaml | 8 +++++++- deploy/values.yaml | 2 ++ 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 charts/kubezero-metrics/templates/istio-service.yaml diff --git a/charts/kubezero-metrics/templates/istio-service.yaml b/charts/kubezero-metrics/templates/istio-service.yaml new file mode 100644 index 0000000..2a3490b --- /dev/null +++ b/charts/kubezero-metrics/templates/istio-service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.grafana.istio.enabled }} +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: grafana + namespace: monitoring +spec: + hosts: + - {{ .Values.grafana.istio.url }} + gateways: + - {{ .Values.grafana.istio.gateway }} + http: + - route: + - destination: + host: grafana +{{- end }} diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 9b76446..1e9de2d 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -1,3 +1,9 @@ +grafana: + istio: + enabled: false + url: "" + gateway: ingressgateway.istio-system.svc.cluster.local + prometheus-operator: alertmanager: enabled: false diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 673a1d6..2a35824 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -94,6 +94,12 @@ kubezero: metrics: enabled: {{ .Values.metrics.enabled }} + {{- if and .Values.metrics.istio.enabled .Values.istio.enabled }} + istio: + {{- with .Values.metrics.istio }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} argo-cd: {{- with index .Values "argo-cd" "server" }} @@ -104,7 +110,7 @@ argo-cd: configs: {{- toYaml . | nindent 4 }} {{- end }} - {{- if and ( not .Values.bootstrap ) ( index .Values "argo-cd" "istio" "enabled" ) .Values.istio.enabled }} + {{- if and ( index .Values "argo-cd" "istio" "enabled" ) .Values.istio.enabled }} istio: {{- with index .Values "argo-cd" "istio" }} {{- toYaml . | nindent 4 }} diff --git a/deploy/values.yaml b/deploy/values.yaml index 4572d31..2bbbfe7 100644 --- a/deploy/values.yaml +++ b/deploy/values.yaml @@ -27,6 +27,8 @@ istio: metrics: enabled: false + istio: + enabled: false # Deprecated once metrics goes live prometheus: -- 2.40.1 From 161fbf9fa2e96707c7e69cbd2d3aa4c1f42fd2ed Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 16:34:57 +0100 Subject: [PATCH 22/96] Config fixes, svc name fix --- charts/kubezero-metrics/templates/istio-service.yaml | 2 +- deploy/templates/values.yaml | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/charts/kubezero-metrics/templates/istio-service.yaml b/charts/kubezero-metrics/templates/istio-service.yaml index 2a3490b..b2437a5 100644 --- a/charts/kubezero-metrics/templates/istio-service.yaml +++ b/charts/kubezero-metrics/templates/istio-service.yaml @@ -12,5 +12,5 @@ spec: http: - route: - destination: - host: grafana + host: metrics-grafana {{- end }} diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 2a35824..bdadb2c 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -94,12 +94,14 @@ kubezero: metrics: enabled: {{ .Values.metrics.enabled }} - {{- if and .Values.metrics.istio.enabled .Values.istio.enabled }} - istio: - {{- with .Values.metrics.istio }} - {{- toYaml . | nindent 6 }} + values: + {{- if and .Values.metrics.istio.enabled .Values.istio.enabled }} + grafana: + istio: + {{- with .Values.metrics.istio }} + {{- toYaml . | nindent 10 }} + {{- end }} {{- end }} - {{- end }} argo-cd: {{- with index .Values "argo-cd" "server" }} -- 2.40.1 From 89cbcc97465a47b78e9a4bef6a4b8f8d7eabcd4c Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 17:01:39 +0100 Subject: [PATCH 23/96] Add all the rules minus alertmanager --- charts/kubezero-metrics/values.yaml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 1e9de2d..5c400a9 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -7,27 +7,31 @@ grafana: prometheus-operator: alertmanager: enabled: false + + + defaultRules: + create: true + rules: + alertmanager: false + coreDns: - enabled: false + enabled: true kubeApiServer: - enabled: false + enabled: true kubeControllerManager: - enabled: false + enabled: true kubeDns: enabled: false kubeEtcd: - enabled: false + enabled: true kubeProxy: - enabled: false + enabled: true kubeScheduler: - enabled: false + enabled: true kubeStateMetrics: - enabled: false + enabled: true kubelet: - enabled: false - - defaultRules: - create: false + enabled: true prometheusOperator: enabled: true -- 2.40.1 From 3682efcdf12a9035f11f7e1d70adada583669bd5 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 17:15:12 +0100 Subject: [PATCH 24/96] Frist mostly working version --- charts/kubezero-metrics/Chart.yaml | 2 +- charts/kubezero-metrics/values.yaml | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index d4a3d16..9a15444 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for prometheus-operator type: application -version: 0.0.2 +version: 0.1.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 5c400a9..de4f334 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -5,15 +5,8 @@ grafana: gateway: ingressgateway.istio-system.svc.cluster.local prometheus-operator: - alertmanager: - enabled: false - - defaultRules: create: true - rules: - alertmanager: false - coreDns: enabled: true kubeApiServer: @@ -102,6 +95,10 @@ prometheus-operator: testFramework: enabled: false + # Todo + alertmanager: + enabled: false + # Metrics adapter prometheus-adapter: tolerations: -- 2.40.1 From f6da71451ebfdaea3c2c840ca06beef1d17e99d6 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 17:44:58 +0100 Subject: [PATCH 25/96] Add istio for prometheus --- .../templates/istio-service.yaml | 17 +++++++++++++++++ charts/kubezero-metrics/values.yaml | 10 +++++++++- deploy/templates/values.yaml | 11 +++++++++-- deploy/values.yaml | 5 ++++- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/charts/kubezero-metrics/templates/istio-service.yaml b/charts/kubezero-metrics/templates/istio-service.yaml index b2437a5..5b7c3bd 100644 --- a/charts/kubezero-metrics/templates/istio-service.yaml +++ b/charts/kubezero-metrics/templates/istio-service.yaml @@ -14,3 +14,20 @@ spec: - destination: host: metrics-grafana {{- end }} +--- +{{- if .Values.prometheus.istio.enabled }} +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: prometheus + namespace: monitoring +spec: + hosts: + - {{ .Values.prometheus.istio.url }} + gateways: + - {{ .Values.prometheus.istio.gateway }} + http: + - route: + - destination: + host: metrics-prometheus-operato-prometheus +{{- end }} diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index de4f334..766cbf0 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -4,6 +4,12 @@ grafana: url: "" gateway: ingressgateway.istio-system.svc.cluster.local +prometheus: + istio: + enabled: false + url: "" + gateway: ingressgateway.istio-system.svc.cluster.local + prometheus-operator: defaultRules: create: true @@ -64,7 +70,9 @@ prometheus-operator: enabled: true prometheusSpec: - retention: 10d + retention: 8d + portName: http-prometheus + resources: requests: memory: 512Mi diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index bdadb2c..d3c1ecc 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -95,10 +95,17 @@ kubezero: metrics: enabled: {{ .Values.metrics.enabled }} values: - {{- if and .Values.metrics.istio.enabled .Values.istio.enabled }} + {{- if and .Values.metrics.istio.grafana.enabled .Values.istio.enabled }} grafana: istio: - {{- with .Values.metrics.istio }} + {{- with .Values.metrics.istio.grafana }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if and .Values.metrics.istio.prometheus.enabled .Values.istio.enabled }} + prometheus: + istio: + {{- with .Values.metrics.istio.prometheus }} {{- toYaml . | nindent 10 }} {{- end }} {{- end }} diff --git a/deploy/values.yaml b/deploy/values.yaml index 2bbbfe7..8bc82cf 100644 --- a/deploy/values.yaml +++ b/deploy/values.yaml @@ -28,7 +28,10 @@ istio: metrics: enabled: false istio: - enabled: false + grafana: + enabled: false + prometheus: + enabled: false # Deprecated once metrics goes live prometheus: -- 2.40.1 From 0a0aef5d08b946ad6e079fa9fdc2337290769890 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 18:30:33 +0100 Subject: [PATCH 26/96] Disable unreachable metrics for now --- charts/kubezero-metrics/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 766cbf0..ff8150d 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -18,15 +18,15 @@ prometheus-operator: kubeApiServer: enabled: true kubeControllerManager: - enabled: true + enabled: false kubeDns: enabled: false kubeEtcd: - enabled: true + enabled: false kubeProxy: - enabled: true + enabled: false kubeScheduler: - enabled: true + enabled: false kubeStateMetrics: enabled: true kubelet: -- 2.40.1 From 6f23589fc55cc71d71008352881ad1adf995bba8 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 18:47:11 +0100 Subject: [PATCH 27/96] Try default settings fro adapter --- charts/kubezero-metrics/values.yaml | 40 +++++++++++++++-------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index ff8150d..5581836 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -17,6 +17,12 @@ prometheus-operator: enabled: true kubeApiServer: enabled: true + kubeStateMetrics: + enabled: true + kubelet: + enabled: true + + # Disabled until we figure out how to scrape etcd with ssl client certs, scheduler/proxy/controller require https since 1.17 kubeControllerManager: enabled: false kubeDns: @@ -27,10 +33,6 @@ prometheus-operator: enabled: false kubeScheduler: enabled: false - kubeStateMetrics: - enabled: true - kubelet: - enabled: true prometheusOperator: enabled: true @@ -118,20 +120,8 @@ prometheus-adapter: # https://github.com/coreos/kube-prometheus/blob/master/manifests/prometheus-adapter-configMap.yaml resource: cpu: - containerQuery: sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}[5m])) by (<<.GroupBy>>) - nodeQuery: sum(1 - irate(node_cpu_seconds_total{mode="idle"}[5m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>) - resources: - overrides: - node: - resource: node - namespace: - resource: namespace - pod: - resource: pod - containerLabel: container - memory: - containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}) by (<<.GroupBy>>) - nodeQuery: sum(node_memory_MemTotal_bytes{job="node-exporter",<<.LabelMatchers>>} - node_memory_MemAvailable_bytes{job="node-exporter",<<.LabelMatchers>>}) by (<<.GroupBy>>) + containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) + nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) resources: overrides: instance: @@ -141,4 +131,16 @@ prometheus-adapter: pod: resource: pod containerLabel: container - window: 5m + memory: + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) + nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) + resources: + overrides: + instance: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m -- 2.40.1 From ee19bd42d8e7927279a1ac3385757d1d91270281 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 18:53:18 +0100 Subject: [PATCH 28/96] Fix scopt for adapter rules --- charts/kubezero-metrics/values.yaml | 53 +++++++++++++++-------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 5581836..a091d29 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -118,29 +118,30 @@ prometheus-adapter: node-role.kubernetes.io/master: "" # Basic rules for HPA to work replacing heaptster, taken from kube-prometheus project # https://github.com/coreos/kube-prometheus/blob/master/manifests/prometheus-adapter-configMap.yaml - resource: - cpu: - containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) - nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) - resources: - overrides: - instance: - resource: node - namespace: - resource: namespace - pod: - resource: pod - containerLabel: container - memory: - containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) - nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) - resources: - overrides: - instance: - resource: node - namespace: - resource: namespace - pod: - resource: pod - containerLabel: container - window: 3m + rules: + resource: + cpu: + containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) + nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) + resources: + overrides: + instance: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) + nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) + resources: + overrides: + instance: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m -- 2.40.1 From 40e992d9c5c3a69f5962ff65a30297c73362a025 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 20:52:57 +0100 Subject: [PATCH 29/96] Temp add custom prometheus url --- charts/kubezero-metrics/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index a091d29..05fe82c 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -111,6 +111,8 @@ prometheus-operator: # Metrics adapter prometheus-adapter: + prometheus: + url: metrics-prometheus-operato-prometheus tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule -- 2.40.1 From a838b2c1087e8edd467571571500c2fa9245229a Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 20:56:52 +0100 Subject: [PATCH 30/96] Temp add custom prometheus url --- charts/kubezero-metrics/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 05fe82c..8250a46 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -112,7 +112,7 @@ prometheus-operator: # Metrics adapter prometheus-adapter: prometheus: - url: metrics-prometheus-operato-prometheus + url: http://metrics-prometheus-operato-prometheus tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule -- 2.40.1 From 61196f4ccc4309965917dba3125a5f7aed7a8254 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 21:29:24 +0100 Subject: [PATCH 31/96] Disable default rules for the adapter --- charts/kubezero-metrics/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 8250a46..6d436c4 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -121,6 +121,7 @@ prometheus-adapter: # Basic rules for HPA to work replacing heaptster, taken from kube-prometheus project # https://github.com/coreos/kube-prometheus/blob/master/manifests/prometheus-adapter-configMap.yaml rules: + default: false resource: cpu: containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) -- 2.40.1 From 50bed0573d9fa8e2adc58b631ded4ec7f032eb6e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 22:15:05 +0100 Subject: [PATCH 32/96] Adapter config from kube-prometheus --- charts/kubezero-metrics/values.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 6d436c4..dc97c43 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -124,11 +124,11 @@ prometheus-adapter: default: false resource: cpu: - containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) - nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) + containerQuery: sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}[5m])) by (<<.GroupBy>>) + nodeQuery: sum(1 - irate(node_cpu_seconds_total{mode="idle"}[5m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>) resources: overrides: - instance: + node: resource: node namespace: resource: namespace @@ -136,8 +136,8 @@ prometheus-adapter: resource: pod containerLabel: container memory: - containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) - nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}) by (<<.GroupBy>>) + nodeQuery: sum(node_memory_MemTotal_bytes{job="node-exporter",<<.LabelMatchers>>} - node_memory_MemAvailable_bytes{job="node-exporter",<<.LabelMatchers>>}) by (<<.GroupBy>>) resources: overrides: instance: @@ -147,4 +147,4 @@ prometheus-adapter: pod: resource: pod containerLabel: container - window: 3m + window: 5m -- 2.40.1 From 6707669929d9b23ae536dcb83fdcd9636631742e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 3 Aug 2020 22:19:16 +0100 Subject: [PATCH 33/96] Adapter config from kube-prometheus --- charts/kubezero-metrics/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index dc97c43..c755fd5 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -140,7 +140,7 @@ prometheus-adapter: nodeQuery: sum(node_memory_MemTotal_bytes{job="node-exporter",<<.LabelMatchers>>} - node_memory_MemAvailable_bytes{job="node-exporter",<<.LabelMatchers>>}) by (<<.GroupBy>>) resources: overrides: - instance: + node: resource: node namespace: resource: namespace -- 2.40.1 From 874039f08c1d6abff9750c6d5d3470f26b921067 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 4 Aug 2020 01:38:26 +0100 Subject: [PATCH 34/96] Add node_exporter relabel for adapter --- charts/kubezero-metrics/values.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index c755fd5..51dab7a 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -67,6 +67,14 @@ prometheus-operator: nodeExporter: enabled: true + serviceMonitor: + relabelings: + - sourceLabels: [__meta_kubernetes_pod_node_name] + separator: ; + regex: ^(.*)$ + targetLabel: node + replacement: $1 + action: replace prometheus: enabled: true -- 2.40.1 From 8c0c48f37d83d5ac12d48cb8a6fb1aef3a86dcf0 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 4 Aug 2020 10:08:10 +0100 Subject: [PATCH 35/96] Enable kube_proxy metrics as a trial --- charts/kubezero-metrics/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 51dab7a..f2256ab 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -30,7 +30,7 @@ prometheus-operator: kubeEtcd: enabled: false kubeProxy: - enabled: false + enabled: true kubeScheduler: enabled: false -- 2.40.1 From da9207bf790c55841813697225f0f77a853f134d Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 4 Aug 2020 14:45:42 +0100 Subject: [PATCH 36/96] Disable kubelet cadadvisor metrics --- CHANGES.md | 14 ++++++++++++++ charts/kubezero-metrics/Chart.yaml | 2 +- charts/kubezero-metrics/values.yaml | 10 +++++++--- deploy/deploy.sh | 4 ++++ deploy/templates/values.yaml | 5 +++++ 5 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 CHANGES.md diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..6c36fad --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,14 @@ +# CFN / Platform +- Kube to 1.17 +- Kube-proxy uses ipvs +- metrics support for kube-proxy +- no reliance on custom resource for S3 buckets anymore + + +# Kubezero +- fully automated one command bootstrap incl. all kubezero components +- migrated from kube-prometheuss to community helm charts for metrics +- latest Grafana incl. peristence +- kube-prometheus adapter improvements / customizations +- integrated EFS CSI driver into Kubezero +- prometheus itself can be exposed via istio ingress on demand to ease development of custom metrics diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 9a15444..b08e8ef 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for prometheus-operator type: application -version: 0.1.0 +version: 0.1.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index f2256ab..3d9af7b 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -19,7 +19,7 @@ prometheus-operator: enabled: true kubeStateMetrics: enabled: true - kubelet: + kubeProxy: enabled: true # Disabled until we figure out how to scrape etcd with ssl client certs, scheduler/proxy/controller require https since 1.17 @@ -29,11 +29,15 @@ prometheus-operator: enabled: false kubeEtcd: enabled: false - kubeProxy: - enabled: true kubeScheduler: enabled: false + kubelet: + enabled: true + serviceMonitor: + # removed with 1.18 anyways + cAdvisor: false + prometheusOperator: enabled: true #image: diff --git a/deploy/deploy.sh b/deploy/deploy.sh index d2b458a..62a4cab 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -50,6 +50,10 @@ EOF echo "Waiting for cert-manager to be deployed..." wait_for kubectl get deployment -n cert-manager cert-manager-webhook 2>/dev/null 1>&2 kubectl rollout status deployment -n cert-manager cert-manager-webhook + + # Now that we have the cert-manager webhook, get the kiam certs in place but do NOT deploy kiam yet + helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set kiam.enabled=false --set istio.enabled=false --set prometheus.enabled=false > generated-values.yaml + helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml wait_for kubectl get Issuer -n kube-system kubezero-local-ca-issuer 2>/dev/null 1>&2 kubectl wait --for=condition=Ready -n kube-system Issuer/kubezero-local-ca-issuer diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index d3c1ecc..efb8c99 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -13,6 +13,11 @@ kubezero: cert-manager: enabled: {{ index .Values "cert-manager" "enabled" }} values: + # Disable all until webhook is in place + {{- if index .Values "cert-manager" "not_ready" }} + localCA: + enabled: false + {{- end }} {{- if eq .Values.platform "aws" }} cert-manager: podAnnotations: -- 2.40.1 From e25c125a439a14d140d3ebed8d964a30a47c664e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 5 Aug 2020 00:50:26 +0100 Subject: [PATCH 37/96] Enable Volumesnapshot/resize, disable leader election of single instance --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 3 ++- .../charts/aws-ebs-csi-driver/templates/deployment.yaml | 6 ++++++ charts/kubezero-aws-ebs-csi-driver/values.yaml | 4 ++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index aead9dc..c8a3654 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,7 +2,8 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.1.1 +version: 0.2.0 +appVersion: 0.5.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png sources: diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml index 3316e96..8cd04c1 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml @@ -91,8 +91,10 @@ spec: {{- if .Values.enableVolumeScheduling }} - --feature-gates=Topology=true {{- end}} + {{- if gt .Values.replicaCount 1 }} - --enable-leader-election - --leader-election-type=leases + {{- end}} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -104,8 +106,10 @@ spec: args: - --csi-address=$(ADDRESS) - --v=5 + {{- if gt .Values.replicaCount 1 }} - --leader-election=true - --leader-election-type=leases + {{- end}} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -117,7 +121,9 @@ spec: image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} args: - --csi-address=$(ADDRESS) + {{- if gt .Values.replicaCount 1 }} - --leader-election=true + {{- end}} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock diff --git a/charts/kubezero-aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml index 9f8bb5b..bcf1052 100644 --- a/charts/kubezero-aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/values.yaml @@ -2,8 +2,8 @@ aws-ebs-csi-driver: replicaCount: 1 enableVolumeScheduling: true - enableVolumeResizing: false - enableVolumeSnapshot: false + enableVolumeResizing: true + enableVolumeSnapshot: true nodeSelector: node-role.kubernetes.io/master: "" -- 2.40.1 From ee6510a2087f2cbae02fbfe9e7d7843be5aa21b4 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 5 Aug 2020 00:56:31 +0100 Subject: [PATCH 38/96] Fix Go template specialness --- .../charts/aws-ebs-csi-driver/templates/deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml index 8cd04c1..209e8fd 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml @@ -91,7 +91,7 @@ spec: {{- if .Values.enableVolumeScheduling }} - --feature-gates=Topology=true {{- end}} - {{- if gt .Values.replicaCount 1 }} + {{- if gt .Values.replicaCount 1.0 }} - --enable-leader-election - --leader-election-type=leases {{- end}} @@ -106,7 +106,7 @@ spec: args: - --csi-address=$(ADDRESS) - --v=5 - {{- if gt .Values.replicaCount 1 }} + {{- if gt .Values.replicaCount 1.0 }} - --leader-election=true - --leader-election-type=leases {{- end}} @@ -121,7 +121,7 @@ spec: image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} args: - --csi-address=$(ADDRESS) - {{- if gt .Values.replicaCount 1 }} + {{- if gt .Values.replicaCount 1.0 }} - --leader-election=true {{- end}} env: -- 2.40.1 From f81ba2c0089428f027156ffcc28df96ce69b721a Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 5 Aug 2020 01:00:11 +0100 Subject: [PATCH 39/96] Volume features need more testing <1.17 --- charts/kubezero-aws-ebs-csi-driver/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml index bcf1052..9f8bb5b 100644 --- a/charts/kubezero-aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/values.yaml @@ -2,8 +2,8 @@ aws-ebs-csi-driver: replicaCount: 1 enableVolumeScheduling: true - enableVolumeResizing: true - enableVolumeSnapshot: true + enableVolumeResizing: false + enableVolumeSnapshot: false nodeSelector: node-role.kubernetes.io/master: "" -- 2.40.1 From ef38eef2455a6a5cc0031d2b7e773d818ab08cdc Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 5 Aug 2020 01:05:05 +0100 Subject: [PATCH 40/96] Finally fix go templating awkwardness --- .../charts/aws-ebs-csi-driver/templates/deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml index 209e8fd..8aeddc1 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml @@ -91,7 +91,7 @@ spec: {{- if .Values.enableVolumeScheduling }} - --feature-gates=Topology=true {{- end}} - {{- if gt .Values.replicaCount 1.0 }} + {{- if gt ( .Values.replicaCount | int ) 1 }} - --enable-leader-election - --leader-election-type=leases {{- end}} @@ -106,7 +106,7 @@ spec: args: - --csi-address=$(ADDRESS) - --v=5 - {{- if gt .Values.replicaCount 1.0 }} + {{- if gt ( .Values.replicaCount | int ) 1 }} - --leader-election=true - --leader-election-type=leases {{- end}} @@ -121,7 +121,7 @@ spec: image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} args: - --csi-address=$(ADDRESS) - {{- if gt .Values.replicaCount 1.0 }} + {{- if gt ( .Values.replicaCount | int ) 1 }} - --leader-election=true {{- end}} env: -- 2.40.1 From 07167f3908c9ce97ef90d5b14667b33aa000e5ce Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 5 Aug 2020 13:29:50 +0100 Subject: [PATCH 41/96] ArgoCd version bump --- charts/kubezero-argo-cd/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-argo-cd/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml index 9c86450..bbebec8 100644 --- a/charts/kubezero-argo-cd/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application name: kubezero-argo-cd -version: 0.3.7 +version: 0.3.8 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,6 +16,6 @@ dependencies: version: ">= 0.1.1" repository: https://zero-down-time.github.io/kubezero/ - name: argo-cd - version: 2.5.0 + version: 2.6.0 repository: https://argoproj.github.io/argo-helm kubeVersion: ">= 1.16.0" -- 2.40.1 From b541463829de92796540f16361150d1da1176846 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 5 Aug 2020 15:42:15 +0100 Subject: [PATCH 42/96] Add cert-manager backup support in bootstrap, enable schedule and controller metrics --- charts/kubezero-cert-manager/backup-all.sh | 7 +++++++ charts/kubezero-metrics/values.yaml | 22 +++++++++++++++++----- deploy/deploy.sh | 22 ++++++++++++++++------ deploy/templates/values.yaml | 2 ++ 4 files changed, 42 insertions(+), 11 deletions(-) create mode 100755 charts/kubezero-cert-manager/backup-all.sh diff --git a/charts/kubezero-cert-manager/backup-all.sh b/charts/kubezero-cert-manager/backup-all.sh new file mode 100755 index 0000000..03707bb --- /dev/null +++ b/charts/kubezero-cert-manager/backup-all.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +kubectl get -A -o yaml issuer,clusterissuer,certificates,certificaterequests > cert-manager-backup.yaml +echo '---' >> cert-manager-backup.yaml +kubectl get -A -o yaml secrets --field-selector type=kubernetes.io/tls >> cert-manager-backup.yaml +echo '---' >> cert-manager-backup.yaml +kubectl get -o yaml secrets -n cert-manager letsencrypt-dns-prod >> cert-manager-backup.yaml diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 3d9af7b..871cfaa 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -23,20 +23,32 @@ prometheus-operator: enabled: true # Disabled until we figure out how to scrape etcd with ssl client certs, scheduler/proxy/controller require https since 1.17 - kubeControllerManager: - enabled: false kubeDns: enabled: false kubeEtcd: enabled: false + + kubeControllerManager: + enabled: true + service: + port: 10257 + targetPort: 10257 + serviceMonitor: + https: true + kubeScheduler: - enabled: false + enabled: true + service: + port: 10259 + targetPort: 10259 + serviceMonitor: + https: true kubelet: enabled: true serviceMonitor: - # removed with 1.18 anyways - cAdvisor: false + # removed with 1.18, but still required for all container metrics ?? + cAdvisor: true prometheusOperator: enabled: true diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 62a4cab..d20ee6e 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -41,7 +41,7 @@ EOF fi # Deploy initial argo-cad - helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set cert-manager.not_ready=true --set istio.enabled=false --set prometheus.enabled=false > generated-values.yaml + helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set cert-manager.not_ready=true --set istio.enabled=false --set metrics.enabled=false > generated-values.yaml helm install -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml # Wait for argocd-server to be running kubectl rollout status deployment -n argocd kubezero-argocd-server @@ -51,20 +51,30 @@ EOF wait_for kubectl get deployment -n cert-manager cert-manager-webhook 2>/dev/null 1>&2 kubectl rollout status deployment -n cert-manager cert-manager-webhook + # Either inject cert-manager backup or bootstrap + if [ -f cert-manager-backup.yaml ]; then + kubectl apply -f cert-manager-backup.yaml + else + helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set istio.enabled=false --set metrics.enabled=false > generated-values.yaml + helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml + wait_for kubectl get Issuer -n kube-system kubezero-local-ca-issuer 2>/dev/null 1>&2 + wait_for kubectl get ClusterIssuer letsencrypt-dns-prod 2>/dev/null 1>&2 + kubectl wait --for=condition=Ready -n kube-system Issuer/kubezero-local-ca-issuer + kubectl wait --for=condition=Ready ClusterIssuer/letsencrypt-dns-prod + fi + # Now that we have the cert-manager webhook, get the kiam certs in place but do NOT deploy kiam yet - helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set kiam.enabled=false --set istio.enabled=false --set prometheus.enabled=false > generated-values.yaml + helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set kiam.enabled=false > generated-values.yaml helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml - wait_for kubectl get Issuer -n kube-system kubezero-local-ca-issuer 2>/dev/null 1>&2 - kubectl wait --for=condition=Ready -n kube-system Issuer/kubezero-local-ca-issuer # Now lets make sure kiam is working - helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set istio.enabled=false --set prometheus.enabled=false > generated-values.yaml + helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true > generated-values.yaml helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml wait_for kubectl get daemonset -n kube-system kiam-agent 2>/dev/null 1>&2 kubectl rollout status daemonset -n kube-system kiam-agent # Install Istio if enabled, but keep ArgoCD istio support disabled for now in case - helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set argo-cd.istio.enabled=false > generated-values.yaml + helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set argo-cd.istio.enabled=false --set metrics.istio.prometheus.enabled=false --set metrics.istio.grafana.enabled=false > generated-values.yaml helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd -f generated-values.yaml wait_for kubectl get deployment -n istio-operator istio-operator 2>/dev/null 1>&2 kubectl rollout status deployment -n istio-operator istio-operator diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index efb8c99..1f2cee1 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -92,10 +92,12 @@ kubezero: values: istiod: replicaCount: {{ ternary 2 1 .Values.HighAvailableControlplane }} + {{- if not ( index .Values "cert-manager" "not_ready" ) }} {{- if .Values.istio.ingress }} ingress: {{- toYaml .Values.istio.ingress | nindent 8 }} {{- end }} + {{- end }} metrics: enabled: {{ .Values.metrics.enabled }} -- 2.40.1 From ae1d721a09293fee8f51edfead1e354548a50e86 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 5 Aug 2020 15:58:37 +0100 Subject: [PATCH 43/96] Controller and scheduler use self-signed certs --- charts/kubezero-metrics/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 871cfaa..0ed9273 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -35,6 +35,7 @@ prometheus-operator: targetPort: 10257 serviceMonitor: https: true + insecureSkipVerify: true kubeScheduler: enabled: true @@ -43,6 +44,7 @@ prometheus-operator: targetPort: 10259 serviceMonitor: https: true + insecureSkipVerify: true kubelet: enabled: true -- 2.40.1 From 7cb0c4432cd72c9538f5f644f435f7de3689c465 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 6 Aug 2020 11:52:16 +0100 Subject: [PATCH 44/96] Add custom prometehus-operator settings --- CHANGES.md | 3 ++- charts/kubezero-metrics/values.yaml | 8 +++++--- deploy/templates/values.yaml | 4 ++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6c36fad..02ab072 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,8 +7,9 @@ # Kubezero - fully automated one command bootstrap incl. all kubezero components -- migrated from kube-prometheuss to community helm charts for metrics +- migrated from kube-prometheuss to prometheus-operator helm charts for metrics - latest Grafana incl. peristence - kube-prometheus adapter improvements / customizations - integrated EFS CSI driver into Kubezero - prometheus itself can be exposed via istio ingress on demand to ease development of custom metrics +- backup script to export all cert-manager items between clusters diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 0ed9273..0d14b5f 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -13,8 +13,12 @@ prometheus: prometheus-operator: defaultRules: create: true + coreDns: enabled: true + kubeDns: + enabled: false + kubeApiServer: enabled: true kubeStateMetrics: @@ -22,9 +26,7 @@ prometheus-operator: kubeProxy: enabled: true - # Disabled until we figure out how to scrape etcd with ssl client certs, scheduler/proxy/controller require https since 1.17 - kubeDns: - enabled: false + # Disabled until we figure out how to scrape etcd with ssl client certs kubeEtcd: enabled: false diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 1f2cee1..71e1337 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -116,6 +116,10 @@ kubezero: {{- toYaml . | nindent 10 }} {{- end }} {{- end }} + {{- if .Values.metrics.prometheus-operator }} + prometheus-operator: + {{- toYaml .Values.metrics.prometheus-operator | nindent 8 }} + {{- end }} argo-cd: {{- with index .Values "argo-cd" "server" }} -- 2.40.1 From d532aaf0c979fe1b7f62aa8d0459df59514b9ab2 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 6 Aug 2020 11:34:32 +0000 Subject: [PATCH 45/96] Add optional istio ingress policies to metrics --- charts/kubezero-metrics/Chart.yaml | 2 +- .../templates/istio-authorization-policy.yaml | 49 +++++++++++++++++++ charts/kubezero-metrics/values.yaml | 1 + deploy/templates/values.yaml | 6 ++- 4 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 charts/kubezero-metrics/templates/istio-authorization-policy.yaml diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index b08e8ef..69fe1ae 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for prometheus-operator type: application -version: 0.1.1 +version: 0.1.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-metrics/templates/istio-authorization-policy.yaml b/charts/kubezero-metrics/templates/istio-authorization-policy.yaml new file mode 100644 index 0000000..a5b01c1 --- /dev/null +++ b/charts/kubezero-metrics/templates/istio-authorization-policy.yaml @@ -0,0 +1,49 @@ +{{- if .Values.grafana.istio.enabled }} +{{- if .Values.grafana.istio.ipBlocks }} +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: grafana-deny-not-in-ipblocks + namespace: istio-system +spec: + selector: + matchLabels: + app: istio-ingressgateway + action: DENY + rules: + - from: + - source: + notIpBlocks: + {{- with .Values.grafana.istio.ipBlocks }} + {{- . | toYaml | nindent 8 }} + {{- end }} + to: + - operation: + hosts: ["{{ .Values.grafana.istio.url }}"] +{{- end }} +{{- end }} +--- +{{- if .Values.prometheus.istio.enabled }} +{{- if .Values.prometheus.istio.ipBlocks }} +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: prometheus-deny-not-in-ipblocks + namespace: istio-system +spec: + selector: + matchLabels: + app: istio-ingressgateway + action: DENY + rules: + - from: + - source: + notIpBlocks: + {{- with .Values.prometheus.istio.ipBlocks }} + {{- . | toYaml | nindent 8 }} + {{- end }} + to: + - operation: + hosts: ["{{ .Values.prometheus.istio.url }}"] +{{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 0d14b5f..881b91a 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -1,6 +1,7 @@ grafana: istio: enabled: false + ipBlocks: [] url: "" gateway: ingressgateway.istio-system.svc.cluster.local diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 71e1337..2b1409a 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -116,9 +116,11 @@ kubezero: {{- toYaml . | nindent 10 }} {{- end }} {{- end }} - {{- if .Values.metrics.prometheus-operator }} + {{- if index .Values "metrics" "prometheus-operator" }} prometheus-operator: - {{- toYaml .Values.metrics.prometheus-operator | nindent 8 }} + {{- with index .Values "metrics" "prometheus-operator" }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} argo-cd: -- 2.40.1 From d34536e9e2439e06700c6ad48ea75231eec4b0c6 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 6 Aug 2020 12:38:40 +0100 Subject: [PATCH 46/96] Add latest docs --- charts/kubezero-argo-cd/README.md | 4 +- charts/kubezero-aws-ebs-csi-driver/README.md | 2 +- charts/kubezero-metrics/README.md | 85 ++++++++++++++++---- 3 files changed, 74 insertions(+), 17 deletions(-) diff --git a/charts/kubezero-argo-cd/README.md b/charts/kubezero-argo-cd/README.md index 38acc14..3f3ff38 100644 --- a/charts/kubezero-argo-cd/README.md +++ b/charts/kubezero-argo-cd/README.md @@ -2,7 +2,7 @@ kubezero-argo-cd ================ KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application -Current chart version is `0.3.7` +Current chart version is `0.3.8` Source code can be found [here](https://kubezero.com) @@ -10,7 +10,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| -| https://argoproj.github.io/argo-helm | argo-cd | 2.5.0 | +| https://argoproj.github.io/argo-helm | argo-cd | 2.6.0 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | ## Chart Values diff --git a/charts/kubezero-aws-ebs-csi-driver/README.md b/charts/kubezero-aws-ebs-csi-driver/README.md index 58e0fca..0ed9f44 100644 --- a/charts/kubezero-aws-ebs-csi-driver/README.md +++ b/charts/kubezero-aws-ebs-csi-driver/README.md @@ -2,7 +2,7 @@ kubezero-aws-ebs-csi-driver =========================== KubeZero Umbrella Chart for aws-ebs-csi-driver -Current chart version is `0.1.1` +Current chart version is `0.2.0` Source code can be found [here](https://kubezero.com) diff --git a/charts/kubezero-metrics/README.md b/charts/kubezero-metrics/README.md index 55028c9..5f42125 100644 --- a/charts/kubezero-metrics/README.md +++ b/charts/kubezero-metrics/README.md @@ -2,7 +2,7 @@ kubezero-metrics ================ KubeZero Umbrella Chart for prometheus-operator -Current chart version is `0.0.1` +Current chart version is `0.1.2` Source code can be found [here](https://kubezero.com) @@ -10,6 +10,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| +| https://kubernetes-charts.storage.googleapis.com/ | prometheus-adapter | 2.5.0 | | https://kubernetes-charts.storage.googleapis.com/ | prometheus-operator | 9.3.0 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | @@ -17,24 +18,80 @@ Source code can be found [here](https://kubezero.com) | Key | Type | Default | Description | |-----|------|---------|-------------| +| grafana.istio.enabled | bool | `false` | | +| grafana.istio.gateway | string | `"ingressgateway.istio-system.svc.cluster.local"` | | +| grafana.istio.ipBlocks | list | `[]` | | +| grafana.istio.url | string | `""` | | +| prometheus-adapter.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| prometheus-adapter.prometheus.url | string | `"http://metrics-prometheus-operato-prometheus"` | | +| prometheus-adapter.rules.default | bool | `false` | | +| prometheus-adapter.rules.resource.cpu.containerLabel | string | `"container"` | | +| prometheus-adapter.rules.resource.cpu.containerQuery | string | `"sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!=\"POD\",container!=\"\",pod!=\"\"}[5m])) by (<<.GroupBy>>)"` | | +| prometheus-adapter.rules.resource.cpu.nodeQuery | string | `"sum(1 - irate(node_cpu_seconds_total{mode=\"idle\"}[5m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>)"` | | +| prometheus-adapter.rules.resource.cpu.resources.overrides.namespace.resource | string | `"namespace"` | | +| prometheus-adapter.rules.resource.cpu.resources.overrides.node.resource | string | `"node"` | | +| prometheus-adapter.rules.resource.cpu.resources.overrides.pod.resource | string | `"pod"` | | +| prometheus-adapter.rules.resource.memory.containerLabel | string | `"container"` | | +| prometheus-adapter.rules.resource.memory.containerQuery | string | `"sum(container_memory_working_set_bytes{<<.LabelMatchers>>,container!=\"POD\",container!=\"\",pod!=\"\"}) by (<<.GroupBy>>)"` | | +| prometheus-adapter.rules.resource.memory.nodeQuery | string | `"sum(node_memory_MemTotal_bytes{job=\"node-exporter\",<<.LabelMatchers>>} - node_memory_MemAvailable_bytes{job=\"node-exporter\",<<.LabelMatchers>>}) by (<<.GroupBy>>)"` | | +| prometheus-adapter.rules.resource.memory.resources.overrides.namespace.resource | string | `"namespace"` | | +| prometheus-adapter.rules.resource.memory.resources.overrides.node.resource | string | `"node"` | | +| prometheus-adapter.rules.resource.memory.resources.overrides.pod.resource | string | `"pod"` | | +| prometheus-adapter.rules.resource.window | string | `"5m"` | | +| prometheus-adapter.tolerations[0].effect | string | `"NoSchedule"` | | +| prometheus-adapter.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | prometheus-operator.alertmanager.enabled | bool | `false` | | -| prometheus-operator.coreDns.enabled | bool | `false` | | -| prometheus-operator.defaultRules.create | bool | `false` | | -| prometheus-operator.grafana.enabled | bool | `false` | | -| prometheus-operator.kubeApiServer.enabled | bool | `false` | | -| prometheus-operator.kubeControllerManager.enabled | bool | `false` | | +| prometheus-operator.coreDns.enabled | bool | `true` | | +| prometheus-operator.defaultRules.create | bool | `true` | | +| prometheus-operator.grafana.enabled | bool | `true` | | +| prometheus-operator.grafana.initChownData.enabled | bool | `false` | | +| prometheus-operator.grafana.persistence.enabled | bool | `true` | | +| prometheus-operator.grafana.persistence.size | string | `"4Gi"` | | +| prometheus-operator.grafana.persistence.storageClassName | string | `"ebs-sc-gp2-xfs"` | | +| prometheus-operator.grafana.plugins[0] | string | `"grafana-piechart-panel"` | | +| prometheus-operator.grafana.service.portName | string | `"http-grafana"` | | +| prometheus-operator.grafana.testFramework.enabled | bool | `false` | | +| prometheus-operator.kubeApiServer.enabled | bool | `true` | | +| prometheus-operator.kubeControllerManager.enabled | bool | `true` | | +| prometheus-operator.kubeControllerManager.service.port | int | `10257` | | +| prometheus-operator.kubeControllerManager.service.targetPort | int | `10257` | | +| prometheus-operator.kubeControllerManager.serviceMonitor.https | bool | `true` | | +| prometheus-operator.kubeControllerManager.serviceMonitor.insecureSkipVerify | bool | `true` | | | prometheus-operator.kubeDns.enabled | bool | `false` | | | prometheus-operator.kubeEtcd.enabled | bool | `false` | | -| prometheus-operator.kubeProxy.enabled | bool | `false` | | -| prometheus-operator.kubeScheduler.enabled | bool | `false` | | -| prometheus-operator.kubeStateMetrics.enabled | bool | `false` | | -| prometheus-operator.kubelet.enabled | bool | `false` | | -| prometheus-operator.nodeExporter.enabled | bool | `false` | | -| prometheus-operator.prometheus.enabled | bool | `false` | | +| prometheus-operator.kubeProxy.enabled | bool | `true` | | +| prometheus-operator.kubeScheduler.enabled | bool | `true` | | +| prometheus-operator.kubeScheduler.service.port | int | `10259` | | +| prometheus-operator.kubeScheduler.service.targetPort | int | `10259` | | +| prometheus-operator.kubeScheduler.serviceMonitor.https | bool | `true` | | +| prometheus-operator.kubeScheduler.serviceMonitor.insecureSkipVerify | bool | `true` | | +| prometheus-operator.kubeStateMetrics.enabled | bool | `true` | | +| prometheus-operator.kubelet.enabled | bool | `true` | | +| prometheus-operator.kubelet.serviceMonitor.cAdvisor | bool | `true` | | +| prometheus-operator.nodeExporter.enabled | bool | `true` | | +| prometheus-operator.nodeExporter.serviceMonitor.relabelings[0].action | string | `"replace"` | | +| prometheus-operator.nodeExporter.serviceMonitor.relabelings[0].regex | string | `"^(.*)$"` | | +| prometheus-operator.nodeExporter.serviceMonitor.relabelings[0].replacement | string | `"$1"` | | +| prometheus-operator.nodeExporter.serviceMonitor.relabelings[0].separator | string | `";"` | | +| prometheus-operator.nodeExporter.serviceMonitor.relabelings[0].sourceLabels[0] | string | `"__meta_kubernetes_pod_node_name"` | | +| prometheus-operator.nodeExporter.serviceMonitor.relabelings[0].targetLabel | string | `"node"` | | +| prometheus-operator.prometheus.enabled | bool | `true` | | +| prometheus-operator.prometheus.prometheusSpec.portName | string | `"http-prometheus"` | | +| prometheus-operator.prometheus.prometheusSpec.resources.requests.memory | string | `"512Mi"` | | +| prometheus-operator.prometheus.prometheusSpec.retention | string | `"8d"` | | +| prometheus-operator.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.accessModes[0] | string | `"ReadWriteOnce"` | | +| prometheus-operator.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage | string | `"8Gi"` | | +| prometheus-operator.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.storageClassName | string | `"ebs-sc-gp2-xfs"` | | | prometheus-operator.prometheusOperator.admissionWebhooks.enabled | bool | `false` | | -| prometheus-operator.prometheusOperator.createCustomResource | bool | `false` | | +| prometheus-operator.prometheusOperator.createCustomResource | bool | `true` | | | prometheus-operator.prometheusOperator.enabled | bool | `true` | | +| prometheus-operator.prometheusOperator.manageCrds | bool | `false` | | | prometheus-operator.prometheusOperator.namespaces.additional[0] | string | `"kube-system"` | | | prometheus-operator.prometheusOperator.namespaces.releaseNamespace | bool | `true` | | -| prometheus-operator.prometheusOperator.serviceMonitor.selfMonitor | bool | `false` | | +| prometheus-operator.prometheusOperator.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | prometheus-operator.prometheusOperator.tlsProxy.enabled | bool | `false` | | +| prometheus-operator.prometheusOperator.tolerations[0].effect | string | `"NoSchedule"` | | +| prometheus-operator.prometheusOperator.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| prometheus.istio.enabled | bool | `false` | | +| prometheus.istio.gateway | string | `"ingressgateway.istio-system.svc.cluster.local"` | | +| prometheus.istio.url | string | `""` | | -- 2.40.1 From 58bf5653596e327642740408ec8893ebae8186a0 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 6 Aug 2020 17:15:20 +0000 Subject: [PATCH 47/96] Add default labels --- .../templates/istio-authorization-policy.yaml | 2 ++ .../templates/istio-authorization-policy.yaml | 4 ++++ charts/kubezero-metrics/templates/istio-service.yaml | 6 ++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-argo-cd/templates/istio-authorization-policy.yaml b/charts/kubezero-argo-cd/templates/istio-authorization-policy.yaml index e9522f1..d757cd6 100644 --- a/charts/kubezero-argo-cd/templates/istio-authorization-policy.yaml +++ b/charts/kubezero-argo-cd/templates/istio-authorization-policy.yaml @@ -5,6 +5,8 @@ kind: AuthorizationPolicy metadata: name: argocd-deny-not-in-ipblocks namespace: istio-system + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: selector: matchLabels: diff --git a/charts/kubezero-metrics/templates/istio-authorization-policy.yaml b/charts/kubezero-metrics/templates/istio-authorization-policy.yaml index a5b01c1..29e832e 100644 --- a/charts/kubezero-metrics/templates/istio-authorization-policy.yaml +++ b/charts/kubezero-metrics/templates/istio-authorization-policy.yaml @@ -5,6 +5,8 @@ kind: AuthorizationPolicy metadata: name: grafana-deny-not-in-ipblocks namespace: istio-system + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: selector: matchLabels: @@ -30,6 +32,8 @@ kind: AuthorizationPolicy metadata: name: prometheus-deny-not-in-ipblocks namespace: istio-system + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: selector: matchLabels: diff --git a/charts/kubezero-metrics/templates/istio-service.yaml b/charts/kubezero-metrics/templates/istio-service.yaml index 5b7c3bd..366962e 100644 --- a/charts/kubezero-metrics/templates/istio-service.yaml +++ b/charts/kubezero-metrics/templates/istio-service.yaml @@ -3,7 +3,8 @@ apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: name: grafana - namespace: monitoring + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: hosts: - {{ .Values.grafana.istio.url }} @@ -20,7 +21,8 @@ apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: name: prometheus - namespace: monitoring + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: hosts: - {{ .Values.prometheus.istio.url }} -- 2.40.1 From bf6c8561538ada28a28fce1f8b5f5af8b12d42a0 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 6 Aug 2020 17:21:27 +0000 Subject: [PATCH 48/96] Extend shared library for naming functions --- charts/kubezero-lib/Chart.yaml | 2 +- charts/kubezero-lib/templates/_helpers.tpl | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-lib/Chart.yaml b/charts/kubezero-lib/Chart.yaml index 5c6e431..68ed401 100644 --- a/charts/kubezero-lib/Chart.yaml +++ b/charts/kubezero-lib/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-lib description: KubeZero helm library - common helm functions and blocks type: library -version: 0.1.2 +version: 0.1.3 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-lib/templates/_helpers.tpl b/charts/kubezero-lib/templates/_helpers.tpl index e211b10..b6d75b6 100644 --- a/charts/kubezero-lib/templates/_helpers.tpl +++ b/charts/kubezero-lib/templates/_helpers.tpl @@ -3,6 +3,27 @@ Common set of labels */ -}} {{- define "kubezero-lib.labels" -}} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +app.kubernetes.io/name: {{ include "kubezero-lib.name" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/part-of: kubezero {{- end -}} + +{{- /* +Common naming functions +*/ -}} +{{- define "kubezero-lib.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "kubezero-lib.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} -- 2.40.1 From 08a73b6e82d9cbedaba6e0ed710992a48b28f129 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 6 Aug 2020 18:43:59 +0100 Subject: [PATCH 49/96] Various deps updates, Istio to 1.6.7 --- charts/kubezero-argo-cd/Chart.yaml | 2 +- charts/kubezero-istio/Chart.yaml | 6 +++--- charts/kubezero-istio/README.md | 6 +++--- charts/kubezero-istio/templates/ingress-certificate.yaml | 2 ++ charts/kubezero-istio/templates/ingress-gateway.yaml | 4 ++++ charts/kubezero-istio/templates/istio-private-ingress.yaml | 2 ++ charts/kubezero-istio/templates/istio.yaml | 2 ++ charts/kubezero-istio/templates/namespace.yaml | 1 + charts/kubezero-istio/update.sh | 2 +- charts/kubezero-istio/values.yaml | 2 +- charts/kubezero-lib/templates/_helpers.tpl | 6 +++++- charts/kubezero-metrics/Chart.yaml | 2 +- 12 files changed, 26 insertions(+), 11 deletions(-) diff --git a/charts/kubezero-argo-cd/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml index bbebec8..4ff15ca 100644 --- a/charts/kubezero-argo-cd/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -13,7 +13,7 @@ maintainers: dependencies: dependencies: - name: kubezero-lib - version: ">= 0.1.1" + version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: argo-cd version: 2.6.0 diff --git a/charts/kubezero-istio/Chart.yaml b/charts/kubezero-istio/Chart.yaml index df7d0d8..f996613 100644 --- a/charts/kubezero-istio/Chart.yaml +++ b/charts/kubezero-istio/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-istio description: KubeZero Umbrella Chart for Istio type: application -version: 0.2.0 -appVersion: 1.6.5 +version: 0.2.1 +appVersion: 1.6.7 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -13,7 +13,7 @@ maintainers: - name: Quarky9 dependencies: - name: kubezero-lib - version: ">= 0.1.1" + version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: istio-operator version: ">= 1.6" diff --git a/charts/kubezero-istio/README.md b/charts/kubezero-istio/README.md index 28ddf22..6b7ba4e 100644 --- a/charts/kubezero-istio/README.md +++ b/charts/kubezero-istio/README.md @@ -5,7 +5,7 @@ KubeZero Umbrella Chart for Istio Installs Istio Operator and KubeZero Istio profile -Current chart version is `0.2.0` +Current chart version is `0.2.1` Source code can be found [here](https://kubezero.com) @@ -14,7 +14,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| | | istio-operator | >= 1.6 | -| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## KubeZero default configuration - mapped istio-operator to run on the controller nodes only @@ -28,7 +28,7 @@ Source code can be found [here](https://kubezero.com) | ingress.replicaCount | int | `2` | | | ingress.type | string | `"NodePort"` | | | istio-operator.hub | string | `"docker.io/istio"` | | -| istio-operator.tag | string | `"1.6.5"` | | +| istio-operator.tag | string | `"1.6.7"` | | | istiod.autoscaleEnabled | bool | `false` | | | istiod.replicaCount | int | `1` | | diff --git a/charts/kubezero-istio/templates/ingress-certificate.yaml b/charts/kubezero-istio/templates/ingress-certificate.yaml index 12a512d..eab9e8c 100644 --- a/charts/kubezero-istio/templates/ingress-certificate.yaml +++ b/charts/kubezero-istio/templates/ingress-certificate.yaml @@ -4,6 +4,8 @@ kind: Certificate metadata: name: public-ingress-cert namespace: istio-system + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: secretName: public-ingress-cert issuerRef: diff --git a/charts/kubezero-istio/templates/ingress-gateway.yaml b/charts/kubezero-istio/templates/ingress-gateway.yaml index 907d107..9d90242 100644 --- a/charts/kubezero-istio/templates/ingress-gateway.yaml +++ b/charts/kubezero-istio/templates/ingress-gateway.yaml @@ -3,6 +3,8 @@ kind: Gateway metadata: name: ingressgateway namespace: istio-system + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: selector: istio: ingressgateway @@ -33,6 +35,8 @@ kind: Gateway metadata: name: private-ingressgateway namespace: istio-system + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: selector: istio: private-ingressgateway diff --git a/charts/kubezero-istio/templates/istio-private-ingress.yaml b/charts/kubezero-istio/templates/istio-private-ingress.yaml index a6e0dfc..c946542 100644 --- a/charts/kubezero-istio/templates/istio-private-ingress.yaml +++ b/charts/kubezero-istio/templates/istio-private-ingress.yaml @@ -4,6 +4,8 @@ kind: IstioOperator metadata: name: kubezero-istio-private-ingress namespace: istio-system + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: profile: empty components: diff --git a/charts/kubezero-istio/templates/istio.yaml b/charts/kubezero-istio/templates/istio.yaml index 8fcc2ed..bea5e87 100644 --- a/charts/kubezero-istio/templates/istio.yaml +++ b/charts/kubezero-istio/templates/istio.yaml @@ -3,6 +3,8 @@ kind: IstioOperator metadata: name: kubezero-istio namespace: istio-system + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: profile: empty addonComponents: diff --git a/charts/kubezero-istio/templates/namespace.yaml b/charts/kubezero-istio/templates/namespace.yaml index 58bebec..92cf7fe 100644 --- a/charts/kubezero-istio/templates/namespace.yaml +++ b/charts/kubezero-istio/templates/namespace.yaml @@ -4,3 +4,4 @@ metadata: name: istio-system labels: istio-injection: disabled +{{ include "kubezero-lib.labels" . | indent 4 }} diff --git a/charts/kubezero-istio/update.sh b/charts/kubezero-istio/update.sh index daa565d..199a678 100755 --- a/charts/kubezero-istio/update.sh +++ b/charts/kubezero-istio/update.sh @@ -1,7 +1,7 @@ #!/bin/bash set -ex -ISTIO_VERSION=1.6.5 +ISTIO_VERSION=1.6.7 NAME="istio-$ISTIO_VERSION" URL="https://github.com/istio/istio/releases/download/${ISTIO_VERSION}/istio-${ISTIO_VERSION}-linux-amd64.tar.gz" diff --git a/charts/kubezero-istio/values.yaml b/charts/kubezero-istio/values.yaml index ca353d7..68d5a42 100644 --- a/charts/kubezero-istio/values.yaml +++ b/charts/kubezero-istio/values.yaml @@ -12,4 +12,4 @@ ingress: istio-operator: hub: docker.io/istio - tag: 1.6.5 + tag: 1.6.7 diff --git a/charts/kubezero-lib/templates/_helpers.tpl b/charts/kubezero-lib/templates/_helpers.tpl index b6d75b6..f0de548 100644 --- a/charts/kubezero-lib/templates/_helpers.tpl +++ b/charts/kubezero-lib/templates/_helpers.tpl @@ -2,7 +2,7 @@ Common set of labels */ -}} {{- define "kubezero-lib.labels" -}} -helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +helm.sh/chart: {{ include "kubezero-lib.chart . }} app.kubernetes.io/name: {{ include "kubezero-lib.name" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/part-of: kubezero @@ -27,3 +27,7 @@ Common naming functions {{- end -}} {{- end -}} {{- end -}} + +{{- define "kubezero-lib.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 69fe1ae..70384a5 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -13,7 +13,7 @@ maintainers: - name: Quarky9 dependencies: - name: kubezero-lib - version: ">= 0.1.1" + version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: prometheus-operator version: 9.3.0 -- 2.40.1 From f43e078c573724ccdc001fca158d6e104d9973fa Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 6 Aug 2020 19:07:06 +0100 Subject: [PATCH 50/96] New istio naming schema for virtualservices --- charts/kubezero-argo-cd/values.yaml | 2 +- charts/kubezero-metrics/values.yaml | 4 ++-- deploy/values.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/kubezero-argo-cd/values.yaml b/charts/kubezero-argo-cd/values.yaml index 749a5f2..c02fed2 100644 --- a/charts/kubezero-argo-cd/values.yaml +++ b/charts/kubezero-argo-cd/values.yaml @@ -70,5 +70,5 @@ argo-cd: # argo-cd.istio.enabled -- Deploy Istio VirtualService to expose ArgoCD enabled: false # argo-cd.istio.gateway -- Name of the Istio gateway to add the VirtualService to - gateway: ingressgateway.istio-system.svc.cluster.local + gateway: istio-system/ingressgateway ipBlocks: [] diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 881b91a..4db7875 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -3,13 +3,13 @@ grafana: enabled: false ipBlocks: [] url: "" - gateway: ingressgateway.istio-system.svc.cluster.local + gateway: istio-system/ingressgateway prometheus: istio: enabled: false url: "" - gateway: ingressgateway.istio-system.svc.cluster.local + gateway: istio-system/ingressgateway prometheus-operator: defaultRules: diff --git a/deploy/values.yaml b/deploy/values.yaml index 8bc82cf..ea9abff 100644 --- a/deploy/values.yaml +++ b/deploy/values.yaml @@ -41,4 +41,4 @@ argo-cd: server: {} istio: enabled: true - gateway: private-ingressgateway.istio-system.svc.cluster.local + gateway: istio-system/private-ingressgateway -- 2.40.1 From 36edd7bb7380f2db8fa65b6000be5c958015856d Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 7 Aug 2020 17:02:22 +0100 Subject: [PATCH 51/96] Update all charts to use latest lib --- charts/kubezero-argo-cd/Chart.yaml | 2 +- charts/kubezero-argo-cd/README.md | 6 +++--- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 4 ++-- charts/kubezero-aws-efs-csi-driver/Chart.yaml | 4 ++-- charts/kubezero-calico/Chart.yaml | 4 ++-- charts/kubezero-cert-manager/Chart.yaml | 4 ++-- charts/kubezero-istio/delete_istio_14.sh | 7 +++++++ charts/kubezero-istio/upgrade_from_14.sh | 15 --------------- charts/kubezero-kiam/Chart.yaml | 4 ++-- .../kubezero-local-volume-provisioner/Chart.yaml | 4 ++-- charts/kubezero-metrics/README.md | 6 +++--- charts/kubezero-metrics/delete_kube-prometheus.sh | 9 +++++++++ charts/kubezero/Chart.yaml | 4 ++-- 13 files changed, 37 insertions(+), 36 deletions(-) create mode 100755 charts/kubezero-istio/delete_istio_14.sh delete mode 100755 charts/kubezero-istio/upgrade_from_14.sh create mode 100755 charts/kubezero-metrics/delete_kube-prometheus.sh diff --git a/charts/kubezero-argo-cd/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml index 4ff15ca..45a1a1a 100644 --- a/charts/kubezero-argo-cd/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application name: kubezero-argo-cd -version: 0.3.8 +version: 0.3.9 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-argo-cd/README.md b/charts/kubezero-argo-cd/README.md index 3f3ff38..199bde5 100644 --- a/charts/kubezero-argo-cd/README.md +++ b/charts/kubezero-argo-cd/README.md @@ -2,7 +2,7 @@ kubezero-argo-cd ================ KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application -Current chart version is `0.3.8` +Current chart version is `0.3.9` Source code can be found [here](https://kubezero.com) @@ -11,7 +11,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| | https://argoproj.github.io/argo-helm | argo-cd | 2.6.0 | -| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Chart Values @@ -23,7 +23,7 @@ Source code can be found [here](https://kubezero.com) | argo-cd.dex.enabled | bool | `false` | | | argo-cd.installCRDs | bool | `false` | | | argo-cd.istio.enabled | bool | `false` | Deploy Istio VirtualService to expose ArgoCD | -| argo-cd.istio.gateway | string | `"ingressgateway.istio-system.svc.cluster.local"` | Name of the Istio gateway to add the VirtualService to | +| argo-cd.istio.gateway | string | `"istio-system/ingressgateway"` | Name of the Istio gateway to add the VirtualService to | | argo-cd.istio.ipBlocks | list | `[]` | | | argo-cd.redis.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | argo-cd.redis.tolerations[0].effect | string | `"NoSchedule"` | | diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index c8a3654..fbde476 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.2.0 +version: 0.2.1 appVersion: 0.5.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -18,6 +18,6 @@ maintainers: - name: Quarky9 dependencies: - name: kubezero-lib - version: ">= 0.1.1" + version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-aws-efs-csi-driver/Chart.yaml b/charts/kubezero-aws-efs-csi-driver/Chart.yaml index 15bec19..57809d3 100644 --- a/charts/kubezero-aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-efs-csi-driver/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: kubezero-aws-efs-csi-driver description: KubeZero Umbrella Chart for aws-efs-csi-driver -version: 0.1.0 +version: 0.1.1 appVersion: 1.0.0 kubeVersion: ">=1.16.0-0" home: https://kubezero.com @@ -18,7 +18,7 @@ maintainers: - name: Quarky9 dependencies: - name: kubezero-lib - version: ">= 0.1.1" + version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ # Once they properly update upstream # - name: aws-ebs-csi-driver diff --git a/charts/kubezero-calico/Chart.yaml b/charts/kubezero-calico/Chart.yaml index 625bbac..afc4384 100644 --- a/charts/kubezero-calico/Chart.yaml +++ b/charts/kubezero-calico/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-calico description: KubeZero Umbrella Chart for Calico type: application -version: 0.1.7 +version: 0.1.8 appVersion: 3.15 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -13,6 +13,6 @@ maintainers: - name: Quarky9 dependencies: - name: kubezero-lib - version: ">= 0.1.1" + version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-cert-manager/Chart.yaml b/charts/kubezero-cert-manager/Chart.yaml index c21b7f4..8a15bf2 100644 --- a/charts/kubezero-cert-manager/Chart.yaml +++ b/charts/kubezero-cert-manager/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-cert-manager description: KubeZero Umbrella Chart for cert-manager type: application -version: 0.3.5 +version: 0.3.6 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -12,7 +12,7 @@ maintainers: - name: Quarky9 dependencies: - name: kubezero-lib - version: ">= 0.1.1" + version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: cert-manager version: 0.15.1 diff --git a/charts/kubezero-istio/delete_istio_14.sh b/charts/kubezero-istio/delete_istio_14.sh new file mode 100755 index 0000000..3078179 --- /dev/null +++ b/charts/kubezero-istio/delete_istio_14.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# First delete old 1.4 +kubectl delete -f ingress-gateway.yaml +kubectl delete -f istio.yaml +kubectl delete -f istio-init.yaml +kubectl delete -f namespace.yaml diff --git a/charts/kubezero-istio/upgrade_from_14.sh b/charts/kubezero-istio/upgrade_from_14.sh deleted file mode 100755 index 49467fd..0000000 --- a/charts/kubezero-istio/upgrade_from_14.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# First delete old 1.4 -kubectl delete -f ingress-gateway.yaml -kubectl delete -f istio.yaml -kubectl delete -f istio-init.yaml -kubectl delete -f namespace.yaml - -# Now we need to install the new Istio Operator via KubeZero - -# deploy the CR for 1.6 -kubectl apply -f istio-1.6.yaml - -# add the additiona private ingress gateway as dedicated CR -kubectl apply -f istio-1.6-private-ingress.yaml diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index f78a914..7bc63b8 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.2.6 +version: 0.2.7 appVersion: 3.6 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -13,7 +13,7 @@ maintainers: - name: Quarky9 dependencies: - name: kubezero-lib - version: ">= 0.1.1" + version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: kiam version: 5.8.1 diff --git a/charts/kubezero-local-volume-provisioner/Chart.yaml b/charts/kubezero-local-volume-provisioner/Chart.yaml index 328614c..303887f 100644 --- a/charts/kubezero-local-volume-provisioner/Chart.yaml +++ b/charts/kubezero-local-volume-provisioner/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-local-volume-provisioner description: KubeZero Umbrella Chart for local-static-provisioner type: application -version: 0.0.1 +version: 0.1.0 appVersion: 2.3.4 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -13,6 +13,6 @@ maintainers: - name: Quarky9 dependencies: - name: kubezero-lib - version: ">= 0.1.1" + version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-metrics/README.md b/charts/kubezero-metrics/README.md index 5f42125..62b7040 100644 --- a/charts/kubezero-metrics/README.md +++ b/charts/kubezero-metrics/README.md @@ -12,14 +12,14 @@ Source code can be found [here](https://kubezero.com) |------------|------|---------| | https://kubernetes-charts.storage.googleapis.com/ | prometheus-adapter | 2.5.0 | | https://kubernetes-charts.storage.googleapis.com/ | prometheus-operator | 9.3.0 | -| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Chart Values | Key | Type | Default | Description | |-----|------|---------|-------------| | grafana.istio.enabled | bool | `false` | | -| grafana.istio.gateway | string | `"ingressgateway.istio-system.svc.cluster.local"` | | +| grafana.istio.gateway | string | `"istio-system/ingressgateway"` | | | grafana.istio.ipBlocks | list | `[]` | | | grafana.istio.url | string | `""` | | | prometheus-adapter.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | @@ -93,5 +93,5 @@ Source code can be found [here](https://kubezero.com) | prometheus-operator.prometheusOperator.tolerations[0].effect | string | `"NoSchedule"` | | | prometheus-operator.prometheusOperator.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | prometheus.istio.enabled | bool | `false` | | -| prometheus.istio.gateway | string | `"ingressgateway.istio-system.svc.cluster.local"` | | +| prometheus.istio.gateway | string | `"istio-system/ingressgateway"` | | | prometheus.istio.url | string | `""` | | diff --git a/charts/kubezero-metrics/delete_kube-prometheus.sh b/charts/kubezero-metrics/delete_kube-prometheus.sh new file mode 100755 index 0000000..e7de5b4 --- /dev/null +++ b/charts/kubezero-metrics/delete_kube-prometheus.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +[ -f istio-authorization-policy.yaml ] && kubectl delete -f istio-authorization-policy.yaml +[ -f istio-service.yaml ] && kubectl delete -f istio-service.yaml + +kubectl delete -f manifests +kubectl delete -f manifests/setup + +kubectl delete namespace monitoring diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index 1bd3f94..ef98338 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero ArgoCD Application - Root App of Apps chart of KubeZero type: application -version: 0.4.1 +version: 0.4.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -13,6 +13,6 @@ maintainers: - name: Quarky9 dependencies: - name: kubezero-lib - version: ">= 0.1.1" + version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ kubeVersion: ">= 1.16.0" -- 2.40.1 From eb4b1022917544ea9a7a71488c3508700690f6af Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 10 Aug 2020 12:33:53 +0100 Subject: [PATCH 52/96] Version bump of aws-ebs csi driver to 0.6.0 --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 4 +- .../charts/aws-ebs-csi-driver/Chart.yaml | 4 +- .../aws-ebs-csi-driver/templates/_helpers.tpl | 17 +- .../templates/clusterrole-attacher.yaml | 20 ++ .../templates/clusterrole-provisioner.yaml | 35 +++ .../templates/clusterrole-resizer.yaml | 31 +++ .../clusterrole-snapshot-controller.yaml | 35 +++ .../templates/clusterrole-snapshotter.yaml | 25 ++ .../clusterrolebinding-attacher.yaml | 15 ++ .../clusterrolebinding-provisioner.yaml | 15 ++ .../templates/clusterrolebinding-resizer.yaml | 18 ++ ...lusterrolebinding-snapshot-controller.yaml | 18 ++ .../clusterrolebinding-snapshotter.yaml | 18 ++ .../{deployment.yaml => controller.yaml} | 27 +- .../templates/csidriver.yaml | 2 + .../templates/{daemonset.yaml => node.yaml} | 22 +- .../aws-ebs-csi-driver/templates/rbac.yaml | 251 ------------------ ...le-snapshot-controller-leaderelection.yaml | 15 ++ ...ng-snapshot-controller-leaderelection.yaml | 19 ++ .../serviceaccount-csi-controller.yaml | 15 ++ ...> serviceaccount-snapshot-controller.yaml} | 13 +- .../templates/statefulset.yaml | 8 +- .../charts/aws-ebs-csi-driver/values.yaml | 2 +- charts/kubezero-aws-ebs-csi-driver/update.sh | 2 + 24 files changed, 342 insertions(+), 289 deletions(-) create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml rename charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/{deployment.yaml => controller.yaml} (87%) rename charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/{daemonset.yaml => node.yaml} (83%) delete mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rbac.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml rename charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/{serviceaccount.yaml => serviceaccount-snapshot-controller.yaml} (50%) diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index fbde476..51053ae 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.2.1 -appVersion: 0.5.0 +version: 0.3.0 +appVersion: 0.6.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png sources: diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml index df6d0fc..336d34f 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 -appVersion: "0.5.0" +appVersion: "0.6.0" name: aws-ebs-csi-driver description: A Helm chart for AWS EBS CSI Driver -version: 0.3.0 +version: 0.5.0 kubeVersion: ">=1.13.0-0" home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver sources: diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl index 7fa1330..fdc77c4 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl @@ -35,13 +35,24 @@ Create chart name and version as used by the chart label. Common labels */}} {{- define "aws-ebs-csi-driver.labels" -}} -app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} +{{ include "aws-ebs-csi-driver.selectorLabels" . }} +{{- if ne .Release.Name "kustomize" }} helm.sh/chart: {{ include "aws-ebs-csi-driver.chart" . }} -app.kubernetes.io/instance: {{ .Release.Name }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} +{{- end -}} + +{{/* +Common selector labels +*/}} +{{- define "aws-ebs-csi-driver.selectorLabels" -}} +app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} +{{- if ne .Release.Name "kustomize" }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} {{- end -}} {{/* @@ -53,6 +64,6 @@ Convert the `--extra-volume-tags` command line arg from a map. {{- $noop := printf "%s=%s" $key $value | append $result.pairs | set $result "pairs" -}} {{- end -}} {{- if gt (len $result.pairs) 0 -}} -- --extra-volume-tags={{- join "," $result.pairs -}} +{{- printf "%s=%s" "- --extra-volume-tags" (join "," $result.pairs) -}} {{- end -}} {{- end -}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml new file mode 100644 index 0000000..0b96724 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml @@ -0,0 +1,20 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-attacher-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml new file mode 100644 index 0000000..84f122a --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml @@ -0,0 +1,35 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-provisioner-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml new file mode 100644 index 0000000..5f0c758 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml @@ -0,0 +1,31 @@ +{{- if .Values.enableVolumeResizing }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-resizer-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +{{- end}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml new file mode 100644 index 0000000..ffdb1b7 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml @@ -0,0 +1,35 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-snapshot-controller-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml new file mode 100644 index 0000000..061b565 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml @@ -0,0 +1,25 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-snapshotter-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml new file mode 100644 index 0000000..1888a9b --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-attacher-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-attacher-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml new file mode 100644 index 0000000..0fd5499 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-provisioner-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-provisioner-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml new file mode 100644 index 0000000..b23063a --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml @@ -0,0 +1,18 @@ +{{- if .Values.enableVolumeResizing }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-resizer-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +{{- end}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml new file mode 100644 index 0000000..df4937a --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml @@ -0,0 +1,18 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshot-controller-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: ebs-snapshot-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-snapshot-controller-role + apiGroup: rbac.authorization.k8s.io + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml new file mode 100644 index 0000000..6ebb1d9 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml @@ -0,0 +1,18 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshotter-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml similarity index 87% rename from charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml index 8aeddc1..1e7b68d 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -4,25 +4,26 @@ apiVersion: apps/v1 metadata: name: ebs-csi-controller namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} spec: replicas: {{ .Values.replicaCount }} selector: matchLabels: app: ebs-csi-controller - app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} template: metadata: labels: app: ebs-csi-controller - app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} {{- if .Values.podAnnotations }} annotations: {{ toYaml .Values.podAnnotations | nindent 8 }} {{- end }} spec: nodeSelector: - beta.kubernetes.io/os: linux + kubernetes.io/os: linux + kubernetes.io/arch: amd64 {{- with .Values.nodeSelector }} {{ toYaml . | indent 8 }} {{- end }} @@ -38,12 +39,18 @@ spec: {{- end }} containers: - name: ebs-plugin - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: + {{- if ne .Release.Name "kustomize" }} - controller + {{ else }} + # - {all,controller,node} # specify the driver mode + {{- end }} - --endpoint=$(CSI_ENDPOINT) - {{ include "aws-ebs-csi-driver.extra-volume-tags" . }} + {{- if .Values.extraVolumeTags }} + {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }} + {{- end }} - --logtostderr - --v=5 env: @@ -91,10 +98,8 @@ spec: {{- if .Values.enableVolumeScheduling }} - --feature-gates=Topology=true {{- end}} - {{- if gt ( .Values.replicaCount | int ) 1 }} - --enable-leader-election - --leader-election-type=leases - {{- end}} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -106,10 +111,8 @@ spec: args: - --csi-address=$(ADDRESS) - --v=5 - {{- if gt ( .Values.replicaCount | int ) 1 }} - --leader-election=true - --leader-election-type=leases - {{- end}} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -121,9 +124,7 @@ spec: image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} args: - --csi-address=$(ADDRESS) - {{- if gt ( .Values.replicaCount | int ) 1 }} - --leader-election=true - {{- end}} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml index 6e427fd..1858e39 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml @@ -2,6 +2,8 @@ apiVersion: storage.k8s.io/v1beta1 kind: CSIDriver metadata: name: ebs.csi.aws.com + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} spec: attachRequired: true podInfoOnMount: false diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/daemonset.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml similarity index 83% rename from charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/daemonset.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml index 1e6e817..8eea8a0 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/daemonset.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml @@ -4,24 +4,34 @@ apiVersion: apps/v1 metadata: name: ebs-csi-node namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} spec: selector: matchLabels: app: ebs-csi-node - app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} template: metadata: labels: app: ebs-csi-node - app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} {{- if .Values.node.podAnnotations }} annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} {{- end }} spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate nodeSelector: - beta.kubernetes.io/os: linux + kubernetes.io/os: linux + kubernetes.io/arch: amd64 hostNetwork: true priorityClassName: system-node-critical tolerations: @@ -33,7 +43,7 @@ spec: - name: ebs-plugin securityContext: privileged: true - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} args: - node - --endpoint=$(CSI_ENDPOINT) diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rbac.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rbac.yaml deleted file mode 100644 index 464c648..0000000 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rbac.yaml +++ /dev/null @@ -1,251 +0,0 @@ ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ebs-external-provisioner-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ebs-csi-provisioner-binding -subjects: - - kind: ServiceAccount - name: ebs-csi-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: ebs-external-provisioner-role - apiGroup: rbac.authorization.k8s.io - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ebs-external-attacher-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ebs-csi-attacher-binding -subjects: - - kind: ServiceAccount - name: ebs-csi-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: ebs-external-attacher-role - apiGroup: rbac.authorization.k8s.io - -{{- if .Values.enableVolumeSnapshot }} ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ebs-external-snapshotter-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ebs-csi-snapshotter-binding -subjects: - - kind: ServiceAccount - name: ebs-csi-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: ebs-external-snapshotter-role - apiGroup: rbac.authorization.k8s.io - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ebs-snapshot-controller-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ebs-csi-snapshot-controller-binding -subjects: - - kind: ServiceAccount - name: ebs-snapshot-controller - namespace: kube-system -roleRef: - kind: ClusterRole - name: ebs-snapshot-controller-role - apiGroup: rbac.authorization.k8s.io - ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ebs-snapshot-controller-leaderelection - namespace: kube-system -rules: -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: snapshot-controller-leaderelection - namespace: kube-system -subjects: - - kind: ServiceAccount - name: ebs-snapshot-controller - namespace: kube-system -roleRef: - kind: Role - name: snapshot-controller-leaderelection - apiGroup: rbac.authorization.k8s.io - -{{- end }} - -{{- if .Values.enableVolumeResizing }} ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ebs-external-resizer-role -rules: - # The following rule should be uncommented for plugins that require secrets - # for provisioning. - # - apiGroups: [""] - # resources: ["secrets"] - # verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ebs-csi-resizer-binding -subjects: - - kind: ServiceAccount - name: ebs-csi-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: ebs-external-resizer-role - apiGroup: rbac.authorization.k8s.io -{{- end}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml new file mode 100644 index 0000000..947d241 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml @@ -0,0 +1,15 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-snapshot-controller-leaderelection + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml new file mode 100644 index 0000000..11d8ed4 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml @@ -0,0 +1,19 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: ebs-snapshot-controller + namespace: kube-system +roleRef: + kind: Role + name: snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml new file mode 100644 index 0000000..9e2cf7c --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ebs-csi-controller-sa + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.controller.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + {{- if eq .Release.Name "kustomize" }} + #Enable if EKS IAM for SA is used + #annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::586565787010:role/ebs-csi-role + {{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml similarity index 50% rename from charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml index 95396d6..baa39c8 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml @@ -1,18 +1,13 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ebs-csi-controller-sa - namespace: kube-system - {{- with .Values.serviceAccount.controller.annotations }} - annotations: {{ toYaml . | nindent 4 }} - {{- end }} - +{{- if .Values.enableVolumeSnapshot }} --- apiVersion: v1 kind: ServiceAccount metadata: name: ebs-snapshot-controller namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} {{- with .Values.serviceAccount.snapshot.annotations }} annotations: {{ toYaml . | nindent 4 }} {{- end }} +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml index 01f36b7..b824609 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml @@ -5,21 +5,25 @@ apiVersion: apps/v1 metadata: name: ebs-snapshot-controller namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} spec: serviceName: ebs-snapshot-controller replicas: 1 selector: matchLabels: app: ebs-snapshot-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} template: metadata: labels: app: ebs-snapshot-controller + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} spec: - serviceAccount: ebs-snapshot-controller + serviceAccountName: ebs-snapshot-controller containers: - name: snapshot-controller - image: quay.io/k8scsi/snapshot-controller:v2.0.1 + image: quay.io/k8scsi/snapshot-controller:v2.1.1 args: - --v=5 - --leader-election=false diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml index b899721..4528b83 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml @@ -18,7 +18,7 @@ sidecars: tag: "v1.2.0" snapshotterImage: repository: quay.io/k8scsi/csi-snapshotter - tag: "v2.0.1" + tag: "v2.1.1" livenessProbeImage: repository: quay.io/k8scsi/livenessprobe tag: "v1.1.0" diff --git a/charts/kubezero-aws-ebs-csi-driver/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh index c7b5df7..afbf0d7 100755 --- a/charts/kubezero-aws-ebs-csi-driver/update.sh +++ b/charts/kubezero-aws-ebs-csi-driver/update.sh @@ -1,6 +1,8 @@ #!/bin/bash set -ex +# Upstream doesnt have proper Helm repo yet so we just download latest release and stuff it into charts + REPO="kubernetes-sigs/aws-ebs-csi-driver" LATEST_RELEASE=$(curl -sL -s https://api.github.com/repos/${REPO}/releases | grep '"tag_name":' | cut -d'"' -f4 | grep -v -E "(alpha|beta|rc)" | sort -t"." -k 1,1 -k 2,2 -k 3,3 -k 4,4 | tail -n 1) -- 2.40.1 From e4193dca7327791b9ed10929982517fa4d638990 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 10 Aug 2020 13:20:36 +0100 Subject: [PATCH 53/96] make nodeselector for private ingress configurable --- charts/kubezero-istio/Chart.yaml | 2 +- charts/kubezero-istio/templates/istio-private-ingress.yaml | 4 ++-- charts/kubezero-istio/values.yaml | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/charts/kubezero-istio/Chart.yaml b/charts/kubezero-istio/Chart.yaml index f996613..ac920b6 100644 --- a/charts/kubezero-istio/Chart.yaml +++ b/charts/kubezero-istio/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-istio description: KubeZero Umbrella Chart for Istio type: application -version: 0.2.1 +version: 0.2.2 appVersion: 1.6.7 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-istio/templates/istio-private-ingress.yaml b/charts/kubezero-istio/templates/istio-private-ingress.yaml index c946542..b20f3b9 100644 --- a/charts/kubezero-istio/templates/istio-private-ingress.yaml +++ b/charts/kubezero-istio/templates/istio-private-ingress.yaml @@ -1,4 +1,4 @@ -{{- if .Values.ingress.private }} +{{- if .Values.ingress.private.enabled }} apiVersion: install.istio.io/v1alpha1 kind: IstioOperator metadata: @@ -36,7 +36,7 @@ spec: value: standard {{- if eq .Values.ingress.type "NodePort" }} nodeSelector: - node.kubernetes.io/ingress.private: "31080_31443_30671_30672_31224" + node.kubernetes.io/ingress.private: {{ .Values.ingress.private.nodeSelector }} {{- end }} resources: limits: diff --git a/charts/kubezero-istio/values.yaml b/charts/kubezero-istio/values.yaml index 68d5a42..dc7017a 100644 --- a/charts/kubezero-istio/values.yaml +++ b/charts/kubezero-istio/values.yaml @@ -6,7 +6,9 @@ ingress: autoscaleEnabled: false replicaCount: 2 type: NodePort - private: true + private: + enabled: true + nodeSelector: "31080_31443_30671_30672_31224" #dnsNames: #- "*.example.com" -- 2.40.1 From 51bef938f63e7f48ff3814d939fb16dc48a45bbc Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 10 Aug 2020 13:28:45 +0100 Subject: [PATCH 54/96] Make sure nodeselector is a string --- charts/kubezero-istio/templates/istio-private-ingress.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-istio/templates/istio-private-ingress.yaml b/charts/kubezero-istio/templates/istio-private-ingress.yaml index b20f3b9..08be1e7 100644 --- a/charts/kubezero-istio/templates/istio-private-ingress.yaml +++ b/charts/kubezero-istio/templates/istio-private-ingress.yaml @@ -36,7 +36,7 @@ spec: value: standard {{- if eq .Values.ingress.type "NodePort" }} nodeSelector: - node.kubernetes.io/ingress.private: {{ .Values.ingress.private.nodeSelector }} + node.kubernetes.io/ingress.private: "{{ .Values.ingress.private.nodeSelector }}" {{- end }} resources: limits: -- 2.40.1 From 3464bae762c1c3bb310b9f30b2db8081f5c4c3cf Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 10 Aug 2020 13:53:41 +0100 Subject: [PATCH 55/96] Add logging as default ns to look for servicemonitors --- charts/kubezero-metrics/Chart.yaml | 2 +- charts/kubezero-metrics/values.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 70384a5..d34a0ae 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for prometheus-operator type: application -version: 0.1.2 +version: 0.1.3 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 4db7875..56b485b 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -85,6 +85,7 @@ prometheus-operator: releaseNamespace: true additional: - kube-system + - logging nodeExporter: enabled: true -- 2.40.1 From f703a6d7e16cb1402fe2055d616e43cdc4a07108 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 11 Aug 2020 15:09:48 +0100 Subject: [PATCH 56/96] Add rabbitmq ingress gateway def --- charts/kubezero-istio/Chart.yaml | 2 +- .../templates/ingress-gateway.yaml | 12 + charts/kubezero-logging/.helmignore | 23 + charts/kubezero-logging/Chart.yaml | 21 + charts/kubezero-logging/README.md | 47 + charts/kubezero-logging/README.md.gotmpl | 41 + charts/kubezero-logging/eck/all-in-one.yaml | 3008 ++++++++++++++++ .../kubezero-logging/eck/kustomization.yaml | 6 + charts/kubezero-logging/eck/map-operator.yaml | 14 + charts/kubezero-logging/eck/update.sh | 7 + .../templates/eck-operator.yaml | 3058 +++++++++++++++++ .../templates/elasticsearch.yaml | 83 + .../templates/es-servicemonitor.yaml | 20 + .../kubezero-logging/templates/es-users.yaml | 12 + .../templates/istio-virtualservice.yaml | 18 + charts/kubezero-logging/templates/kibana.yaml | 35 + charts/kubezero-logging/values-test.yaml | 26 + charts/kubezero-logging/values.yaml | 27 + 18 files changed, 6459 insertions(+), 1 deletion(-) create mode 100644 charts/kubezero-logging/.helmignore create mode 100644 charts/kubezero-logging/Chart.yaml create mode 100644 charts/kubezero-logging/README.md create mode 100644 charts/kubezero-logging/README.md.gotmpl create mode 100644 charts/kubezero-logging/eck/all-in-one.yaml create mode 100644 charts/kubezero-logging/eck/kustomization.yaml create mode 100644 charts/kubezero-logging/eck/map-operator.yaml create mode 100755 charts/kubezero-logging/eck/update.sh create mode 100644 charts/kubezero-logging/templates/eck-operator.yaml create mode 100644 charts/kubezero-logging/templates/elasticsearch.yaml create mode 100644 charts/kubezero-logging/templates/es-servicemonitor.yaml create mode 100644 charts/kubezero-logging/templates/es-users.yaml create mode 100644 charts/kubezero-logging/templates/istio-virtualservice.yaml create mode 100644 charts/kubezero-logging/templates/kibana.yaml create mode 100644 charts/kubezero-logging/values-test.yaml create mode 100644 charts/kubezero-logging/values.yaml diff --git a/charts/kubezero-istio/Chart.yaml b/charts/kubezero-istio/Chart.yaml index ac920b6..e4ad7c1 100644 --- a/charts/kubezero-istio/Chart.yaml +++ b/charts/kubezero-istio/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-istio description: KubeZero Umbrella Chart for Istio type: application -version: 0.2.2 +version: 0.2.3 appVersion: 1.6.7 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-istio/templates/ingress-gateway.yaml b/charts/kubezero-istio/templates/ingress-gateway.yaml index 9d90242..f183be0 100644 --- a/charts/kubezero-istio/templates/ingress-gateway.yaml +++ b/charts/kubezero-istio/templates/ingress-gateway.yaml @@ -60,6 +60,18 @@ spec: privateKey: /etc/istio/ingressgateway-certs/tls.key serverCertificate: /etc/istio/ingressgateway-certs/tls.crt credentialName: public-ingress-cert + - port: + number: 5672 + name: amqp + protocol: TCP + hosts: + - "*" + - port: + number: 5671 + name: amqps + protocol: TCP + hosts: + - "*" - port: number: 24224 name: fluentd-forward diff --git a/charts/kubezero-logging/.helmignore b/charts/kubezero-logging/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/kubezero-logging/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml new file mode 100644 index 0000000..22e5aaf --- /dev/null +++ b/charts/kubezero-logging/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: kubezero-logging +description: KubeZero Umbrella Chart for complete EFK stack +type: application +version: 0.0.1 +appVersion: 1.2.0 +home: https://kubezero.com +icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png +keywords: + - kubezero + - elasticsearch + - kibana + - fluentd + - fluent-bit +maintainers: + - name: Quarky9 +dependencies: + - name: kubezero-lib + version: ">= 0.1.3" + repository: https://zero-down-time.github.io/kubezero/ +kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md new file mode 100644 index 0000000..ad980cc --- /dev/null +++ b/charts/kubezero-logging/README.md @@ -0,0 +1,47 @@ +kubezero-logging +================ +KubeZero Umbrella Chart for complete EFK stack + +Current chart version is `0.0.1` + +Source code can be found [here](https://kubezero.com) + +## Chart Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | + +## Changes from upstream +### ECK +- Operator mapped to controller nodes + +### ElasticSearch + +- Installed Plugins: + - repository-s3 + - elasticsearch-prometheus-exporter + +- [Cross AZ Zone awareness](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-advanced-node-scheduling.html#k8s-availability-zone-awareness) is implemented via nodeSets + +## Manual tasks ATM + +- Install index template +- setup Kibana +- create `logstash-*` Index Pattern + +## Chart Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| es.replicas | int | `2` | | +| es.storage.class | string | `"local-sc-xfs"` | | +| es.storage.size | string | `"16Gi"` | | +| fullnameOverride | string | `"logging"` | | +| kibana.replicas | int | `1` | | +| version | string | `"7.6.0"` | | + +## Resources: + +- https://www.elastic.co/downloads/elastic-cloud-kubernetes +- https://github.com/elastic/cloud-on-k8s diff --git a/charts/kubezero-logging/README.md.gotmpl b/charts/kubezero-logging/README.md.gotmpl new file mode 100644 index 0000000..ae55a2b --- /dev/null +++ b/charts/kubezero-logging/README.md.gotmpl @@ -0,0 +1,41 @@ +{{ template "chart.header" . }} +{{ template "chart.description" . }} + +{{ template "chart.versionLine" . }} + +{{ template "chart.sourceLinkLine" . }} + +{{ template "chart.requirementsSection" . }} + +## Changes from upstream +### ECK +- Operator mapped to controller nodes + +### ES + +- SSL disabled ( Todo: provide cluster certs and setup Kibana/Fluentd to use https incl. client certs ) + +- Installed Plugins: + - repository-s3 + - elasticsearch-prometheus-exporter + +- [Cross AZ Zone awareness](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-advanced-node-scheduling.html#k8s-availability-zone-awareness) is implemented via nodeSets + +### Kibana + +- increased timeout to ES to 3 minutes + + +## Manual tasks ATM + +- Install index template +- setup Kibana +- create `logstash-*` Index Pattern + + +{{ template "chart.valuesSection" . }} + +## Resources: + +- https://www.elastic.co/downloads/elastic-cloud-kubernetes +- https://github.com/elastic/cloud-on-k8s diff --git a/charts/kubezero-logging/eck/all-in-one.yaml b/charts/kubezero-logging/eck/all-in-one.yaml new file mode 100644 index 0000000..40cada8 --- /dev/null +++ b/charts/kubezero-logging/eck/all-in-one.yaml @@ -0,0 +1,3008 @@ +--- +# Source: crds/all-crds.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: apmservers.apm.k8s.elastic.co +spec: + additionalPrinterColumns: + - JSONPath: .status.health + name: health + type: string + - JSONPath: .status.availableNodes + description: Available nodes + name: nodes + type: integer + - JSONPath: .spec.version + description: APM version + name: version + type: string + - JSONPath: .metadata.creationTimestamp + name: age + type: date + group: apm.k8s.elastic.co + names: + categories: + - elastic + kind: ApmServer + listKind: ApmServerList + plural: apmservers + shortNames: + - apm + singular: apmserver + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: ApmServer represents an APM Server resource in a Kubernetes cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ApmServerSpec holds the specification of an APM Server. + properties: + config: + description: 'Config holds the APM Server configuration. See: https://www.elastic.co/guide/en/apm/server/current/configuring-howto-apm-server.html' + type: object + count: + description: Count of APM Server instances to deploy. + format: int32 + type: integer + elasticsearchRef: + description: ElasticsearchRef is a reference to the output Elasticsearch + cluster running in the same Kubernetes cluster. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + required: + - name + type: object + http: + description: HTTP holds the HTTP layer configuration for the APM Server + resource. + properties: + service: + description: Service defines the template for the associated Kubernetes + Service object. + properties: + metadata: + description: ObjectMeta is the metadata of the service. The + name and namespace provided here are managed by ECK and will + be ignored. + type: object + spec: + description: Spec is the specification of the service. + properties: + clusterIP: + description: 'clusterIP is the IP address of the service + and is usually assigned randomly by the master. If an + address is specified manually and is not in use by others, + it will be allocated to the service; otherwise, creation + of the service will fail. This field can not be changed + through updates. Valid values are "None", empty string + (""), or a valid IP address. "None" can be specified for + headless services when proxying is not required. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this + service. These IPs are not managed by Kubernetes. The + user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external + load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that + kubedns or equivalent will return as a CNAME record for + this service. No proxying will be involved. Must be a + valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires Type to be ExternalName. + type: string + externalTrafficPolicy: + description: externalTrafficPolicy denotes if this Service + desires to route external traffic to node-local or cluster-wide + endpoints. "Local" preserves the client source IP and + avoids a second hop for LoadBalancer and Nodeport type + services, but risks potentially imbalanced traffic spreading. + "Cluster" obscures the client source IP and may cause + a second hop to another node, but should have good overall + load-spreading. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. If not specified, HealthCheckNodePort + is created by the service api backend with the allocated + nodePort. Will use user-specified nodePort value if specified + by the client. Only effects when Type is set to LoadBalancer + and ExternalTrafficPolicy is set to Local. + format: int32 + type: integer + ipFamily: + description: ipFamily specifies whether this Service has + a preference for a particular IP family (e.g. IPv4 vs. + IPv6). If a specific IP family is requested, the clusterIP + field will be allocated from that family, if it is available + in the cluster. If no IP family is requested, the cluster's + primary IP family will be used. Other IP fields (loadBalancerIP, + loadBalancerSourceRanges, externalIPs) and controllers + which allocate external load-balancers should use the + same IP family. Endpoints for this Service will be of + this family. This field is immutable after creation. + Assigning a ServiceIPFamily not available in the cluster + (e.g. IPv6 in IPv4 only cluster) is an error condition + and will fail during clusterIP assignment. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer + LoadBalancer will get created with the IP specified in + this field. This feature depends on whether the underlying + cloud-provider supports specifying the loadBalancerIP + when a load balancer is created. This field will be ignored + if the cloud-provider does not support the feature.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client + IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this + service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field + in the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this + service is exposed when type=NodePort or LoadBalancer. + Usually assigned by the system. If specified, it + will be allocated to the service if unused or else + creation of the service will fail. Default is to + auto-allocate a port if the ServiceType of this + Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this + service. + format: int32 + type: integer + protocol: + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access + on the pods targeted by the service. Number must + be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored + for services with clusterIP=None, and should be + omitted or set equal to the ''port'' field. More + info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + required: + - port + type: object + type: array + publishNotReadyAddresses: + description: publishNotReadyAddresses, when set to true, + indicates that DNS implementations must publish the notReadyAddresses + of subsets for the Endpoints associated with the Service. + The default value is false. The primary use case for setting + this field is to use a StatefulSet's Headless Service + to propagate SRV records for its Pods without respect + to their readiness for purpose of peer discovery. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: + https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of + Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + topologyKeys: + description: topologyKeys is a preference-order list of + topology keys which implementations of services should + use to preferentially sort endpoints when accessing this + Service, it can not be used at the same time as externalTrafficPolicy=Local. + Topology keys must be valid label keys and at most 16 + keys may be specified. Endpoints are chosen based on the + first topology key with available backends. If this field + is specified and all entries have no backends that match + the topology of the client, the service has no backends + for that client and connections should fail. The special + value "*" may be used to mean "any topology". This catch-all + value, if used, only makes sense as the last value in + the list. If this is not specified or empty, no topology + constraints will be applied. + items: + type: string + type: array + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ExternalName" + maps to the specified externalName. "ClusterIP" allocates + a cluster-internal IP address for load-balancing to endpoints. + Endpoints are determined by the selector or if that is + not specified, by manual construction of an Endpoints + object. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints + rather than a stable IP. "NodePort" builds on ClusterIP + and allocates a port on every node which routes to the + clusterIP. "LoadBalancer" builds on NodePort and creates + an external load-balancer (if supported in the current + cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + tls: + description: TLS defines options for configuring TLS for HTTP. + properties: + certificate: + description: "Certificate is a reference to a Kubernetes secret + that contains the certificate and private key for enabling + TLS. The referenced secret should contain the following: \n + - `ca.crt`: The certificate authority (optional). - `tls.crt`: + The certificate (or a chain). - `tls.key`: The private key + to the first certificate in the certificate chain." + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + selfSignedCertificate: + description: SelfSignedCertificate allows configuring the self-signed + certificate generated by the operator. + properties: + disabled: + description: Disabled indicates that the provisioning of + the self-signed certifcate should be disabled. + type: boolean + subjectAltNames: + description: SubjectAlternativeNames is a list of SANs to + include in the generated HTTP TLS certificate. + items: + description: SubjectAlternativeName represents a SAN entry + in a x509 certificate. + properties: + dns: + description: DNS is the DNS name of the subject. + type: string + ip: + description: IP is the IP address of the subject. + type: string + type: object + type: array + type: object + type: object + type: object + image: + description: Image is the APM Server Docker image to deploy. + type: string + kibanaRef: + description: KibanaRef is a reference to a Kibana instance running in + the same Kubernetes cluster. It allows APM agent central configuration + management in Kibana. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + required: + - name + type: object + podTemplate: + description: PodTemplate provides customisation options (labels, annotations, + affinity rules, resource requests, and so on) for the APM Server pods. + type: object + secureSettings: + description: SecureSettings is a list of references to Kubernetes secrets + containing sensitive configuration options for APM Server. + items: + description: SecretSource defines a data source based on a Kubernetes + Secret. + properties: + entries: + description: Entries define how to project each key-value pair + in the secret to filesystem paths. If not defined, all keys + will be projected to similarly named paths in the filesystem. + If defined, only the specified keys will be projected to the + corresponding paths. + items: + description: KeyToPath defines how to map a key in a Secret + object to a filesystem path. + properties: + key: + description: Key is the key contained in the secret. + type: string + path: + description: Path is the relative file path to map the key + to. Path must not be an absolute file path and must not + contain any ".." components. + type: string + required: + - key + type: object + type: array + secretName: + description: SecretName is the name of the secret. + type: string + required: + - secretName + type: object + type: array + serviceAccountName: + description: ServiceAccountName is used to check access from the current + resource to a resource (eg. Elasticsearch) in a different namespace. + Can only be used if ECK is enforcing RBAC on references. + type: string + version: + description: Version of the APM Server. + type: string + required: + - version + type: object + status: + description: ApmServerStatus defines the observed state of ApmServer + properties: + availableNodes: + format: int32 + type: integer + elasticsearchAssociationStatus: + description: ElasticsearchAssociationStatus is the status of any auto-linking + to Elasticsearch clusters. + type: string + health: + description: ApmServerHealth expresses the status of the Apm Server + instances. + type: string + kibanaAssociationStatus: + description: KibanaAssociationStatus is the status of any auto-linking + to Kibana. + type: string + secretTokenSecret: + description: SecretTokenSecretName is the name of the Secret that contains + the secret token + type: string + service: + description: ExternalService is the name of the service the agents should + connect to. + type: string + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true + - name: v1beta1 + served: true + storage: false + - name: v1alpha1 + served: false + storage: false +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: beats.beat.k8s.elastic.co +spec: + additionalPrinterColumns: + - JSONPath: .status.health + name: health + type: string + - JSONPath: .status.availableNodes + description: Available nodes + name: available + type: integer + - JSONPath: .status.expectedNodes + description: Expected nodes + name: expected + type: integer + - JSONPath: .spec.type + description: Beat type + name: type + type: string + - JSONPath: .spec.version + description: Beat version + name: version + type: string + - JSONPath: .metadata.creationTimestamp + name: age + type: date + group: beat.k8s.elastic.co + names: + categories: + - elastic + kind: Beat + listKind: BeatList + plural: beats + shortNames: + - beat + singular: beat + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: Beat is the Schema for the Beats API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BeatSpec defines the desired state of a Beat. + properties: + config: + description: Config holds the Beat configuration. At most one of [`Config`, + `ConfigRef`] can be specified. + type: object + configRef: + description: ConfigRef contains a reference to an existing Kubernetes + Secret holding the Beat configuration. Beat settings must be specified + as yaml, under a single "beat.yml" entry. At most one of [`Config`, + `ConfigRef`] can be specified. + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + daemonSet: + description: DaemonSet specifies the Beat should be deployed as a DaemonSet, + and allows providing its spec. Cannot be used along with `deployment`. + If both are absent a default for the Type is used. + properties: {} + type: object + deployment: + description: Deployment specifies the Beat should be deployed as a Deployment, + and allows providing its spec. Cannot be used along with `daemonSet`. + If both are absent a default for the Type is used. + properties: + replicas: + format: int32 + type: integer + type: object + elasticsearchRef: + description: ElasticsearchRef is a reference to an Elasticsearch cluster + running in the same Kubernetes cluster. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + required: + - name + type: object + image: + description: Image is the Beat Docker image to deploy. Version and Type + have to match the Beat in the image. + type: string + kibanaRef: + description: KibanaRef is a reference to a Kibana instance running in + the same Kubernetes cluster. It allows automatic setup of dashboards + and visualizations. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + required: + - name + type: object + secureSettings: + description: SecureSettings is a list of references to Kubernetes Secrets + containing sensitive configuration options for the Beat. Secrets data + can be then referenced in the Beat config using the Secret's keys + or as specified in `Entries` field of each SecureSetting. + items: + description: SecretSource defines a data source based on a Kubernetes + Secret. + properties: + entries: + description: Entries define how to project each key-value pair + in the secret to filesystem paths. If not defined, all keys + will be projected to similarly named paths in the filesystem. + If defined, only the specified keys will be projected to the + corresponding paths. + items: + description: KeyToPath defines how to map a key in a Secret + object to a filesystem path. + properties: + key: + description: Key is the key contained in the secret. + type: string + path: + description: Path is the relative file path to map the key + to. Path must not be an absolute file path and must not + contain any ".." components. + type: string + required: + - key + type: object + type: array + secretName: + description: SecretName is the name of the secret. + type: string + required: + - secretName + type: object + type: array + serviceAccountName: + description: ServiceAccountName is used to check access from the current + resource to Elasticsearch resource in a different namespace. Can only + be used if ECK is enforcing RBAC on references. + type: string + type: + description: Type is the type of the Beat to deploy (filebeat, metricbeat, + heartbeat, auditbeat, journalbeat, packetbeat, etc.). Any string can + be used, but well-known types will have the image field defaulted + and have the appropriate Elasticsearch roles created automatically. + It also allows for dashboard setup when combined with a `KibanaRef`. + maxLength: 20 + pattern: '[a-zA-Z0-9-]+' + type: string + version: + description: Version of the Beat. + type: string + required: + - type + - version + type: object + status: + description: BeatStatus defines the observed state of a Beat. + properties: + availableNodes: + format: int32 + type: integer + elasticsearchAssociationStatus: + description: AssociationStatus is the status of an association resource. + type: string + expectedNodes: + format: int32 + type: integer + health: + type: string + kibanaAssociationStatus: + description: AssociationStatus is the status of an association resource. + type: string + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: elasticsearches.elasticsearch.k8s.elastic.co +spec: + additionalPrinterColumns: + - JSONPath: .status.health + name: health + type: string + - JSONPath: .status.availableNodes + description: Available nodes + name: nodes + type: integer + - JSONPath: .spec.version + description: Elasticsearch version + name: version + type: string + - JSONPath: .status.phase + name: phase + type: string + - JSONPath: .metadata.creationTimestamp + name: age + type: date + group: elasticsearch.k8s.elastic.co + names: + categories: + - elastic + kind: Elasticsearch + listKind: ElasticsearchList + plural: elasticsearches + shortNames: + - es + singular: elasticsearch + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: Elasticsearch represents an Elasticsearch resource in a Kubernetes + cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ElasticsearchSpec holds the specification of an Elasticsearch + cluster. + properties: + auth: + description: Auth contains user authentication and authorization security + settings for Elasticsearch. + properties: + fileRealm: + description: FileRealm to propagate to the Elasticsearch cluster. + items: + description: FileRealmSource references users to create in the + Elasticsearch cluster. + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + type: array + roles: + description: Roles to propagate to the Elasticsearch cluster. + items: + description: RoleSource references roles to create in the Elasticsearch + cluster. + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + type: array + type: object + http: + description: HTTP holds HTTP layer settings for Elasticsearch. + properties: + service: + description: Service defines the template for the associated Kubernetes + Service object. + properties: + metadata: + description: ObjectMeta is the metadata of the service. The + name and namespace provided here are managed by ECK and will + be ignored. + type: object + spec: + description: Spec is the specification of the service. + properties: + clusterIP: + description: 'clusterIP is the IP address of the service + and is usually assigned randomly by the master. If an + address is specified manually and is not in use by others, + it will be allocated to the service; otherwise, creation + of the service will fail. This field can not be changed + through updates. Valid values are "None", empty string + (""), or a valid IP address. "None" can be specified for + headless services when proxying is not required. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this + service. These IPs are not managed by Kubernetes. The + user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external + load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that + kubedns or equivalent will return as a CNAME record for + this service. No proxying will be involved. Must be a + valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires Type to be ExternalName. + type: string + externalTrafficPolicy: + description: externalTrafficPolicy denotes if this Service + desires to route external traffic to node-local or cluster-wide + endpoints. "Local" preserves the client source IP and + avoids a second hop for LoadBalancer and Nodeport type + services, but risks potentially imbalanced traffic spreading. + "Cluster" obscures the client source IP and may cause + a second hop to another node, but should have good overall + load-spreading. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. If not specified, HealthCheckNodePort + is created by the service api backend with the allocated + nodePort. Will use user-specified nodePort value if specified + by the client. Only effects when Type is set to LoadBalancer + and ExternalTrafficPolicy is set to Local. + format: int32 + type: integer + ipFamily: + description: ipFamily specifies whether this Service has + a preference for a particular IP family (e.g. IPv4 vs. + IPv6). If a specific IP family is requested, the clusterIP + field will be allocated from that family, if it is available + in the cluster. If no IP family is requested, the cluster's + primary IP family will be used. Other IP fields (loadBalancerIP, + loadBalancerSourceRanges, externalIPs) and controllers + which allocate external load-balancers should use the + same IP family. Endpoints for this Service will be of + this family. This field is immutable after creation. + Assigning a ServiceIPFamily not available in the cluster + (e.g. IPv6 in IPv4 only cluster) is an error condition + and will fail during clusterIP assignment. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer + LoadBalancer will get created with the IP specified in + this field. This feature depends on whether the underlying + cloud-provider supports specifying the loadBalancerIP + when a load balancer is created. This field will be ignored + if the cloud-provider does not support the feature.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client + IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this + service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field + in the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this + service is exposed when type=NodePort or LoadBalancer. + Usually assigned by the system. If specified, it + will be allocated to the service if unused or else + creation of the service will fail. Default is to + auto-allocate a port if the ServiceType of this + Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this + service. + format: int32 + type: integer + protocol: + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access + on the pods targeted by the service. Number must + be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored + for services with clusterIP=None, and should be + omitted or set equal to the ''port'' field. More + info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + required: + - port + type: object + type: array + publishNotReadyAddresses: + description: publishNotReadyAddresses, when set to true, + indicates that DNS implementations must publish the notReadyAddresses + of subsets for the Endpoints associated with the Service. + The default value is false. The primary use case for setting + this field is to use a StatefulSet's Headless Service + to propagate SRV records for its Pods without respect + to their readiness for purpose of peer discovery. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: + https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of + Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + topologyKeys: + description: topologyKeys is a preference-order list of + topology keys which implementations of services should + use to preferentially sort endpoints when accessing this + Service, it can not be used at the same time as externalTrafficPolicy=Local. + Topology keys must be valid label keys and at most 16 + keys may be specified. Endpoints are chosen based on the + first topology key with available backends. If this field + is specified and all entries have no backends that match + the topology of the client, the service has no backends + for that client and connections should fail. The special + value "*" may be used to mean "any topology". This catch-all + value, if used, only makes sense as the last value in + the list. If this is not specified or empty, no topology + constraints will be applied. + items: + type: string + type: array + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ExternalName" + maps to the specified externalName. "ClusterIP" allocates + a cluster-internal IP address for load-balancing to endpoints. + Endpoints are determined by the selector or if that is + not specified, by manual construction of an Endpoints + object. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints + rather than a stable IP. "NodePort" builds on ClusterIP + and allocates a port on every node which routes to the + clusterIP. "LoadBalancer" builds on NodePort and creates + an external load-balancer (if supported in the current + cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + tls: + description: TLS defines options for configuring TLS for HTTP. + properties: + certificate: + description: "Certificate is a reference to a Kubernetes secret + that contains the certificate and private key for enabling + TLS. The referenced secret should contain the following: \n + - `ca.crt`: The certificate authority (optional). - `tls.crt`: + The certificate (or a chain). - `tls.key`: The private key + to the first certificate in the certificate chain." + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + selfSignedCertificate: + description: SelfSignedCertificate allows configuring the self-signed + certificate generated by the operator. + properties: + disabled: + description: Disabled indicates that the provisioning of + the self-signed certifcate should be disabled. + type: boolean + subjectAltNames: + description: SubjectAlternativeNames is a list of SANs to + include in the generated HTTP TLS certificate. + items: + description: SubjectAlternativeName represents a SAN entry + in a x509 certificate. + properties: + dns: + description: DNS is the DNS name of the subject. + type: string + ip: + description: IP is the IP address of the subject. + type: string + type: object + type: array + type: object + type: object + type: object + image: + description: Image is the Elasticsearch Docker image to deploy. + type: string + nodeSets: + description: 'NodeSets allow specifying groups of Elasticsearch nodes + sharing the same configuration and Pod templates. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-orchestration.html' + items: + description: NodeSet is the specification for a group of Elasticsearch + nodes sharing the same configuration and a Pod template. + properties: + config: + description: Config holds the Elasticsearch configuration. + type: object + count: + description: Count of Elasticsearch nodes to deploy. + format: int32 + minimum: 1 + type: integer + name: + description: Name of this set of nodes. Becomes a part of the + Elasticsearch node.name setting. + maxLength: 23 + pattern: '[a-zA-Z0-9-]+' + type: string + podTemplate: + description: PodTemplate provides customisation options (labels, + annotations, affinity rules, resource requests, and so on) for + the Pods belonging to this NodeSet. + type: object + volumeClaimTemplates: + description: 'VolumeClaimTemplates is a list of persistent volume + claims to be used by each Pod in this NodeSet. Every claim in + this list must have a matching volumeMount in one of the containers + defined in the PodTemplate. Items defined here take precedence + over any default claims added by the operator with the same + name. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-volume-claim-templates.html' + items: + description: PersistentVolumeClaim is a user's request for and + claim to a persistent volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + type: object + spec: + description: 'Spec defines the desired characteristics of + a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: This field requires the VolumeSnapshotDataSource + alpha feature gate to be enabled and currently VolumeSnapshot + is the only supported data source. If the provisioner + can support VolumeSnapshot data source, it will create + a new volume and data will be restored to the volume + at the same time. If the provisioner does not support + VolumeSnapshot data source, volume will not be created + and the failure will be reported as an event. In the + future, we plan to support more data source types + and the behavior of the provisioner may change. + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + selector: + description: A label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required by the + claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem is implied + when not included in claim spec. This is a beta feature. + type: string + volumeName: + description: VolumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + status: + description: 'Status represents the current information/status + of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the actual access + modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: Represents the actual resources of the + underlying volume. + type: object + conditions: + description: Current Condition of persistent volume + claim. If underlying persistent volume is being resized + then the Condition will be set to 'ResizeStarted'. + items: + description: PersistentVolumeClaimCondition contails + details about state of pvc + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned + from one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating + details about last transition. + type: string + reason: + description: Unique, this should be a short, machine + understandable string that gives the reason + for condition's last transition. If it reports + "ResizeStarted" that means the underlying persistent + volume is being resized. + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType + is a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: Phase represents the current phase of PersistentVolumeClaim. + type: string + type: object + type: object + type: array + required: + - count + - name + type: object + minItems: 1 + type: array + podDisruptionBudget: + description: PodDisruptionBudget provides access to the default pod + disruption budget for the Elasticsearch cluster. The default budget + selects all cluster pods and sets `maxUnavailable` to 1. To disable, + set `PodDisruptionBudget` to the empty value (`{}` in YAML). + properties: + metadata: + description: ObjectMeta is the metadata of the PDB. The name and + namespace provided here are managed by ECK and will be ignored. + type: object + spec: + description: Spec is the specification of the PDB. + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: An eviction is allowed if at most "maxUnavailable" + pods selected by "selector" are unavailable after the eviction, + i.e. even in absence of the evicted pod. For example, one + can prevent all voluntary evictions by specifying 0. This + is a mutually exclusive setting with "minAvailable". + minAvailable: + anyOf: + - type: integer + - type: string + description: An eviction is allowed if at least "minAvailable" + pods selected by "selector" will still be available after + the eviction, i.e. even in the absence of the evicted pod. So + for example you can prevent all voluntary evictions by specifying + "100%". + selector: + description: Label query over pods whose evictions are managed + by the disruption budget. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + type: object + type: object + remoteClusters: + description: RemoteClusters enables you to establish uni-directional + connections to a remote Elasticsearch cluster. + items: + description: RemoteCluster declares a remote Elasticsearch cluster + connection. + properties: + elasticsearchRef: + description: ElasticsearchRef is a reference to an Elasticsearch + cluster running within the same k8s cluster. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, + defaults to the current namespace. + type: string + required: + - name + type: object + name: + description: Name is the name of the remote cluster as it is set + in the Elasticsearch settings. The name is expected to be unique + for each remote clusters. + minLength: 1 + type: string + required: + - name + type: object + type: array + secureSettings: + description: SecureSettings is a list of references to Kubernetes secrets + containing sensitive configuration options for Elasticsearch. + items: + description: SecretSource defines a data source based on a Kubernetes + Secret. + properties: + entries: + description: Entries define how to project each key-value pair + in the secret to filesystem paths. If not defined, all keys + will be projected to similarly named paths in the filesystem. + If defined, only the specified keys will be projected to the + corresponding paths. + items: + description: KeyToPath defines how to map a key in a Secret + object to a filesystem path. + properties: + key: + description: Key is the key contained in the secret. + type: string + path: + description: Path is the relative file path to map the key + to. Path must not be an absolute file path and must not + contain any ".." components. + type: string + required: + - key + type: object + type: array + secretName: + description: SecretName is the name of the secret. + type: string + required: + - secretName + type: object + type: array + serviceAccountName: + description: ServiceAccountName is used to check access from the current + resource to a resource (eg. a remote Elasticsearch cluster) in a different + namespace. Can only be used if ECK is enforcing RBAC on references. + type: string + transport: + description: Transport holds transport layer settings for Elasticsearch. + properties: + service: + description: Service defines the template for the associated Kubernetes + Service object. + properties: + metadata: + description: ObjectMeta is the metadata of the service. The + name and namespace provided here are managed by ECK and will + be ignored. + type: object + spec: + description: Spec is the specification of the service. + properties: + clusterIP: + description: 'clusterIP is the IP address of the service + and is usually assigned randomly by the master. If an + address is specified manually and is not in use by others, + it will be allocated to the service; otherwise, creation + of the service will fail. This field can not be changed + through updates. Valid values are "None", empty string + (""), or a valid IP address. "None" can be specified for + headless services when proxying is not required. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this + service. These IPs are not managed by Kubernetes. The + user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external + load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that + kubedns or equivalent will return as a CNAME record for + this service. No proxying will be involved. Must be a + valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires Type to be ExternalName. + type: string + externalTrafficPolicy: + description: externalTrafficPolicy denotes if this Service + desires to route external traffic to node-local or cluster-wide + endpoints. "Local" preserves the client source IP and + avoids a second hop for LoadBalancer and Nodeport type + services, but risks potentially imbalanced traffic spreading. + "Cluster" obscures the client source IP and may cause + a second hop to another node, but should have good overall + load-spreading. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. If not specified, HealthCheckNodePort + is created by the service api backend with the allocated + nodePort. Will use user-specified nodePort value if specified + by the client. Only effects when Type is set to LoadBalancer + and ExternalTrafficPolicy is set to Local. + format: int32 + type: integer + ipFamily: + description: ipFamily specifies whether this Service has + a preference for a particular IP family (e.g. IPv4 vs. + IPv6). If a specific IP family is requested, the clusterIP + field will be allocated from that family, if it is available + in the cluster. If no IP family is requested, the cluster's + primary IP family will be used. Other IP fields (loadBalancerIP, + loadBalancerSourceRanges, externalIPs) and controllers + which allocate external load-balancers should use the + same IP family. Endpoints for this Service will be of + this family. This field is immutable after creation. + Assigning a ServiceIPFamily not available in the cluster + (e.g. IPv6 in IPv4 only cluster) is an error condition + and will fail during clusterIP assignment. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer + LoadBalancer will get created with the IP specified in + this field. This feature depends on whether the underlying + cloud-provider supports specifying the loadBalancerIP + when a load balancer is created. This field will be ignored + if the cloud-provider does not support the feature.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client + IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this + service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field + in the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this + service is exposed when type=NodePort or LoadBalancer. + Usually assigned by the system. If specified, it + will be allocated to the service if unused or else + creation of the service will fail. Default is to + auto-allocate a port if the ServiceType of this + Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this + service. + format: int32 + type: integer + protocol: + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access + on the pods targeted by the service. Number must + be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored + for services with clusterIP=None, and should be + omitted or set equal to the ''port'' field. More + info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + required: + - port + type: object + type: array + publishNotReadyAddresses: + description: publishNotReadyAddresses, when set to true, + indicates that DNS implementations must publish the notReadyAddresses + of subsets for the Endpoints associated with the Service. + The default value is false. The primary use case for setting + this field is to use a StatefulSet's Headless Service + to propagate SRV records for its Pods without respect + to their readiness for purpose of peer discovery. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: + https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of + Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + topologyKeys: + description: topologyKeys is a preference-order list of + topology keys which implementations of services should + use to preferentially sort endpoints when accessing this + Service, it can not be used at the same time as externalTrafficPolicy=Local. + Topology keys must be valid label keys and at most 16 + keys may be specified. Endpoints are chosen based on the + first topology key with available backends. If this field + is specified and all entries have no backends that match + the topology of the client, the service has no backends + for that client and connections should fail. The special + value "*" may be used to mean "any topology". This catch-all + value, if used, only makes sense as the last value in + the list. If this is not specified or empty, no topology + constraints will be applied. + items: + type: string + type: array + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ExternalName" + maps to the specified externalName. "ClusterIP" allocates + a cluster-internal IP address for load-balancing to endpoints. + Endpoints are determined by the selector or if that is + not specified, by manual construction of an Endpoints + object. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints + rather than a stable IP. "NodePort" builds on ClusterIP + and allocates a port on every node which routes to the + clusterIP. "LoadBalancer" builds on NodePort and creates + an external load-balancer (if supported in the current + cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + type: object + updateStrategy: + description: UpdateStrategy specifies how updates to the cluster should + be performed. + properties: + changeBudget: + description: ChangeBudget defines the constraints to consider when + applying changes to the Elasticsearch cluster. + properties: + maxSurge: + description: MaxSurge is the maximum number of new pods that + can be created exceeding the original number of pods defined + in the specification. MaxSurge is only taken into consideration + when scaling up. Setting a negative value will disable the + restriction. Defaults to unbounded if not specified. + format: int32 + type: integer + maxUnavailable: + description: MaxUnavailable is the maximum number of pods that + can be unavailable (not ready) during the update due to circumstances + under the control of the operator. Setting a negative value + will disable this restriction. Defaults to 1 if not specified. + format: int32 + type: integer + type: object + type: object + version: + description: Version of Elasticsearch. + type: string + required: + - nodeSets + - version + type: object + status: + description: ElasticsearchStatus defines the observed state of Elasticsearch + properties: + availableNodes: + format: int32 + type: integer + health: + description: ElasticsearchHealth is the health of the cluster as returned + by the health API. + type: string + phase: + description: ElasticsearchOrchestrationPhase is the phase Elasticsearch + is in from the controller point of view. + type: string + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true + - name: v1beta1 + served: true + storage: false + - name: v1alpha1 + served: false + storage: false +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: enterprisesearches.enterprisesearch.k8s.elastic.co +spec: + additionalPrinterColumns: + - JSONPath: .status.health + name: health + type: string + - JSONPath: .status.availableNodes + description: Available nodes + name: nodes + type: integer + - JSONPath: .spec.version + description: Enterprise Search version + name: version + type: string + - JSONPath: .metadata.creationTimestamp + name: age + type: date + group: enterprisesearch.k8s.elastic.co + names: + categories: + - elastic + kind: EnterpriseSearch + listKind: EnterpriseSearchList + plural: enterprisesearches + shortNames: + - ent + singular: enterprisesearch + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: EnterpriseSearch is a Kubernetes CRD to represent Enterprise Search. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EnterpriseSearchSpec holds the specification of an Enterprise + Search resource. + properties: + config: + description: Config holds the Enterprise Search configuration. + type: object + configRef: + description: ConfigRef contains a reference to an existing Kubernetes + Secret holding the Enterprise Search configuration. Configuration + settings are merged and have precedence over settings specified in + `config`. + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + count: + description: Count of Enterprise Search instances to deploy. + format: int32 + type: integer + elasticsearchRef: + description: ElasticsearchRef is a reference to the Elasticsearch cluster + running in the same Kubernetes cluster. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + required: + - name + type: object + http: + description: HTTP holds the HTTP layer configuration for Enterprise + Search resource. + properties: + service: + description: Service defines the template for the associated Kubernetes + Service object. + properties: + metadata: + description: ObjectMeta is the metadata of the service. The + name and namespace provided here are managed by ECK and will + be ignored. + type: object + spec: + description: Spec is the specification of the service. + properties: + clusterIP: + description: 'clusterIP is the IP address of the service + and is usually assigned randomly by the master. If an + address is specified manually and is not in use by others, + it will be allocated to the service; otherwise, creation + of the service will fail. This field can not be changed + through updates. Valid values are "None", empty string + (""), or a valid IP address. "None" can be specified for + headless services when proxying is not required. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this + service. These IPs are not managed by Kubernetes. The + user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external + load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that + kubedns or equivalent will return as a CNAME record for + this service. No proxying will be involved. Must be a + valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires Type to be ExternalName. + type: string + externalTrafficPolicy: + description: externalTrafficPolicy denotes if this Service + desires to route external traffic to node-local or cluster-wide + endpoints. "Local" preserves the client source IP and + avoids a second hop for LoadBalancer and Nodeport type + services, but risks potentially imbalanced traffic spreading. + "Cluster" obscures the client source IP and may cause + a second hop to another node, but should have good overall + load-spreading. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. If not specified, HealthCheckNodePort + is created by the service api backend with the allocated + nodePort. Will use user-specified nodePort value if specified + by the client. Only effects when Type is set to LoadBalancer + and ExternalTrafficPolicy is set to Local. + format: int32 + type: integer + ipFamily: + description: ipFamily specifies whether this Service has + a preference for a particular IP family (e.g. IPv4 vs. + IPv6). If a specific IP family is requested, the clusterIP + field will be allocated from that family, if it is available + in the cluster. If no IP family is requested, the cluster's + primary IP family will be used. Other IP fields (loadBalancerIP, + loadBalancerSourceRanges, externalIPs) and controllers + which allocate external load-balancers should use the + same IP family. Endpoints for this Service will be of + this family. This field is immutable after creation. + Assigning a ServiceIPFamily not available in the cluster + (e.g. IPv6 in IPv4 only cluster) is an error condition + and will fail during clusterIP assignment. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer + LoadBalancer will get created with the IP specified in + this field. This feature depends on whether the underlying + cloud-provider supports specifying the loadBalancerIP + when a load balancer is created. This field will be ignored + if the cloud-provider does not support the feature.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client + IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this + service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field + in the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this + service is exposed when type=NodePort or LoadBalancer. + Usually assigned by the system. If specified, it + will be allocated to the service if unused or else + creation of the service will fail. Default is to + auto-allocate a port if the ServiceType of this + Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this + service. + format: int32 + type: integer + protocol: + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access + on the pods targeted by the service. Number must + be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored + for services with clusterIP=None, and should be + omitted or set equal to the ''port'' field. More + info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + required: + - port + type: object + type: array + publishNotReadyAddresses: + description: publishNotReadyAddresses, when set to true, + indicates that DNS implementations must publish the notReadyAddresses + of subsets for the Endpoints associated with the Service. + The default value is false. The primary use case for setting + this field is to use a StatefulSet's Headless Service + to propagate SRV records for its Pods without respect + to their readiness for purpose of peer discovery. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: + https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of + Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + topologyKeys: + description: topologyKeys is a preference-order list of + topology keys which implementations of services should + use to preferentially sort endpoints when accessing this + Service, it can not be used at the same time as externalTrafficPolicy=Local. + Topology keys must be valid label keys and at most 16 + keys may be specified. Endpoints are chosen based on the + first topology key with available backends. If this field + is specified and all entries have no backends that match + the topology of the client, the service has no backends + for that client and connections should fail. The special + value "*" may be used to mean "any topology". This catch-all + value, if used, only makes sense as the last value in + the list. If this is not specified or empty, no topology + constraints will be applied. + items: + type: string + type: array + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ExternalName" + maps to the specified externalName. "ClusterIP" allocates + a cluster-internal IP address for load-balancing to endpoints. + Endpoints are determined by the selector or if that is + not specified, by manual construction of an Endpoints + object. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints + rather than a stable IP. "NodePort" builds on ClusterIP + and allocates a port on every node which routes to the + clusterIP. "LoadBalancer" builds on NodePort and creates + an external load-balancer (if supported in the current + cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + tls: + description: TLS defines options for configuring TLS for HTTP. + properties: + certificate: + description: "Certificate is a reference to a Kubernetes secret + that contains the certificate and private key for enabling + TLS. The referenced secret should contain the following: \n + - `ca.crt`: The certificate authority (optional). - `tls.crt`: + The certificate (or a chain). - `tls.key`: The private key + to the first certificate in the certificate chain." + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + selfSignedCertificate: + description: SelfSignedCertificate allows configuring the self-signed + certificate generated by the operator. + properties: + disabled: + description: Disabled indicates that the provisioning of + the self-signed certifcate should be disabled. + type: boolean + subjectAltNames: + description: SubjectAlternativeNames is a list of SANs to + include in the generated HTTP TLS certificate. + items: + description: SubjectAlternativeName represents a SAN entry + in a x509 certificate. + properties: + dns: + description: DNS is the DNS name of the subject. + type: string + ip: + description: IP is the IP address of the subject. + type: string + type: object + type: array + type: object + type: object + type: object + image: + description: Image is the Enterprise Search Docker image to deploy. + type: string + podTemplate: + description: PodTemplate provides customisation options (labels, annotations, + affinity rules, resource requests, and so on) for the Enterprise Search + pods. + type: object + serviceAccountName: + description: ServiceAccountName is used to check access from the current + resource to a resource (eg. Elasticsearch) in a different namespace. + Can only be used if ECK is enforcing RBAC on references. + type: string + version: + description: Version of Enterprise Search. + type: string + type: object + status: + description: EnterpriseSearchStatus defines the observed state of EnterpriseSearch + properties: + associationStatus: + description: Association is the status of any auto-linking to Elasticsearch + clusters. + type: string + availableNodes: + format: int32 + type: integer + health: + description: EnterpriseSearchHealth expresses the health of the Enterprise + Search instances. + type: string + service: + description: ExternalService is the name of the service associated to + the Enterprise Search Pods. + type: string + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: kibanas.kibana.k8s.elastic.co +spec: + additionalPrinterColumns: + - JSONPath: .status.health + name: health + type: string + - JSONPath: .status.availableNodes + description: Available nodes + name: nodes + type: integer + - JSONPath: .spec.version + description: Kibana version + name: version + type: string + - JSONPath: .metadata.creationTimestamp + name: age + type: date + group: kibana.k8s.elastic.co + names: + categories: + - elastic + kind: Kibana + listKind: KibanaList + plural: kibanas + shortNames: + - kb + singular: kibana + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: Kibana represents a Kibana resource in a Kubernetes cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KibanaSpec holds the specification of a Kibana instance. + properties: + config: + description: 'Config holds the Kibana configuration. See: https://www.elastic.co/guide/en/kibana/current/settings.html' + type: object + count: + description: Count of Kibana instances to deploy. + format: int32 + type: integer + elasticsearchRef: + description: ElasticsearchRef is a reference to an Elasticsearch cluster + running in the same Kubernetes cluster. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + required: + - name + type: object + http: + description: HTTP holds the HTTP layer configuration for Kibana. + properties: + service: + description: Service defines the template for the associated Kubernetes + Service object. + properties: + metadata: + description: ObjectMeta is the metadata of the service. The + name and namespace provided here are managed by ECK and will + be ignored. + type: object + spec: + description: Spec is the specification of the service. + properties: + clusterIP: + description: 'clusterIP is the IP address of the service + and is usually assigned randomly by the master. If an + address is specified manually and is not in use by others, + it will be allocated to the service; otherwise, creation + of the service will fail. This field can not be changed + through updates. Valid values are "None", empty string + (""), or a valid IP address. "None" can be specified for + headless services when proxying is not required. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this + service. These IPs are not managed by Kubernetes. The + user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external + load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that + kubedns or equivalent will return as a CNAME record for + this service. No proxying will be involved. Must be a + valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires Type to be ExternalName. + type: string + externalTrafficPolicy: + description: externalTrafficPolicy denotes if this Service + desires to route external traffic to node-local or cluster-wide + endpoints. "Local" preserves the client source IP and + avoids a second hop for LoadBalancer and Nodeport type + services, but risks potentially imbalanced traffic spreading. + "Cluster" obscures the client source IP and may cause + a second hop to another node, but should have good overall + load-spreading. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. If not specified, HealthCheckNodePort + is created by the service api backend with the allocated + nodePort. Will use user-specified nodePort value if specified + by the client. Only effects when Type is set to LoadBalancer + and ExternalTrafficPolicy is set to Local. + format: int32 + type: integer + ipFamily: + description: ipFamily specifies whether this Service has + a preference for a particular IP family (e.g. IPv4 vs. + IPv6). If a specific IP family is requested, the clusterIP + field will be allocated from that family, if it is available + in the cluster. If no IP family is requested, the cluster's + primary IP family will be used. Other IP fields (loadBalancerIP, + loadBalancerSourceRanges, externalIPs) and controllers + which allocate external load-balancers should use the + same IP family. Endpoints for this Service will be of + this family. This field is immutable after creation. + Assigning a ServiceIPFamily not available in the cluster + (e.g. IPv6 in IPv4 only cluster) is an error condition + and will fail during clusterIP assignment. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer + LoadBalancer will get created with the IP specified in + this field. This feature depends on whether the underlying + cloud-provider supports specifying the loadBalancerIP + when a load balancer is created. This field will be ignored + if the cloud-provider does not support the feature.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client + IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this + service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field + in the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this + service is exposed when type=NodePort or LoadBalancer. + Usually assigned by the system. If specified, it + will be allocated to the service if unused or else + creation of the service will fail. Default is to + auto-allocate a port if the ServiceType of this + Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this + service. + format: int32 + type: integer + protocol: + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access + on the pods targeted by the service. Number must + be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored + for services with clusterIP=None, and should be + omitted or set equal to the ''port'' field. More + info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + required: + - port + type: object + type: array + publishNotReadyAddresses: + description: publishNotReadyAddresses, when set to true, + indicates that DNS implementations must publish the notReadyAddresses + of subsets for the Endpoints associated with the Service. + The default value is false. The primary use case for setting + this field is to use a StatefulSet's Headless Service + to propagate SRV records for its Pods without respect + to their readiness for purpose of peer discovery. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: + https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of + Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + topologyKeys: + description: topologyKeys is a preference-order list of + topology keys which implementations of services should + use to preferentially sort endpoints when accessing this + Service, it can not be used at the same time as externalTrafficPolicy=Local. + Topology keys must be valid label keys and at most 16 + keys may be specified. Endpoints are chosen based on the + first topology key with available backends. If this field + is specified and all entries have no backends that match + the topology of the client, the service has no backends + for that client and connections should fail. The special + value "*" may be used to mean "any topology". This catch-all + value, if used, only makes sense as the last value in + the list. If this is not specified or empty, no topology + constraints will be applied. + items: + type: string + type: array + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ExternalName" + maps to the specified externalName. "ClusterIP" allocates + a cluster-internal IP address for load-balancing to endpoints. + Endpoints are determined by the selector or if that is + not specified, by manual construction of an Endpoints + object. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints + rather than a stable IP. "NodePort" builds on ClusterIP + and allocates a port on every node which routes to the + clusterIP. "LoadBalancer" builds on NodePort and creates + an external load-balancer (if supported in the current + cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + tls: + description: TLS defines options for configuring TLS for HTTP. + properties: + certificate: + description: "Certificate is a reference to a Kubernetes secret + that contains the certificate and private key for enabling + TLS. The referenced secret should contain the following: \n + - `ca.crt`: The certificate authority (optional). - `tls.crt`: + The certificate (or a chain). - `tls.key`: The private key + to the first certificate in the certificate chain." + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + selfSignedCertificate: + description: SelfSignedCertificate allows configuring the self-signed + certificate generated by the operator. + properties: + disabled: + description: Disabled indicates that the provisioning of + the self-signed certifcate should be disabled. + type: boolean + subjectAltNames: + description: SubjectAlternativeNames is a list of SANs to + include in the generated HTTP TLS certificate. + items: + description: SubjectAlternativeName represents a SAN entry + in a x509 certificate. + properties: + dns: + description: DNS is the DNS name of the subject. + type: string + ip: + description: IP is the IP address of the subject. + type: string + type: object + type: array + type: object + type: object + type: object + image: + description: Image is the Kibana Docker image to deploy. + type: string + podTemplate: + description: PodTemplate provides customisation options (labels, annotations, + affinity rules, resource requests, and so on) for the Kibana pods + type: object + secureSettings: + description: SecureSettings is a list of references to Kubernetes secrets + containing sensitive configuration options for Kibana. + items: + description: SecretSource defines a data source based on a Kubernetes + Secret. + properties: + entries: + description: Entries define how to project each key-value pair + in the secret to filesystem paths. If not defined, all keys + will be projected to similarly named paths in the filesystem. + If defined, only the specified keys will be projected to the + corresponding paths. + items: + description: KeyToPath defines how to map a key in a Secret + object to a filesystem path. + properties: + key: + description: Key is the key contained in the secret. + type: string + path: + description: Path is the relative file path to map the key + to. Path must not be an absolute file path and must not + contain any ".." components. + type: string + required: + - key + type: object + type: array + secretName: + description: SecretName is the name of the secret. + type: string + required: + - secretName + type: object + type: array + serviceAccountName: + description: ServiceAccountName is used to check access from the current + resource to a resource (eg. Elasticsearch) in a different namespace. + Can only be used if ECK is enforcing RBAC on references. + type: string + version: + description: Version of Kibana. + type: string + required: + - version + type: object + status: + description: KibanaStatus defines the observed state of Kibana + properties: + associationStatus: + description: AssociationStatus is the status of an association resource. + type: string + availableNodes: + format: int32 + type: integer + health: + description: KibanaHealth expresses the status of the Kibana instances. + type: string + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true + - name: v1beta1 + served: true + storage: false + - name: v1alpha1 + served: false + storage: false +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +# Source: eck/templates/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: elastic-system +--- +# Source: eck/templates/service-account.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elastic-operator + namespace: elastic-system +--- +# Source: eck/templates/webhook.yaml +apiVersion: v1 +kind: Secret +metadata: + name: "elastic-webhook-server-cert" + namespace: elastic-system +--- +# Source: eck/templates/cluster-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: elastic-operator +rules: +- apiGroups: + - "authorization.k8s.io" + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - "" + resources: + - pods + - endpoints + - events + - persistentvolumeclaims + - secrets + - services + - configmaps + - serviceaccounts + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - apps + resources: + - deployments + - statefulsets + - daemonsets + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - elasticsearch.k8s.elastic.co + resources: + - elasticsearches + - elasticsearches/status + - elasticsearches/finalizers + - enterpriselicenses + - enterpriselicenses/status + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - kibana.k8s.elastic.co + resources: + - kibanas + - kibanas/status + - kibanas/finalizers + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - apm.k8s.elastic.co + resources: + - apmservers + - apmservers/status + - apmservers/finalizers + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - enterprisesearch.k8s.elastic.co + resources: + - enterprisesearches + - enterprisesearches/status + - enterprisesearches/finalizers + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - beat.k8s.elastic.co + resources: + - beats + - beats/status + - beats/finalizers + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +--- +# Source: eck/templates/cluster-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "elastic-operator-view" + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: + - apiGroups: ["elasticsearch.k8s.elastic.co"] + resources: ["elasticsearches"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apm.k8s.elastic.co"] + resources: ["apmservers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["kibana.k8s.elastic.co"] + resources: ["kibanas"] + verbs: ["get", "list", "watch"] + - apiGroups: ["enterprisesearch.k8s.elastic.co"] + resources: ["enterprisesearches"] + verbs: ["get", "list", "watch"] + - apiGroups: ["beat.k8s.elastic.co"] + resources: ["beats"] + verbs: ["get", "list", "watch"] +--- +# Source: eck/templates/cluster-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "elastic-operator-edit" + labels: + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: + - apiGroups: ["elasticsearch.k8s.elastic.co"] + resources: ["elasticsearches"] + verbs: ["create", "delete", "deletecollection", "patch", "update"] + - apiGroups: ["apm.k8s.elastic.co"] + resources: ["apmservers"] + verbs: ["create", "delete", "deletecollection", "patch", "update"] + - apiGroups: ["kibana.k8s.elastic.co"] + resources: ["kibanas"] + verbs: ["create", "delete", "deletecollection", "patch", "update"] + - apiGroups: ["enterprisesearch.k8s.elastic.co"] + resources: ["enterprisesearches"] + verbs: ["create", "delete", "deletecollection", "patch", "update"] + - apiGroups: ["beat.k8s.elastic.co"] + resources: ["beats"] + verbs: ["create", "delete", "deletecollection", "patch", "update"] +--- +# Source: eck/templates/managed-ns-role-bindings.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: elastic-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: elastic-operator +subjects: +- kind: ServiceAccount + name: elastic-operator + namespace: elastic-system +--- +# Source: eck/templates/operator-role-binding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: elastic-operator + namespace: elastic-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: elastic-operator +subjects: +- kind: ServiceAccount + name: elastic-operator + namespace: elastic-system +--- +# Source: eck/templates/webhook.yaml +apiVersion: v1 +kind: Service +metadata: + name: elastic-webhook-server + namespace: elastic-system +spec: + ports: + - name: https + port: 443 + targetPort: 9443 + selector: + control-plane: elastic-operator +--- +# Source: eck/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elastic-operator + namespace: elastic-system + labels: + control-plane: elastic-operator +spec: + selector: + matchLabels: + control-plane: elastic-operator + serviceName: elastic-operator + template: + metadata: + annotations: + # Rename the fields "error" to "error.message" and "source" to "event.source" + # This is to avoid a conflict with the ECS "error" and "source" documents. + "co.elastic.logs/raw": "[{\"type\":\"container\",\"json.keys_under_root\":true,\"paths\":[\"/var/log/containers/*${data.kubernetes.container.id}.log\"],\"processors\":[{\"convert\":{\"mode\":\"rename\",\"ignore_missing\":true,\"fields\":[{\"from\":\"error\",\"to\":\"_error\"}]}},{\"convert\":{\"mode\":\"rename\",\"ignore_missing\":true,\"fields\":[{\"from\":\"_error\",\"to\":\"error.message\"}]}},{\"convert\":{\"mode\":\"rename\",\"ignore_missing\":true,\"fields\":[{\"from\":\"source\",\"to\":\"_source\"}]}},{\"convert\":{\"mode\":\"rename\",\"ignore_missing\":true,\"fields\":[{\"from\":\"_source\",\"to\":\"event.source\"}]}}]}]" + labels: + control-plane: elastic-operator + spec: + terminationGracePeriodSeconds: 10 + serviceAccountName: elastic-operator + containers: + - image: "docker.elastic.co/eck/eck-operator:1.2.0" + imagePullPolicy: IfNotPresent + name: manager + args: + - "manager" + - "--log-verbosity=0" + - "--metrics-port=0" + - "--container-registry=docker.elastic.co" + - "--max-concurrent-reconciles=3" + - "--ca-cert-validity=8760h" + - "--ca-cert-rotate-before=24h" + - "--cert-validity=8760h" + - "--cert-rotate-before=24h" + - "--enable-webhook" + env: + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_IMAGE + value: "docker.elastic.co/eck/eck-operator:1.2.0" + - name: WEBHOOK_SECRET + value: "elastic-webhook-server-cert" + resources: + limits: + cpu: 1 + memory: 512Mi + requests: + cpu: 100m + memory: 150Mi + ports: + - containerPort: 9443 + name: https-webhook + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: "elastic-webhook-server-cert" +--- +# Source: eck/templates/webhook.yaml +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: elastic-webhook.k8s.elastic.co +webhooks: +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-apm-k8s-elastic-co-v1-apmserver + failurePolicy: Ignore + name: elastic-apm-validation-v1.k8s.elastic.co + rules: + - apiGroups: + - apm.k8s.elastic.co + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - apmservers +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-apm-k8s-elastic-co-v1beta1-apmserver + failurePolicy: Ignore + name: elastic-apm-validation-v1beta1.k8s.elastic.co + rules: + - apiGroups: + - apm.k8s.elastic.co + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - apmservers +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-beat-k8s-elastic-co-v1beta1-beat + failurePolicy: Ignore + name: elastic-beat-validation-v1beta1.k8s.elastic.co + rules: + - apiGroups: + - beat.k8s.elastic.co + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - beats +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-elasticsearch-k8s-elastic-co-v1-elasticsearch + failurePolicy: Ignore + name: elastic-es-validation-v1.k8s.elastic.co + rules: + - apiGroups: + - elasticsearch.k8s.elastic.co + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - elasticsearches +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-elasticsearch-k8s-elastic-co-v1beta1-elasticsearch + failurePolicy: Ignore + name: elastic-es-validation-v1beta1.k8s.elastic.co + rules: + - apiGroups: + - elasticsearch.k8s.elastic.co + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - elasticsearches +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-kibana-k8s-elastic-co-v1-kibana + failurePolicy: Ignore + name: elastic-kb-validation-v1.k8s.elastic.co + rules: + - apiGroups: + - kibana.k8s.elastic.co + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - kibanas +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-kibana-k8s-elastic-co-v1beta1-kibana + failurePolicy: Ignore + name: elastic-kb-validation-v1beta1.k8s.elastic.co + rules: + - apiGroups: + - kibana.k8s.elastic.co + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - kibanas + diff --git a/charts/kubezero-logging/eck/kustomization.yaml b/charts/kubezero-logging/eck/kustomization.yaml new file mode 100644 index 0000000..6adfff1 --- /dev/null +++ b/charts/kubezero-logging/eck/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- all-in-one.yaml + +# map operator to controller nodes +patchesStrategicMerge: +- map-operator.yaml diff --git a/charts/kubezero-logging/eck/map-operator.yaml b/charts/kubezero-logging/eck/map-operator.yaml new file mode 100644 index 0000000..e21c16f --- /dev/null +++ b/charts/kubezero-logging/eck/map-operator.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elastic-operator +spec: + template: + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/charts/kubezero-logging/eck/update.sh b/charts/kubezero-logging/eck/update.sh new file mode 100755 index 0000000..e67214a --- /dev/null +++ b/charts/kubezero-logging/eck/update.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +ECK_VERSION=1.2.0 + +curl -o all-in-one.yaml https://download.elastic.co/downloads/eck/${ECK_VERSION}/all-in-one.yaml + +kubectl kustomize . > ../templates/eck-operator.yaml diff --git a/charts/kubezero-logging/templates/eck-operator.yaml b/charts/kubezero-logging/templates/eck-operator.yaml new file mode 100644 index 0000000..82cc137 --- /dev/null +++ b/charts/kubezero-logging/templates/eck-operator.yaml @@ -0,0 +1,3058 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: elastic-system +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: apmservers.apm.k8s.elastic.co +spec: + additionalPrinterColumns: + - JSONPath: .status.health + name: health + type: string + - JSONPath: .status.availableNodes + description: Available nodes + name: nodes + type: integer + - JSONPath: .spec.version + description: APM version + name: version + type: string + - JSONPath: .metadata.creationTimestamp + name: age + type: date + group: apm.k8s.elastic.co + names: + categories: + - elastic + kind: ApmServer + listKind: ApmServerList + plural: apmservers + shortNames: + - apm + singular: apmserver + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: ApmServer represents an APM Server resource in a Kubernetes cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ApmServerSpec holds the specification of an APM Server. + properties: + config: + description: 'Config holds the APM Server configuration. See: https://www.elastic.co/guide/en/apm/server/current/configuring-howto-apm-server.html' + type: object + count: + description: Count of APM Server instances to deploy. + format: int32 + type: integer + elasticsearchRef: + description: ElasticsearchRef is a reference to the output Elasticsearch + cluster running in the same Kubernetes cluster. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + required: + - name + type: object + http: + description: HTTP holds the HTTP layer configuration for the APM Server + resource. + properties: + service: + description: Service defines the template for the associated Kubernetes + Service object. + properties: + metadata: + description: ObjectMeta is the metadata of the service. The + name and namespace provided here are managed by ECK and will + be ignored. + type: object + spec: + description: Spec is the specification of the service. + properties: + clusterIP: + description: 'clusterIP is the IP address of the service + and is usually assigned randomly by the master. If an + address is specified manually and is not in use by others, + it will be allocated to the service; otherwise, creation + of the service will fail. This field can not be changed + through updates. Valid values are "None", empty string + (""), or a valid IP address. "None" can be specified for + headless services when proxying is not required. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this + service. These IPs are not managed by Kubernetes. The + user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external + load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that + kubedns or equivalent will return as a CNAME record for + this service. No proxying will be involved. Must be a + valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires Type to be ExternalName. + type: string + externalTrafficPolicy: + description: externalTrafficPolicy denotes if this Service + desires to route external traffic to node-local or cluster-wide + endpoints. "Local" preserves the client source IP and + avoids a second hop for LoadBalancer and Nodeport type + services, but risks potentially imbalanced traffic spreading. + "Cluster" obscures the client source IP and may cause + a second hop to another node, but should have good overall + load-spreading. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. If not specified, HealthCheckNodePort + is created by the service api backend with the allocated + nodePort. Will use user-specified nodePort value if specified + by the client. Only effects when Type is set to LoadBalancer + and ExternalTrafficPolicy is set to Local. + format: int32 + type: integer + ipFamily: + description: ipFamily specifies whether this Service has + a preference for a particular IP family (e.g. IPv4 vs. + IPv6). If a specific IP family is requested, the clusterIP + field will be allocated from that family, if it is available + in the cluster. If no IP family is requested, the cluster's + primary IP family will be used. Other IP fields (loadBalancerIP, + loadBalancerSourceRanges, externalIPs) and controllers + which allocate external load-balancers should use the + same IP family. Endpoints for this Service will be of + this family. This field is immutable after creation. + Assigning a ServiceIPFamily not available in the cluster + (e.g. IPv6 in IPv4 only cluster) is an error condition + and will fail during clusterIP assignment. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer + LoadBalancer will get created with the IP specified in + this field. This feature depends on whether the underlying + cloud-provider supports specifying the loadBalancerIP + when a load balancer is created. This field will be ignored + if the cloud-provider does not support the feature.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client + IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this + service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field + in the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this + service is exposed when type=NodePort or LoadBalancer. + Usually assigned by the system. If specified, it + will be allocated to the service if unused or else + creation of the service will fail. Default is to + auto-allocate a port if the ServiceType of this + Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this + service. + format: int32 + type: integer + protocol: + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access + on the pods targeted by the service. Number must + be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored + for services with clusterIP=None, and should be + omitted or set equal to the ''port'' field. More + info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + required: + - port + type: object + type: array + publishNotReadyAddresses: + description: publishNotReadyAddresses, when set to true, + indicates that DNS implementations must publish the notReadyAddresses + of subsets for the Endpoints associated with the Service. + The default value is false. The primary use case for setting + this field is to use a StatefulSet's Headless Service + to propagate SRV records for its Pods without respect + to their readiness for purpose of peer discovery. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: + https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of + Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + topologyKeys: + description: topologyKeys is a preference-order list of + topology keys which implementations of services should + use to preferentially sort endpoints when accessing this + Service, it can not be used at the same time as externalTrafficPolicy=Local. + Topology keys must be valid label keys and at most 16 + keys may be specified. Endpoints are chosen based on the + first topology key with available backends. If this field + is specified and all entries have no backends that match + the topology of the client, the service has no backends + for that client and connections should fail. The special + value "*" may be used to mean "any topology". This catch-all + value, if used, only makes sense as the last value in + the list. If this is not specified or empty, no topology + constraints will be applied. + items: + type: string + type: array + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ExternalName" + maps to the specified externalName. "ClusterIP" allocates + a cluster-internal IP address for load-balancing to endpoints. + Endpoints are determined by the selector or if that is + not specified, by manual construction of an Endpoints + object. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints + rather than a stable IP. "NodePort" builds on ClusterIP + and allocates a port on every node which routes to the + clusterIP. "LoadBalancer" builds on NodePort and creates + an external load-balancer (if supported in the current + cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + tls: + description: TLS defines options for configuring TLS for HTTP. + properties: + certificate: + description: "Certificate is a reference to a Kubernetes secret + that contains the certificate and private key for enabling + TLS. The referenced secret should contain the following: \n + - `ca.crt`: The certificate authority (optional). - `tls.crt`: + The certificate (or a chain). - `tls.key`: The private key + to the first certificate in the certificate chain." + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + selfSignedCertificate: + description: SelfSignedCertificate allows configuring the self-signed + certificate generated by the operator. + properties: + disabled: + description: Disabled indicates that the provisioning of + the self-signed certifcate should be disabled. + type: boolean + subjectAltNames: + description: SubjectAlternativeNames is a list of SANs to + include in the generated HTTP TLS certificate. + items: + description: SubjectAlternativeName represents a SAN entry + in a x509 certificate. + properties: + dns: + description: DNS is the DNS name of the subject. + type: string + ip: + description: IP is the IP address of the subject. + type: string + type: object + type: array + type: object + type: object + type: object + image: + description: Image is the APM Server Docker image to deploy. + type: string + kibanaRef: + description: KibanaRef is a reference to a Kibana instance running in + the same Kubernetes cluster. It allows APM agent central configuration + management in Kibana. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + required: + - name + type: object + podTemplate: + description: PodTemplate provides customisation options (labels, annotations, + affinity rules, resource requests, and so on) for the APM Server pods. + type: object + secureSettings: + description: SecureSettings is a list of references to Kubernetes secrets + containing sensitive configuration options for APM Server. + items: + description: SecretSource defines a data source based on a Kubernetes + Secret. + properties: + entries: + description: Entries define how to project each key-value pair + in the secret to filesystem paths. If not defined, all keys + will be projected to similarly named paths in the filesystem. + If defined, only the specified keys will be projected to the + corresponding paths. + items: + description: KeyToPath defines how to map a key in a Secret + object to a filesystem path. + properties: + key: + description: Key is the key contained in the secret. + type: string + path: + description: Path is the relative file path to map the key + to. Path must not be an absolute file path and must not + contain any ".." components. + type: string + required: + - key + type: object + type: array + secretName: + description: SecretName is the name of the secret. + type: string + required: + - secretName + type: object + type: array + serviceAccountName: + description: ServiceAccountName is used to check access from the current + resource to a resource (eg. Elasticsearch) in a different namespace. + Can only be used if ECK is enforcing RBAC on references. + type: string + version: + description: Version of the APM Server. + type: string + required: + - version + type: object + status: + description: ApmServerStatus defines the observed state of ApmServer + properties: + availableNodes: + format: int32 + type: integer + elasticsearchAssociationStatus: + description: ElasticsearchAssociationStatus is the status of any auto-linking + to Elasticsearch clusters. + type: string + health: + description: ApmServerHealth expresses the status of the Apm Server + instances. + type: string + kibanaAssociationStatus: + description: KibanaAssociationStatus is the status of any auto-linking + to Kibana. + type: string + secretTokenSecret: + description: SecretTokenSecretName is the name of the Secret that contains + the secret token + type: string + service: + description: ExternalService is the name of the service the agents should + connect to. + type: string + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true + - name: v1beta1 + served: true + storage: false + - name: v1alpha1 + served: false + storage: false +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: beats.beat.k8s.elastic.co +spec: + additionalPrinterColumns: + - JSONPath: .status.health + name: health + type: string + - JSONPath: .status.availableNodes + description: Available nodes + name: available + type: integer + - JSONPath: .status.expectedNodes + description: Expected nodes + name: expected + type: integer + - JSONPath: .spec.type + description: Beat type + name: type + type: string + - JSONPath: .spec.version + description: Beat version + name: version + type: string + - JSONPath: .metadata.creationTimestamp + name: age + type: date + group: beat.k8s.elastic.co + names: + categories: + - elastic + kind: Beat + listKind: BeatList + plural: beats + shortNames: + - beat + singular: beat + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: Beat is the Schema for the Beats API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BeatSpec defines the desired state of a Beat. + properties: + config: + description: Config holds the Beat configuration. At most one of [`Config`, + `ConfigRef`] can be specified. + type: object + configRef: + description: ConfigRef contains a reference to an existing Kubernetes + Secret holding the Beat configuration. Beat settings must be specified + as yaml, under a single "beat.yml" entry. At most one of [`Config`, + `ConfigRef`] can be specified. + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + daemonSet: + description: DaemonSet specifies the Beat should be deployed as a DaemonSet, + and allows providing its spec. Cannot be used along with `deployment`. + If both are absent a default for the Type is used. + properties: {} + type: object + deployment: + description: Deployment specifies the Beat should be deployed as a Deployment, + and allows providing its spec. Cannot be used along with `daemonSet`. + If both are absent a default for the Type is used. + properties: + replicas: + format: int32 + type: integer + type: object + elasticsearchRef: + description: ElasticsearchRef is a reference to an Elasticsearch cluster + running in the same Kubernetes cluster. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + required: + - name + type: object + image: + description: Image is the Beat Docker image to deploy. Version and Type + have to match the Beat in the image. + type: string + kibanaRef: + description: KibanaRef is a reference to a Kibana instance running in + the same Kubernetes cluster. It allows automatic setup of dashboards + and visualizations. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + required: + - name + type: object + secureSettings: + description: SecureSettings is a list of references to Kubernetes Secrets + containing sensitive configuration options for the Beat. Secrets data + can be then referenced in the Beat config using the Secret's keys + or as specified in `Entries` field of each SecureSetting. + items: + description: SecretSource defines a data source based on a Kubernetes + Secret. + properties: + entries: + description: Entries define how to project each key-value pair + in the secret to filesystem paths. If not defined, all keys + will be projected to similarly named paths in the filesystem. + If defined, only the specified keys will be projected to the + corresponding paths. + items: + description: KeyToPath defines how to map a key in a Secret + object to a filesystem path. + properties: + key: + description: Key is the key contained in the secret. + type: string + path: + description: Path is the relative file path to map the key + to. Path must not be an absolute file path and must not + contain any ".." components. + type: string + required: + - key + type: object + type: array + secretName: + description: SecretName is the name of the secret. + type: string + required: + - secretName + type: object + type: array + serviceAccountName: + description: ServiceAccountName is used to check access from the current + resource to Elasticsearch resource in a different namespace. Can only + be used if ECK is enforcing RBAC on references. + type: string + type: + description: Type is the type of the Beat to deploy (filebeat, metricbeat, + heartbeat, auditbeat, journalbeat, packetbeat, etc.). Any string can + be used, but well-known types will have the image field defaulted + and have the appropriate Elasticsearch roles created automatically. + It also allows for dashboard setup when combined with a `KibanaRef`. + maxLength: 20 + pattern: '[a-zA-Z0-9-]+' + type: string + version: + description: Version of the Beat. + type: string + required: + - type + - version + type: object + status: + description: BeatStatus defines the observed state of a Beat. + properties: + availableNodes: + format: int32 + type: integer + elasticsearchAssociationStatus: + description: AssociationStatus is the status of an association resource. + type: string + expectedNodes: + format: int32 + type: integer + health: + type: string + kibanaAssociationStatus: + description: AssociationStatus is the status of an association resource. + type: string + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: elasticsearches.elasticsearch.k8s.elastic.co +spec: + additionalPrinterColumns: + - JSONPath: .status.health + name: health + type: string + - JSONPath: .status.availableNodes + description: Available nodes + name: nodes + type: integer + - JSONPath: .spec.version + description: Elasticsearch version + name: version + type: string + - JSONPath: .status.phase + name: phase + type: string + - JSONPath: .metadata.creationTimestamp + name: age + type: date + group: elasticsearch.k8s.elastic.co + names: + categories: + - elastic + kind: Elasticsearch + listKind: ElasticsearchList + plural: elasticsearches + shortNames: + - es + singular: elasticsearch + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: Elasticsearch represents an Elasticsearch resource in a Kubernetes + cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ElasticsearchSpec holds the specification of an Elasticsearch + cluster. + properties: + auth: + description: Auth contains user authentication and authorization security + settings for Elasticsearch. + properties: + fileRealm: + description: FileRealm to propagate to the Elasticsearch cluster. + items: + description: FileRealmSource references users to create in the + Elasticsearch cluster. + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + type: array + roles: + description: Roles to propagate to the Elasticsearch cluster. + items: + description: RoleSource references roles to create in the Elasticsearch + cluster. + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + type: array + type: object + http: + description: HTTP holds HTTP layer settings for Elasticsearch. + properties: + service: + description: Service defines the template for the associated Kubernetes + Service object. + properties: + metadata: + description: ObjectMeta is the metadata of the service. The + name and namespace provided here are managed by ECK and will + be ignored. + type: object + spec: + description: Spec is the specification of the service. + properties: + clusterIP: + description: 'clusterIP is the IP address of the service + and is usually assigned randomly by the master. If an + address is specified manually and is not in use by others, + it will be allocated to the service; otherwise, creation + of the service will fail. This field can not be changed + through updates. Valid values are "None", empty string + (""), or a valid IP address. "None" can be specified for + headless services when proxying is not required. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this + service. These IPs are not managed by Kubernetes. The + user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external + load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that + kubedns or equivalent will return as a CNAME record for + this service. No proxying will be involved. Must be a + valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires Type to be ExternalName. + type: string + externalTrafficPolicy: + description: externalTrafficPolicy denotes if this Service + desires to route external traffic to node-local or cluster-wide + endpoints. "Local" preserves the client source IP and + avoids a second hop for LoadBalancer and Nodeport type + services, but risks potentially imbalanced traffic spreading. + "Cluster" obscures the client source IP and may cause + a second hop to another node, but should have good overall + load-spreading. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. If not specified, HealthCheckNodePort + is created by the service api backend with the allocated + nodePort. Will use user-specified nodePort value if specified + by the client. Only effects when Type is set to LoadBalancer + and ExternalTrafficPolicy is set to Local. + format: int32 + type: integer + ipFamily: + description: ipFamily specifies whether this Service has + a preference for a particular IP family (e.g. IPv4 vs. + IPv6). If a specific IP family is requested, the clusterIP + field will be allocated from that family, if it is available + in the cluster. If no IP family is requested, the cluster's + primary IP family will be used. Other IP fields (loadBalancerIP, + loadBalancerSourceRanges, externalIPs) and controllers + which allocate external load-balancers should use the + same IP family. Endpoints for this Service will be of + this family. This field is immutable after creation. + Assigning a ServiceIPFamily not available in the cluster + (e.g. IPv6 in IPv4 only cluster) is an error condition + and will fail during clusterIP assignment. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer + LoadBalancer will get created with the IP specified in + this field. This feature depends on whether the underlying + cloud-provider supports specifying the loadBalancerIP + when a load balancer is created. This field will be ignored + if the cloud-provider does not support the feature.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client + IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this + service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field + in the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this + service is exposed when type=NodePort or LoadBalancer. + Usually assigned by the system. If specified, it + will be allocated to the service if unused or else + creation of the service will fail. Default is to + auto-allocate a port if the ServiceType of this + Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this + service. + format: int32 + type: integer + protocol: + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access + on the pods targeted by the service. Number must + be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored + for services with clusterIP=None, and should be + omitted or set equal to the ''port'' field. More + info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + required: + - port + type: object + type: array + publishNotReadyAddresses: + description: publishNotReadyAddresses, when set to true, + indicates that DNS implementations must publish the notReadyAddresses + of subsets for the Endpoints associated with the Service. + The default value is false. The primary use case for setting + this field is to use a StatefulSet's Headless Service + to propagate SRV records for its Pods without respect + to their readiness for purpose of peer discovery. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: + https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of + Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + topologyKeys: + description: topologyKeys is a preference-order list of + topology keys which implementations of services should + use to preferentially sort endpoints when accessing this + Service, it can not be used at the same time as externalTrafficPolicy=Local. + Topology keys must be valid label keys and at most 16 + keys may be specified. Endpoints are chosen based on the + first topology key with available backends. If this field + is specified and all entries have no backends that match + the topology of the client, the service has no backends + for that client and connections should fail. The special + value "*" may be used to mean "any topology". This catch-all + value, if used, only makes sense as the last value in + the list. If this is not specified or empty, no topology + constraints will be applied. + items: + type: string + type: array + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ExternalName" + maps to the specified externalName. "ClusterIP" allocates + a cluster-internal IP address for load-balancing to endpoints. + Endpoints are determined by the selector or if that is + not specified, by manual construction of an Endpoints + object. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints + rather than a stable IP. "NodePort" builds on ClusterIP + and allocates a port on every node which routes to the + clusterIP. "LoadBalancer" builds on NodePort and creates + an external load-balancer (if supported in the current + cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + tls: + description: TLS defines options for configuring TLS for HTTP. + properties: + certificate: + description: "Certificate is a reference to a Kubernetes secret + that contains the certificate and private key for enabling + TLS. The referenced secret should contain the following: \n + - `ca.crt`: The certificate authority (optional). - `tls.crt`: + The certificate (or a chain). - `tls.key`: The private key + to the first certificate in the certificate chain." + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + selfSignedCertificate: + description: SelfSignedCertificate allows configuring the self-signed + certificate generated by the operator. + properties: + disabled: + description: Disabled indicates that the provisioning of + the self-signed certifcate should be disabled. + type: boolean + subjectAltNames: + description: SubjectAlternativeNames is a list of SANs to + include in the generated HTTP TLS certificate. + items: + description: SubjectAlternativeName represents a SAN entry + in a x509 certificate. + properties: + dns: + description: DNS is the DNS name of the subject. + type: string + ip: + description: IP is the IP address of the subject. + type: string + type: object + type: array + type: object + type: object + type: object + image: + description: Image is the Elasticsearch Docker image to deploy. + type: string + nodeSets: + description: 'NodeSets allow specifying groups of Elasticsearch nodes + sharing the same configuration and Pod templates. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-orchestration.html' + items: + description: NodeSet is the specification for a group of Elasticsearch + nodes sharing the same configuration and a Pod template. + properties: + config: + description: Config holds the Elasticsearch configuration. + type: object + count: + description: Count of Elasticsearch nodes to deploy. + format: int32 + minimum: 1 + type: integer + name: + description: Name of this set of nodes. Becomes a part of the + Elasticsearch node.name setting. + maxLength: 23 + pattern: '[a-zA-Z0-9-]+' + type: string + podTemplate: + description: PodTemplate provides customisation options (labels, + annotations, affinity rules, resource requests, and so on) for + the Pods belonging to this NodeSet. + type: object + volumeClaimTemplates: + description: 'VolumeClaimTemplates is a list of persistent volume + claims to be used by each Pod in this NodeSet. Every claim in + this list must have a matching volumeMount in one of the containers + defined in the PodTemplate. Items defined here take precedence + over any default claims added by the operator with the same + name. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-volume-claim-templates.html' + items: + description: PersistentVolumeClaim is a user's request for and + claim to a persistent volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + type: object + spec: + description: 'Spec defines the desired characteristics of + a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: This field requires the VolumeSnapshotDataSource + alpha feature gate to be enabled and currently VolumeSnapshot + is the only supported data source. If the provisioner + can support VolumeSnapshot data source, it will create + a new volume and data will be restored to the volume + at the same time. If the provisioner does not support + VolumeSnapshot data source, volume will not be created + and the failure will be reported as an event. In the + future, we plan to support more data source types + and the behavior of the provisioner may change. + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + selector: + description: A label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required by the + claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem is implied + when not included in claim spec. This is a beta feature. + type: string + volumeName: + description: VolumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + status: + description: 'Status represents the current information/status + of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the actual access + modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: Represents the actual resources of the + underlying volume. + type: object + conditions: + description: Current Condition of persistent volume + claim. If underlying persistent volume is being resized + then the Condition will be set to 'ResizeStarted'. + items: + description: PersistentVolumeClaimCondition contails + details about state of pvc + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned + from one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating + details about last transition. + type: string + reason: + description: Unique, this should be a short, machine + understandable string that gives the reason + for condition's last transition. If it reports + "ResizeStarted" that means the underlying persistent + volume is being resized. + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType + is a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: Phase represents the current phase of PersistentVolumeClaim. + type: string + type: object + type: object + type: array + required: + - count + - name + type: object + minItems: 1 + type: array + podDisruptionBudget: + description: PodDisruptionBudget provides access to the default pod + disruption budget for the Elasticsearch cluster. The default budget + selects all cluster pods and sets `maxUnavailable` to 1. To disable, + set `PodDisruptionBudget` to the empty value (`{}` in YAML). + properties: + metadata: + description: ObjectMeta is the metadata of the PDB. The name and + namespace provided here are managed by ECK and will be ignored. + type: object + spec: + description: Spec is the specification of the PDB. + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: An eviction is allowed if at most "maxUnavailable" + pods selected by "selector" are unavailable after the eviction, + i.e. even in absence of the evicted pod. For example, one + can prevent all voluntary evictions by specifying 0. This + is a mutually exclusive setting with "minAvailable". + minAvailable: + anyOf: + - type: integer + - type: string + description: An eviction is allowed if at least "minAvailable" + pods selected by "selector" will still be available after + the eviction, i.e. even in the absence of the evicted pod. So + for example you can prevent all voluntary evictions by specifying + "100%". + selector: + description: Label query over pods whose evictions are managed + by the disruption budget. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + type: object + type: object + remoteClusters: + description: RemoteClusters enables you to establish uni-directional + connections to a remote Elasticsearch cluster. + items: + description: RemoteCluster declares a remote Elasticsearch cluster + connection. + properties: + elasticsearchRef: + description: ElasticsearchRef is a reference to an Elasticsearch + cluster running within the same k8s cluster. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, + defaults to the current namespace. + type: string + required: + - name + type: object + name: + description: Name is the name of the remote cluster as it is set + in the Elasticsearch settings. The name is expected to be unique + for each remote clusters. + minLength: 1 + type: string + required: + - name + type: object + type: array + secureSettings: + description: SecureSettings is a list of references to Kubernetes secrets + containing sensitive configuration options for Elasticsearch. + items: + description: SecretSource defines a data source based on a Kubernetes + Secret. + properties: + entries: + description: Entries define how to project each key-value pair + in the secret to filesystem paths. If not defined, all keys + will be projected to similarly named paths in the filesystem. + If defined, only the specified keys will be projected to the + corresponding paths. + items: + description: KeyToPath defines how to map a key in a Secret + object to a filesystem path. + properties: + key: + description: Key is the key contained in the secret. + type: string + path: + description: Path is the relative file path to map the key + to. Path must not be an absolute file path and must not + contain any ".." components. + type: string + required: + - key + type: object + type: array + secretName: + description: SecretName is the name of the secret. + type: string + required: + - secretName + type: object + type: array + serviceAccountName: + description: ServiceAccountName is used to check access from the current + resource to a resource (eg. a remote Elasticsearch cluster) in a different + namespace. Can only be used if ECK is enforcing RBAC on references. + type: string + transport: + description: Transport holds transport layer settings for Elasticsearch. + properties: + service: + description: Service defines the template for the associated Kubernetes + Service object. + properties: + metadata: + description: ObjectMeta is the metadata of the service. The + name and namespace provided here are managed by ECK and will + be ignored. + type: object + spec: + description: Spec is the specification of the service. + properties: + clusterIP: + description: 'clusterIP is the IP address of the service + and is usually assigned randomly by the master. If an + address is specified manually and is not in use by others, + it will be allocated to the service; otherwise, creation + of the service will fail. This field can not be changed + through updates. Valid values are "None", empty string + (""), or a valid IP address. "None" can be specified for + headless services when proxying is not required. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this + service. These IPs are not managed by Kubernetes. The + user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external + load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that + kubedns or equivalent will return as a CNAME record for + this service. No proxying will be involved. Must be a + valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires Type to be ExternalName. + type: string + externalTrafficPolicy: + description: externalTrafficPolicy denotes if this Service + desires to route external traffic to node-local or cluster-wide + endpoints. "Local" preserves the client source IP and + avoids a second hop for LoadBalancer and Nodeport type + services, but risks potentially imbalanced traffic spreading. + "Cluster" obscures the client source IP and may cause + a second hop to another node, but should have good overall + load-spreading. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. If not specified, HealthCheckNodePort + is created by the service api backend with the allocated + nodePort. Will use user-specified nodePort value if specified + by the client. Only effects when Type is set to LoadBalancer + and ExternalTrafficPolicy is set to Local. + format: int32 + type: integer + ipFamily: + description: ipFamily specifies whether this Service has + a preference for a particular IP family (e.g. IPv4 vs. + IPv6). If a specific IP family is requested, the clusterIP + field will be allocated from that family, if it is available + in the cluster. If no IP family is requested, the cluster's + primary IP family will be used. Other IP fields (loadBalancerIP, + loadBalancerSourceRanges, externalIPs) and controllers + which allocate external load-balancers should use the + same IP family. Endpoints for this Service will be of + this family. This field is immutable after creation. + Assigning a ServiceIPFamily not available in the cluster + (e.g. IPv6 in IPv4 only cluster) is an error condition + and will fail during clusterIP assignment. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer + LoadBalancer will get created with the IP specified in + this field. This feature depends on whether the underlying + cloud-provider supports specifying the loadBalancerIP + when a load balancer is created. This field will be ignored + if the cloud-provider does not support the feature.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client + IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this + service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field + in the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this + service is exposed when type=NodePort or LoadBalancer. + Usually assigned by the system. If specified, it + will be allocated to the service if unused or else + creation of the service will fail. Default is to + auto-allocate a port if the ServiceType of this + Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this + service. + format: int32 + type: integer + protocol: + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access + on the pods targeted by the service. Number must + be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored + for services with clusterIP=None, and should be + omitted or set equal to the ''port'' field. More + info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + required: + - port + type: object + type: array + publishNotReadyAddresses: + description: publishNotReadyAddresses, when set to true, + indicates that DNS implementations must publish the notReadyAddresses + of subsets for the Endpoints associated with the Service. + The default value is false. The primary use case for setting + this field is to use a StatefulSet's Headless Service + to propagate SRV records for its Pods without respect + to their readiness for purpose of peer discovery. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: + https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of + Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + topologyKeys: + description: topologyKeys is a preference-order list of + topology keys which implementations of services should + use to preferentially sort endpoints when accessing this + Service, it can not be used at the same time as externalTrafficPolicy=Local. + Topology keys must be valid label keys and at most 16 + keys may be specified. Endpoints are chosen based on the + first topology key with available backends. If this field + is specified and all entries have no backends that match + the topology of the client, the service has no backends + for that client and connections should fail. The special + value "*" may be used to mean "any topology". This catch-all + value, if used, only makes sense as the last value in + the list. If this is not specified or empty, no topology + constraints will be applied. + items: + type: string + type: array + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ExternalName" + maps to the specified externalName. "ClusterIP" allocates + a cluster-internal IP address for load-balancing to endpoints. + Endpoints are determined by the selector or if that is + not specified, by manual construction of an Endpoints + object. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints + rather than a stable IP. "NodePort" builds on ClusterIP + and allocates a port on every node which routes to the + clusterIP. "LoadBalancer" builds on NodePort and creates + an external load-balancer (if supported in the current + cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + type: object + updateStrategy: + description: UpdateStrategy specifies how updates to the cluster should + be performed. + properties: + changeBudget: + description: ChangeBudget defines the constraints to consider when + applying changes to the Elasticsearch cluster. + properties: + maxSurge: + description: MaxSurge is the maximum number of new pods that + can be created exceeding the original number of pods defined + in the specification. MaxSurge is only taken into consideration + when scaling up. Setting a negative value will disable the + restriction. Defaults to unbounded if not specified. + format: int32 + type: integer + maxUnavailable: + description: MaxUnavailable is the maximum number of pods that + can be unavailable (not ready) during the update due to circumstances + under the control of the operator. Setting a negative value + will disable this restriction. Defaults to 1 if not specified. + format: int32 + type: integer + type: object + type: object + version: + description: Version of Elasticsearch. + type: string + required: + - nodeSets + - version + type: object + status: + description: ElasticsearchStatus defines the observed state of Elasticsearch + properties: + availableNodes: + format: int32 + type: integer + health: + description: ElasticsearchHealth is the health of the cluster as returned + by the health API. + type: string + phase: + description: ElasticsearchOrchestrationPhase is the phase Elasticsearch + is in from the controller point of view. + type: string + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true + - name: v1beta1 + served: true + storage: false + - name: v1alpha1 + served: false + storage: false +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: enterprisesearches.enterprisesearch.k8s.elastic.co +spec: + additionalPrinterColumns: + - JSONPath: .status.health + name: health + type: string + - JSONPath: .status.availableNodes + description: Available nodes + name: nodes + type: integer + - JSONPath: .spec.version + description: Enterprise Search version + name: version + type: string + - JSONPath: .metadata.creationTimestamp + name: age + type: date + group: enterprisesearch.k8s.elastic.co + names: + categories: + - elastic + kind: EnterpriseSearch + listKind: EnterpriseSearchList + plural: enterprisesearches + shortNames: + - ent + singular: enterprisesearch + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: EnterpriseSearch is a Kubernetes CRD to represent Enterprise Search. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EnterpriseSearchSpec holds the specification of an Enterprise + Search resource. + properties: + config: + description: Config holds the Enterprise Search configuration. + type: object + configRef: + description: ConfigRef contains a reference to an existing Kubernetes + Secret holding the Enterprise Search configuration. Configuration + settings are merged and have precedence over settings specified in + `config`. + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + count: + description: Count of Enterprise Search instances to deploy. + format: int32 + type: integer + elasticsearchRef: + description: ElasticsearchRef is a reference to the Elasticsearch cluster + running in the same Kubernetes cluster. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + required: + - name + type: object + http: + description: HTTP holds the HTTP layer configuration for Enterprise + Search resource. + properties: + service: + description: Service defines the template for the associated Kubernetes + Service object. + properties: + metadata: + description: ObjectMeta is the metadata of the service. The + name and namespace provided here are managed by ECK and will + be ignored. + type: object + spec: + description: Spec is the specification of the service. + properties: + clusterIP: + description: 'clusterIP is the IP address of the service + and is usually assigned randomly by the master. If an + address is specified manually and is not in use by others, + it will be allocated to the service; otherwise, creation + of the service will fail. This field can not be changed + through updates. Valid values are "None", empty string + (""), or a valid IP address. "None" can be specified for + headless services when proxying is not required. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this + service. These IPs are not managed by Kubernetes. The + user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external + load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that + kubedns or equivalent will return as a CNAME record for + this service. No proxying will be involved. Must be a + valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires Type to be ExternalName. + type: string + externalTrafficPolicy: + description: externalTrafficPolicy denotes if this Service + desires to route external traffic to node-local or cluster-wide + endpoints. "Local" preserves the client source IP and + avoids a second hop for LoadBalancer and Nodeport type + services, but risks potentially imbalanced traffic spreading. + "Cluster" obscures the client source IP and may cause + a second hop to another node, but should have good overall + load-spreading. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. If not specified, HealthCheckNodePort + is created by the service api backend with the allocated + nodePort. Will use user-specified nodePort value if specified + by the client. Only effects when Type is set to LoadBalancer + and ExternalTrafficPolicy is set to Local. + format: int32 + type: integer + ipFamily: + description: ipFamily specifies whether this Service has + a preference for a particular IP family (e.g. IPv4 vs. + IPv6). If a specific IP family is requested, the clusterIP + field will be allocated from that family, if it is available + in the cluster. If no IP family is requested, the cluster's + primary IP family will be used. Other IP fields (loadBalancerIP, + loadBalancerSourceRanges, externalIPs) and controllers + which allocate external load-balancers should use the + same IP family. Endpoints for this Service will be of + this family. This field is immutable after creation. + Assigning a ServiceIPFamily not available in the cluster + (e.g. IPv6 in IPv4 only cluster) is an error condition + and will fail during clusterIP assignment. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer + LoadBalancer will get created with the IP specified in + this field. This feature depends on whether the underlying + cloud-provider supports specifying the loadBalancerIP + when a load balancer is created. This field will be ignored + if the cloud-provider does not support the feature.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client + IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this + service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field + in the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this + service is exposed when type=NodePort or LoadBalancer. + Usually assigned by the system. If specified, it + will be allocated to the service if unused or else + creation of the service will fail. Default is to + auto-allocate a port if the ServiceType of this + Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this + service. + format: int32 + type: integer + protocol: + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access + on the pods targeted by the service. Number must + be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored + for services with clusterIP=None, and should be + omitted or set equal to the ''port'' field. More + info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + required: + - port + type: object + type: array + publishNotReadyAddresses: + description: publishNotReadyAddresses, when set to true, + indicates that DNS implementations must publish the notReadyAddresses + of subsets for the Endpoints associated with the Service. + The default value is false. The primary use case for setting + this field is to use a StatefulSet's Headless Service + to propagate SRV records for its Pods without respect + to their readiness for purpose of peer discovery. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: + https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of + Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + topologyKeys: + description: topologyKeys is a preference-order list of + topology keys which implementations of services should + use to preferentially sort endpoints when accessing this + Service, it can not be used at the same time as externalTrafficPolicy=Local. + Topology keys must be valid label keys and at most 16 + keys may be specified. Endpoints are chosen based on the + first topology key with available backends. If this field + is specified and all entries have no backends that match + the topology of the client, the service has no backends + for that client and connections should fail. The special + value "*" may be used to mean "any topology". This catch-all + value, if used, only makes sense as the last value in + the list. If this is not specified or empty, no topology + constraints will be applied. + items: + type: string + type: array + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ExternalName" + maps to the specified externalName. "ClusterIP" allocates + a cluster-internal IP address for load-balancing to endpoints. + Endpoints are determined by the selector or if that is + not specified, by manual construction of an Endpoints + object. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints + rather than a stable IP. "NodePort" builds on ClusterIP + and allocates a port on every node which routes to the + clusterIP. "LoadBalancer" builds on NodePort and creates + an external load-balancer (if supported in the current + cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + tls: + description: TLS defines options for configuring TLS for HTTP. + properties: + certificate: + description: "Certificate is a reference to a Kubernetes secret + that contains the certificate and private key for enabling + TLS. The referenced secret should contain the following: \n + - `ca.crt`: The certificate authority (optional). - `tls.crt`: + The certificate (or a chain). - `tls.key`: The private key + to the first certificate in the certificate chain." + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + selfSignedCertificate: + description: SelfSignedCertificate allows configuring the self-signed + certificate generated by the operator. + properties: + disabled: + description: Disabled indicates that the provisioning of + the self-signed certifcate should be disabled. + type: boolean + subjectAltNames: + description: SubjectAlternativeNames is a list of SANs to + include in the generated HTTP TLS certificate. + items: + description: SubjectAlternativeName represents a SAN entry + in a x509 certificate. + properties: + dns: + description: DNS is the DNS name of the subject. + type: string + ip: + description: IP is the IP address of the subject. + type: string + type: object + type: array + type: object + type: object + type: object + image: + description: Image is the Enterprise Search Docker image to deploy. + type: string + podTemplate: + description: PodTemplate provides customisation options (labels, annotations, + affinity rules, resource requests, and so on) for the Enterprise Search + pods. + type: object + serviceAccountName: + description: ServiceAccountName is used to check access from the current + resource to a resource (eg. Elasticsearch) in a different namespace. + Can only be used if ECK is enforcing RBAC on references. + type: string + version: + description: Version of Enterprise Search. + type: string + type: object + status: + description: EnterpriseSearchStatus defines the observed state of EnterpriseSearch + properties: + associationStatus: + description: Association is the status of any auto-linking to Elasticsearch + clusters. + type: string + availableNodes: + format: int32 + type: integer + health: + description: EnterpriseSearchHealth expresses the health of the Enterprise + Search instances. + type: string + service: + description: ExternalService is the name of the service associated to + the Enterprise Search Pods. + type: string + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: kibanas.kibana.k8s.elastic.co +spec: + additionalPrinterColumns: + - JSONPath: .status.health + name: health + type: string + - JSONPath: .status.availableNodes + description: Available nodes + name: nodes + type: integer + - JSONPath: .spec.version + description: Kibana version + name: version + type: string + - JSONPath: .metadata.creationTimestamp + name: age + type: date + group: kibana.k8s.elastic.co + names: + categories: + - elastic + kind: Kibana + listKind: KibanaList + plural: kibanas + shortNames: + - kb + singular: kibana + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: Kibana represents a Kibana resource in a Kubernetes cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KibanaSpec holds the specification of a Kibana instance. + properties: + config: + description: 'Config holds the Kibana configuration. See: https://www.elastic.co/guide/en/kibana/current/settings.html' + type: object + count: + description: Count of Kibana instances to deploy. + format: int32 + type: integer + elasticsearchRef: + description: ElasticsearchRef is a reference to an Elasticsearch cluster + running in the same Kubernetes cluster. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + required: + - name + type: object + http: + description: HTTP holds the HTTP layer configuration for Kibana. + properties: + service: + description: Service defines the template for the associated Kubernetes + Service object. + properties: + metadata: + description: ObjectMeta is the metadata of the service. The + name and namespace provided here are managed by ECK and will + be ignored. + type: object + spec: + description: Spec is the specification of the service. + properties: + clusterIP: + description: 'clusterIP is the IP address of the service + and is usually assigned randomly by the master. If an + address is specified manually and is not in use by others, + it will be allocated to the service; otherwise, creation + of the service will fail. This field can not be changed + through updates. Valid values are "None", empty string + (""), or a valid IP address. "None" can be specified for + headless services when proxying is not required. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this + service. These IPs are not managed by Kubernetes. The + user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external + load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that + kubedns or equivalent will return as a CNAME record for + this service. No proxying will be involved. Must be a + valid RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires Type to be ExternalName. + type: string + externalTrafficPolicy: + description: externalTrafficPolicy denotes if this Service + desires to route external traffic to node-local or cluster-wide + endpoints. "Local" preserves the client source IP and + avoids a second hop for LoadBalancer and Nodeport type + services, but risks potentially imbalanced traffic spreading. + "Cluster" obscures the client source IP and may cause + a second hop to another node, but should have good overall + load-spreading. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. If not specified, HealthCheckNodePort + is created by the service api backend with the allocated + nodePort. Will use user-specified nodePort value if specified + by the client. Only effects when Type is set to LoadBalancer + and ExternalTrafficPolicy is set to Local. + format: int32 + type: integer + ipFamily: + description: ipFamily specifies whether this Service has + a preference for a particular IP family (e.g. IPv4 vs. + IPv6). If a specific IP family is requested, the clusterIP + field will be allocated from that family, if it is available + in the cluster. If no IP family is requested, the cluster's + primary IP family will be used. Other IP fields (loadBalancerIP, + loadBalancerSourceRanges, externalIPs) and controllers + which allocate external load-balancers should use the + same IP family. Endpoints for this Service will be of + this family. This field is immutable after creation. + Assigning a ServiceIPFamily not available in the cluster + (e.g. IPv6 in IPv4 only cluster) is an error condition + and will fail during clusterIP assignment. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer + LoadBalancer will get created with the IP specified in + this field. This feature depends on whether the underlying + cloud-provider supports specifying the loadBalancerIP + when a load balancer is created. This field will be ignored + if the cloud-provider does not support the feature.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client + IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this + service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field + in the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this + service is exposed when type=NodePort or LoadBalancer. + Usually assigned by the system. If specified, it + will be allocated to the service if unused or else + creation of the service will fail. Default is to + auto-allocate a port if the ServiceType of this + Service requires one. More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this + service. + format: int32 + type: integer + protocol: + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access + on the pods targeted by the service. Number must + be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored + for services with clusterIP=None, and should be + omitted or set equal to the ''port'' field. More + info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + required: + - port + type: object + type: array + publishNotReadyAddresses: + description: publishNotReadyAddresses, when set to true, + indicates that DNS implementations must publish the notReadyAddresses + of subsets for the Endpoints associated with the Service. + The default value is false. The primary use case for setting + this field is to use a StatefulSet's Headless Service + to propagate SRV records for its Pods without respect + to their readiness for purpose of peer discovery. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: + https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of + Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + topologyKeys: + description: topologyKeys is a preference-order list of + topology keys which implementations of services should + use to preferentially sort endpoints when accessing this + Service, it can not be used at the same time as externalTrafficPolicy=Local. + Topology keys must be valid label keys and at most 16 + keys may be specified. Endpoints are chosen based on the + first topology key with available backends. If this field + is specified and all entries have no backends that match + the topology of the client, the service has no backends + for that client and connections should fail. The special + value "*" may be used to mean "any topology". This catch-all + value, if used, only makes sense as the last value in + the list. If this is not specified or empty, no topology + constraints will be applied. + items: + type: string + type: array + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ExternalName" + maps to the specified externalName. "ClusterIP" allocates + a cluster-internal IP address for load-balancing to endpoints. + Endpoints are determined by the selector or if that is + not specified, by manual construction of an Endpoints + object. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints + rather than a stable IP. "NodePort" builds on ClusterIP + and allocates a port on every node which routes to the + clusterIP. "LoadBalancer" builds on NodePort and creates + an external load-balancer (if supported in the current + cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + tls: + description: TLS defines options for configuring TLS for HTTP. + properties: + certificate: + description: "Certificate is a reference to a Kubernetes secret + that contains the certificate and private key for enabling + TLS. The referenced secret should contain the following: \n + - `ca.crt`: The certificate authority (optional). - `tls.crt`: + The certificate (or a chain). - `tls.key`: The private key + to the first certificate in the certificate chain." + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + selfSignedCertificate: + description: SelfSignedCertificate allows configuring the self-signed + certificate generated by the operator. + properties: + disabled: + description: Disabled indicates that the provisioning of + the self-signed certifcate should be disabled. + type: boolean + subjectAltNames: + description: SubjectAlternativeNames is a list of SANs to + include in the generated HTTP TLS certificate. + items: + description: SubjectAlternativeName represents a SAN entry + in a x509 certificate. + properties: + dns: + description: DNS is the DNS name of the subject. + type: string + ip: + description: IP is the IP address of the subject. + type: string + type: object + type: array + type: object + type: object + type: object + image: + description: Image is the Kibana Docker image to deploy. + type: string + podTemplate: + description: PodTemplate provides customisation options (labels, annotations, + affinity rules, resource requests, and so on) for the Kibana pods + type: object + secureSettings: + description: SecureSettings is a list of references to Kubernetes secrets + containing sensitive configuration options for Kibana. + items: + description: SecretSource defines a data source based on a Kubernetes + Secret. + properties: + entries: + description: Entries define how to project each key-value pair + in the secret to filesystem paths. If not defined, all keys + will be projected to similarly named paths in the filesystem. + If defined, only the specified keys will be projected to the + corresponding paths. + items: + description: KeyToPath defines how to map a key in a Secret + object to a filesystem path. + properties: + key: + description: Key is the key contained in the secret. + type: string + path: + description: Path is the relative file path to map the key + to. Path must not be an absolute file path and must not + contain any ".." components. + type: string + required: + - key + type: object + type: array + secretName: + description: SecretName is the name of the secret. + type: string + required: + - secretName + type: object + type: array + serviceAccountName: + description: ServiceAccountName is used to check access from the current + resource to a resource (eg. Elasticsearch) in a different namespace. + Can only be used if ECK is enforcing RBAC on references. + type: string + version: + description: Version of Kibana. + type: string + required: + - version + type: object + status: + description: KibanaStatus defines the observed state of Kibana + properties: + associationStatus: + description: AssociationStatus is the status of an association resource. + type: string + availableNodes: + format: int32 + type: integer + health: + description: KibanaHealth expresses the status of the Kibana instances. + type: string + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true + - name: v1beta1 + served: true + storage: false + - name: v1alpha1 + served: false + storage: false +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: elastic-webhook.k8s.elastic.co +webhooks: +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-apm-k8s-elastic-co-v1-apmserver + failurePolicy: Ignore + name: elastic-apm-validation-v1.k8s.elastic.co + rules: + - apiGroups: + - apm.k8s.elastic.co + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - apmservers +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-apm-k8s-elastic-co-v1beta1-apmserver + failurePolicy: Ignore + name: elastic-apm-validation-v1beta1.k8s.elastic.co + rules: + - apiGroups: + - apm.k8s.elastic.co + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - apmservers +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-beat-k8s-elastic-co-v1beta1-beat + failurePolicy: Ignore + name: elastic-beat-validation-v1beta1.k8s.elastic.co + rules: + - apiGroups: + - beat.k8s.elastic.co + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - beats +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-elasticsearch-k8s-elastic-co-v1-elasticsearch + failurePolicy: Ignore + name: elastic-es-validation-v1.k8s.elastic.co + rules: + - apiGroups: + - elasticsearch.k8s.elastic.co + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - elasticsearches +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-elasticsearch-k8s-elastic-co-v1beta1-elasticsearch + failurePolicy: Ignore + name: elastic-es-validation-v1beta1.k8s.elastic.co + rules: + - apiGroups: + - elasticsearch.k8s.elastic.co + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - elasticsearches +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-kibana-k8s-elastic-co-v1-kibana + failurePolicy: Ignore + name: elastic-kb-validation-v1.k8s.elastic.co + rules: + - apiGroups: + - kibana.k8s.elastic.co + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - kibanas +- clientConfig: + caBundle: Cg== + service: + name: elastic-webhook-server + namespace: elastic-system + path: /validate-kibana-k8s-elastic-co-v1beta1-kibana + failurePolicy: Ignore + name: elastic-kb-validation-v1beta1.k8s.elastic.co + rules: + - apiGroups: + - kibana.k8s.elastic.co + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - kibanas +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elastic-operator + namespace: elastic-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + name: elastic-operator-edit +rules: +- apiGroups: + - elasticsearch.k8s.elastic.co + resources: + - elasticsearches + verbs: + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - apm.k8s.elastic.co + resources: + - apmservers + verbs: + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - kibana.k8s.elastic.co + resources: + - kibanas + verbs: + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - enterprisesearch.k8s.elastic.co + resources: + - enterprisesearches + verbs: + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - beat.k8s.elastic.co + resources: + - beats + verbs: + - create + - delete + - deletecollection + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: elastic-operator-view +rules: +- apiGroups: + - elasticsearch.k8s.elastic.co + resources: + - elasticsearches + verbs: + - get + - list + - watch +- apiGroups: + - apm.k8s.elastic.co + resources: + - apmservers + verbs: + - get + - list + - watch +- apiGroups: + - kibana.k8s.elastic.co + resources: + - kibanas + verbs: + - get + - list + - watch +- apiGroups: + - enterprisesearch.k8s.elastic.co + resources: + - enterprisesearches + verbs: + - get + - list + - watch +- apiGroups: + - beat.k8s.elastic.co + resources: + - beats + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: elastic-operator +rules: +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - "" + resources: + - pods + - endpoints + - events + - persistentvolumeclaims + - secrets + - services + - configmaps + - serviceaccounts + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - apps + resources: + - deployments + - statefulsets + - daemonsets + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - elasticsearch.k8s.elastic.co + resources: + - elasticsearches + - elasticsearches/status + - elasticsearches/finalizers + - enterpriselicenses + - enterpriselicenses/status + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - kibana.k8s.elastic.co + resources: + - kibanas + - kibanas/status + - kibanas/finalizers + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - apm.k8s.elastic.co + resources: + - apmservers + - apmservers/status + - apmservers/finalizers + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - enterprisesearch.k8s.elastic.co + resources: + - enterprisesearches + - enterprisesearches/status + - enterprisesearches/finalizers + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - beat.k8s.elastic.co + resources: + - beats + - beats/status + - beats/finalizers + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: elastic-operator + namespace: elastic-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: elastic-operator +subjects: +- kind: ServiceAccount + name: elastic-operator + namespace: elastic-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: elastic-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: elastic-operator +subjects: +- kind: ServiceAccount + name: elastic-operator + namespace: elastic-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: elastic-webhook-server-cert + namespace: elastic-system +--- +apiVersion: v1 +kind: Service +metadata: + name: elastic-webhook-server + namespace: elastic-system +spec: + ports: + - name: https + port: 443 + targetPort: 9443 + selector: + control-plane: elastic-operator +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + control-plane: elastic-operator + name: elastic-operator + namespace: elastic-system +spec: + selector: + matchLabels: + control-plane: elastic-operator + serviceName: elastic-operator + template: + metadata: + annotations: + co.elastic.logs/raw: '[{"type":"container","json.keys_under_root":true,"paths":["/var/log/containers/*${data.kubernetes.container.id}.log"],"processors":[{"convert":{"mode":"rename","ignore_missing":true,"fields":[{"from":"error","to":"_error"}]}},{"convert":{"mode":"rename","ignore_missing":true,"fields":[{"from":"_error","to":"error.message"}]}},{"convert":{"mode":"rename","ignore_missing":true,"fields":[{"from":"source","to":"_source"}]}},{"convert":{"mode":"rename","ignore_missing":true,"fields":[{"from":"_source","to":"event.source"}]}}]}]' + labels: + control-plane: elastic-operator + spec: + containers: + - args: + - manager + - --log-verbosity=0 + - --metrics-port=0 + - --container-registry=docker.elastic.co + - --max-concurrent-reconciles=3 + - --ca-cert-validity=8760h + - --ca-cert-rotate-before=24h + - --cert-validity=8760h + - --cert-rotate-before=24h + - --enable-webhook + env: + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_IMAGE + value: docker.elastic.co/eck/eck-operator:1.2.0 + - name: WEBHOOK_SECRET + value: elastic-webhook-server-cert + image: docker.elastic.co/eck/eck-operator:1.2.0 + imagePullPolicy: IfNotPresent + name: manager + ports: + - containerPort: 9443 + name: https-webhook + protocol: TCP + resources: + limits: + cpu: 1 + memory: 512Mi + requests: + cpu: 100m + memory: 150Mi + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: elastic-operator + terminationGracePeriodSeconds: 10 + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: elastic-webhook-server-cert diff --git a/charts/kubezero-logging/templates/elasticsearch.yaml b/charts/kubezero-logging/templates/elasticsearch.yaml new file mode 100644 index 0000000..25ad994 --- /dev/null +++ b/charts/kubezero-logging/templates/elasticsearch.yaml @@ -0,0 +1,83 @@ +{{- if .Values.es.nodeSets }} +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: {{ template "kubezero-lib.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + version: {{ .Values.version }} + nodeSets: + {{- range .Values.es.nodeSets }} + - name: {{ .name }} + config: + node.master: true + node.data: true + node.ingest: true + node.ml: false + prometheus.indices: false + {{- if .zone }} + node.attr.zone: {{ .zone }} + cluster.routing.allocation.awareness.attributes: zone + {{- end }} + podTemplate: + {{- if $.Values.es.s3_snapshot_iamrole }} + metadata: + annotations: + iam.amazonaws.com/role: {{ $.Values.es.s3_snapshot_iamrole }} + {{- end }} + spec: + initContainers: + - name: install-plugins + command: + - sh + - -c + - | + bin/elasticsearch-plugin install --batch repository-s3 && bin/elasticsearch-plugin install --batch https://github.com/vvanholl/elasticsearch-prometheus-exporter/releases/download/{{ $.Values.version }}.0/prometheus-exporter-{{ $.Values.version }}.0.zip + containers: + - name: elasticsearch + resources: + requests: + cpu: 100m + memory: 2500Mi + limits: + memory: 4Gi + env: + - name: ES_JAVA_OPTS + value: "-Xms2g -Xmx2g" + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + elasticsearch.k8s.elastic.co/cluster-name: {{ $.Values.name }} + topologyKey: kubernetes.io/hostname + {{- if .zone }} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: failure-domain.beta.kubernetes.io/zone + operator: In + values: + - {{ .zone }} + {{- end }} + count: {{ .count }} + volumeClaimTemplates: + - metadata: + name: elasticsearch-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .storage.size }} + storageClassName: {{ .storage.class }} + {{- end }} + http: + tls: + selfSignedCertificate: + disabled: true +{{- end }} diff --git a/charts/kubezero-logging/templates/es-servicemonitor.yaml b/charts/kubezero-logging/templates/es-servicemonitor.yaml new file mode 100644 index 0000000..3a574b2 --- /dev/null +++ b/charts/kubezero-logging/templates/es-servicemonitor.yaml @@ -0,0 +1,20 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: es-{{ .Values.name }} + namespace: {{ .Release.Namespace }} +spec: + endpoints: + - basicAuth: + username: + name: {{ .Values.name }}-es-elastic-user + key: username + password: + name: {{ .Values.name }}-es-elastic-user + key: elastic + port: http + path: /_prometheus/metrics + selector: + matchLabels: + common.k8s.elastic.co/type: elasticsearch + elasticsearch.k8s.elastic.co/cluster-name: {{ .Values.name }} diff --git a/charts/kubezero-logging/templates/es-users.yaml b/charts/kubezero-logging/templates/es-users.yaml new file mode 100644 index 0000000..afbf188 --- /dev/null +++ b/charts/kubezero-logging/templates/es-users.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + labels: + common.k8s.elastic.co/type: elasticsearch + elasticsearch.k8s.elastic.co/cluster-name: {{ .Values.name }} + name: {{ .Values.name }}-es-elastic-user + namespace: {{ .Release.Namespace }} +data: + username: {{ "elastic" | b64enc | quote }} + elastic: {{ .Values.es.elastic_password | b64enc | quote }} diff --git a/charts/kubezero-logging/templates/istio-virtualservice.yaml b/charts/kubezero-logging/templates/istio-virtualservice.yaml new file mode 100644 index 0000000..205cba4 --- /dev/null +++ b/charts/kubezero-logging/templates/istio-virtualservice.yaml @@ -0,0 +1,18 @@ +{{- if .Values.kibana.istio.enabled }} +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: {{ template "kubezero-lib.fullname" . }}-kibana + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + hosts: + - {{ .Values.kibana.istio.url }} + gateways: + - {{ default "istio-system/ingressgateway" .Values.kibana.istio.gateway }} + http: + - route: + - destination: + host: {{ template "kubezero-lib.fullname" . }}-kb-http +{{- end }} diff --git a/charts/kubezero-logging/templates/kibana.yaml b/charts/kubezero-logging/templates/kibana.yaml new file mode 100644 index 0000000..f80539a --- /dev/null +++ b/charts/kubezero-logging/templates/kibana.yaml @@ -0,0 +1,35 @@ +{{- if .Values.kibana }} +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: {{ template "kubezero-lib.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + version: {{ .Values.version }} + count: {{ .Values.kibana.count }} + elasticsearchRef: + name: {{ template "kubezero-lib.fullname" . }} + namespace: {{ .Release.Namespace }} + config: + elasticsearch.requestTimeout: 180000 + elasticsearch.shardTimeout: 180000 + #xpack.monitoring.enabled: false + #xpack.monitoring.ui.enabled: false + #xpack.ml.enabled: false + podTemplate: + spec: + containers: + - name: kibana + resources: + requests: + memory: 1Gi + cpu: 100m + limits: + memory: 2Gi + http: + tls: + selfSignedCertificate: + disabled: true +{{- end }} diff --git a/charts/kubezero-logging/values-test.yaml b/charts/kubezero-logging/values-test.yaml new file mode 100644 index 0000000..f15d69a --- /dev/null +++ b/charts/kubezero-logging/values-test.yaml @@ -0,0 +1,26 @@ +# Default values for zdt-logging. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# This is for backwards compatibility with older zdt-logging setup +fullnameOverride: logging + +# Version for ElasticSearch and Kibana have to match so we define it at top-level +version: 7.6.0 + +es: + nodeSets: + - count: 2 + storage: + size: 16Gi + class: local-sc-xfs + zone: us-west-2a + s3_snapshot_iamrole: "" # INSERT_CLOUDFORMATION_OUTPUT_ElasticSearchSnapshots + elastic_password: "" # super_secret_elastic_password + +kibana: + replicas: 1 + istio: + enabled: true + url: kibana.example.com + gateway: istio-system/private-ingressgateway diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml new file mode 100644 index 0000000..5e43797 --- /dev/null +++ b/charts/kubezero-logging/values.yaml @@ -0,0 +1,27 @@ +# Default values for zdt-logging. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# This is for backwards compatibility with older zdt-logging setup +# fullnameOverride: logging + +# Version for ElasticSearch and Kibana have to match so we define it at top-level +version: 7.8.1 + +es: + nodeSets: [] + #- count: 2 + # storage: + # size: 16Gi + # class: local-sc-xfs + # zone: us-west-2a + s3_snapshot_iamrole: "" # INSERT_CLOUDFORMATION_OUTPUT_ElasticSearchSnapshots + elastic_password: "" # super_secret_elastic_password + +kibana: + #replicas: 1 + #servicename: kibana.example.com + istio: + enabled: false + gateway: "istio-system/ingressgateway" + url: "" # kibana.example.com -- 2.40.1 From 7eabb00e72b8f1344b4a389782304936e50189c1 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 13 Aug 2020 19:44:50 +0100 Subject: [PATCH 57/96] Various logging fixes to get a first version of ES and Kibana running --- charts/kubezero-logging/Chart.yaml | 4 ++-- charts/kubezero-logging/eck/all-in-one.yaml | 12 +++++----- charts/kubezero-logging/eck/update.sh | 2 +- .../templates/eck-operator.yaml | 12 +++++----- .../templates/elasticsearch.yaml | 13 +++++++--- .../templates/es-servicemonitor.yaml | 20 ---------------- .../templates/{es-users.yaml => secrets.yaml} | 6 +++-- .../templates/servicemonitor.yaml | 24 +++++++++++++++++++ charts/kubezero-logging/values-test.yaml | 8 +++++-- charts/kubezero-logging/values.yaml | 8 +++++-- 10 files changed, 65 insertions(+), 44 deletions(-) delete mode 100644 charts/kubezero-logging/templates/es-servicemonitor.yaml rename charts/kubezero-logging/templates/{es-users.yaml => secrets.yaml} (55%) create mode 100644 charts/kubezero-logging/templates/servicemonitor.yaml diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index 22e5aaf..87fc2f0 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.0.1 -appVersion: 1.2.0 +version: 0.0.2 +appVersion: 1.2.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-logging/eck/all-in-one.yaml b/charts/kubezero-logging/eck/all-in-one.yaml index 40cada8..e2a1574 100644 --- a/charts/kubezero-logging/eck/all-in-one.yaml +++ b/charts/kubezero-logging/eck/all-in-one.yaml @@ -1028,8 +1028,8 @@ spec: description: Image is the Elasticsearch Docker image to deploy. type: string nodeSets: - description: 'NodeSets allow specifying groups of Elasticsearch nodes - sharing the same configuration and Pod templates. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-orchestration.html' + description: NodeSets allow specifying groups of Elasticsearch nodes + sharing the same configuration and Pod templates. items: description: NodeSet is the specification for a group of Elasticsearch nodes sharing the same configuration and a Pod template. @@ -1054,12 +1054,12 @@ spec: the Pods belonging to this NodeSet. type: object volumeClaimTemplates: - description: 'VolumeClaimTemplates is a list of persistent volume + description: VolumeClaimTemplates is a list of persistent volume claims to be used by each Pod in this NodeSet. Every claim in this list must have a matching volumeMount in one of the containers defined in the PodTemplate. Items defined here take precedence over any default claims added by the operator with the same - name. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-volume-claim-templates.html' + name. items: description: PersistentVolumeClaim is a user's request for and claim to a persistent volume @@ -2829,7 +2829,7 @@ spec: terminationGracePeriodSeconds: 10 serviceAccountName: elastic-operator containers: - - image: "docker.elastic.co/eck/eck-operator:1.2.0" + - image: "docker.elastic.co/eck/eck-operator:1.2.1" imagePullPolicy: IfNotPresent name: manager args: @@ -2849,7 +2849,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: OPERATOR_IMAGE - value: "docker.elastic.co/eck/eck-operator:1.2.0" + value: "docker.elastic.co/eck/eck-operator:1.2.1" - name: WEBHOOK_SECRET value: "elastic-webhook-server-cert" resources: diff --git a/charts/kubezero-logging/eck/update.sh b/charts/kubezero-logging/eck/update.sh index e67214a..7083513 100755 --- a/charts/kubezero-logging/eck/update.sh +++ b/charts/kubezero-logging/eck/update.sh @@ -1,6 +1,6 @@ #!/bin/bash -ECK_VERSION=1.2.0 +ECK_VERSION=1.2.1 curl -o all-in-one.yaml https://download.elastic.co/downloads/eck/${ECK_VERSION}/all-in-one.yaml diff --git a/charts/kubezero-logging/templates/eck-operator.yaml b/charts/kubezero-logging/templates/eck-operator.yaml index 82cc137..54bb221 100644 --- a/charts/kubezero-logging/templates/eck-operator.yaml +++ b/charts/kubezero-logging/templates/eck-operator.yaml @@ -1031,8 +1031,8 @@ spec: description: Image is the Elasticsearch Docker image to deploy. type: string nodeSets: - description: 'NodeSets allow specifying groups of Elasticsearch nodes - sharing the same configuration and Pod templates. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-orchestration.html' + description: NodeSets allow specifying groups of Elasticsearch nodes + sharing the same configuration and Pod templates. items: description: NodeSet is the specification for a group of Elasticsearch nodes sharing the same configuration and a Pod template. @@ -1057,12 +1057,12 @@ spec: the Pods belonging to this NodeSet. type: object volumeClaimTemplates: - description: 'VolumeClaimTemplates is a list of persistent volume + description: VolumeClaimTemplates is a list of persistent volume claims to be used by each Pod in this NodeSet. Every claim in this list must have a matching volumeMount in one of the containers defined in the PodTemplate. Items defined here take precedence over any default claims added by the operator with the same - name. See: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-volume-claim-templates.html' + name. items: description: PersistentVolumeClaim is a user's request for and claim to a persistent volume @@ -3021,10 +3021,10 @@ spec: fieldRef: fieldPath: metadata.namespace - name: OPERATOR_IMAGE - value: docker.elastic.co/eck/eck-operator:1.2.0 + value: docker.elastic.co/eck/eck-operator:1.2.1 - name: WEBHOOK_SECRET value: elastic-webhook-server-cert - image: docker.elastic.co/eck/eck-operator:1.2.0 + image: docker.elastic.co/eck/eck-operator:1.2.1 imagePullPolicy: IfNotPresent name: manager ports: diff --git a/charts/kubezero-logging/templates/elasticsearch.yaml b/charts/kubezero-logging/templates/elasticsearch.yaml index 25ad994..94b8f1f 100644 --- a/charts/kubezero-logging/templates/elasticsearch.yaml +++ b/charts/kubezero-logging/templates/elasticsearch.yaml @@ -25,16 +25,23 @@ spec: {{- if $.Values.es.s3_snapshot_iamrole }} metadata: annotations: - iam.amazonaws.com/role: {{ $.Values.es.s3_snapshot_iamrole }} + iam.amazonaws.com/role: {{ $.Values.es.s3Snapshot.iamrole }} {{- end }} spec: + {{- if or $.Values.es.prometheus $.Values.es.s3Snapshot.enabled }} initContainers: - name: install-plugins command: - sh - -c - | - bin/elasticsearch-plugin install --batch repository-s3 && bin/elasticsearch-plugin install --batch https://github.com/vvanholl/elasticsearch-prometheus-exporter/releases/download/{{ $.Values.version }}.0/prometheus-exporter-{{ $.Values.version }}.0.zip + {{- if $.Values.es.s3Snapshot.enabled }} + bin/elasticsearch-plugin install --batch repository-s3 + {{- end }} + {{- if $.Values.es.prometheus }} + bin/elasticsearch-plugin install --batch https://github.com/vvanholl/elasticsearch-prometheus-exporter/releases/download/{{ $.Values.version }}.0/prometheus-exporter-{{ $.Values.version }}.0.zip + {{- end }} + {{- end }} containers: - name: elasticsearch resources: @@ -52,7 +59,7 @@ spec: - podAffinityTerm: labelSelector: matchLabels: - elasticsearch.k8s.elastic.co/cluster-name: {{ $.Values.name }} + elasticsearch.k8s.elastic.co/cluster-name: {{ template "kubezero-lib.fullname" $ }} topologyKey: kubernetes.io/hostname {{- if .zone }} nodeAffinity: diff --git a/charts/kubezero-logging/templates/es-servicemonitor.yaml b/charts/kubezero-logging/templates/es-servicemonitor.yaml deleted file mode 100644 index 3a574b2..0000000 --- a/charts/kubezero-logging/templates/es-servicemonitor.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: es-{{ .Values.name }} - namespace: {{ .Release.Namespace }} -spec: - endpoints: - - basicAuth: - username: - name: {{ .Values.name }}-es-elastic-user - key: username - password: - name: {{ .Values.name }}-es-elastic-user - key: elastic - port: http - path: /_prometheus/metrics - selector: - matchLabels: - common.k8s.elastic.co/type: elasticsearch - elasticsearch.k8s.elastic.co/cluster-name: {{ .Values.name }} diff --git a/charts/kubezero-logging/templates/es-users.yaml b/charts/kubezero-logging/templates/secrets.yaml similarity index 55% rename from charts/kubezero-logging/templates/es-users.yaml rename to charts/kubezero-logging/templates/secrets.yaml index afbf188..eb7d395 100644 --- a/charts/kubezero-logging/templates/es-users.yaml +++ b/charts/kubezero-logging/templates/secrets.yaml @@ -4,9 +4,11 @@ type: Opaque metadata: labels: common.k8s.elastic.co/type: elasticsearch - elasticsearch.k8s.elastic.co/cluster-name: {{ .Values.name }} - name: {{ .Values.name }}-es-elastic-user + elasticsearch.k8s.elastic.co/cluster-name: {{ template "kubezero-lib.fullname" $ }} + name: {{ template "kubezero-lib.fullname" $ }}-elastic-user namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} data: username: {{ "elastic" | b64enc | quote }} elastic: {{ .Values.es.elastic_password | b64enc | quote }} diff --git a/charts/kubezero-logging/templates/servicemonitor.yaml b/charts/kubezero-logging/templates/servicemonitor.yaml new file mode 100644 index 0000000..208b31c --- /dev/null +++ b/charts/kubezero-logging/templates/servicemonitor.yaml @@ -0,0 +1,24 @@ +{{- if .Values.es.prometheus }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kubezero-lib.fullname" . }}-es + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + endpoints: + - basicAuth: + username: + name: {{ template "kubezero-lib.fullname" $ }}-es-elastic-user + key: username + password: + name: {{ template "kubezero-lib.fullname" $ }}-es-elastic-user + key: elastic + port: http + path: /_prometheus/metrics + selector: + matchLabels: + common.k8s.elastic.co/type: elasticsearch + elasticsearch.k8s.elastic.co/cluster-name: {{ template "kubezero-lib.fullname" $ }} +{{- end }} diff --git a/charts/kubezero-logging/values-test.yaml b/charts/kubezero-logging/values-test.yaml index f15d69a..b67a8f9 100644 --- a/charts/kubezero-logging/values-test.yaml +++ b/charts/kubezero-logging/values-test.yaml @@ -15,8 +15,12 @@ es: size: 16Gi class: local-sc-xfs zone: us-west-2a - s3_snapshot_iamrole: "" # INSERT_CLOUDFORMATION_OUTPUT_ElasticSearchSnapshots - elastic_password: "" # super_secret_elastic_password + s3Snapshot: + enabled: true + iamrole: "dfsf" # INSERT_CLOUDFORMATION_OUTPUT_ElasticSearchSnapshots + elastic_password: "dsfsfs" # super_secret_elastic_password + + prometheus: true kibana: replicas: 1 diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 5e43797..a0304fb 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This is for backwards compatibility with older zdt-logging setup -# fullnameOverride: logging +# fullnameOverride: "" # Version for ElasticSearch and Kibana have to match so we define it at top-level version: 7.8.1 @@ -15,9 +15,13 @@ es: # size: 16Gi # class: local-sc-xfs # zone: us-west-2a - s3_snapshot_iamrole: "" # INSERT_CLOUDFORMATION_OUTPUT_ElasticSearchSnapshots + s3Snapshot: + enabled: false + iamrole: "" # INSERT_CLOUDFORMATION_OUTPUT_ElasticSearchSnapshots elastic_password: "" # super_secret_elastic_password + prometheus: false + kibana: #replicas: 1 #servicename: kibana.example.com -- 2.40.1 From 4f4fa92b6b6ed074fab03849913a1d98e95016c4 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 14 Aug 2020 15:52:10 +0100 Subject: [PATCH 58/96] More logging fixes... ready for first trial --- .../kubezero-logging/templates/elasticsearch.yaml | 6 +++--- charts/kubezero-logging/templates/secrets.yaml | 2 +- charts/kubezero-logging/values-test.yaml | 8 ++++---- charts/kubezero-logging/values.yaml | 2 +- charts/kubezero/Chart.yaml | 2 +- charts/kubezero/templates/logging.yaml | 13 +++++++++++++ charts/kubezero/values-all.yaml | 6 ++++++ charts/kubezero/values.yaml | 3 +++ deploy/deploy.sh | 4 ++-- deploy/templates/values.yaml | 8 ++++++-- deploy/values.yaml | 3 +-- 11 files changed, 41 insertions(+), 16 deletions(-) create mode 100644 charts/kubezero/templates/logging.yaml diff --git a/charts/kubezero-logging/templates/elasticsearch.yaml b/charts/kubezero-logging/templates/elasticsearch.yaml index 94b8f1f..f99ba51 100644 --- a/charts/kubezero-logging/templates/elasticsearch.yaml +++ b/charts/kubezero-logging/templates/elasticsearch.yaml @@ -22,7 +22,7 @@ spec: cluster.routing.allocation.awareness.attributes: zone {{- end }} podTemplate: - {{- if $.Values.es.s3_snapshot_iamrole }} + {{- if $.Values.es.s3Snapshot.iamrole }} metadata: annotations: iam.amazonaws.com/role: {{ $.Values.es.s3Snapshot.iamrole }} @@ -36,10 +36,10 @@ spec: - -c - | {{- if $.Values.es.s3Snapshot.enabled }} - bin/elasticsearch-plugin install --batch repository-s3 + bin/elasticsearch-plugin install --batch repository-s3; {{- end }} {{- if $.Values.es.prometheus }} - bin/elasticsearch-plugin install --batch https://github.com/vvanholl/elasticsearch-prometheus-exporter/releases/download/{{ $.Values.version }}.0/prometheus-exporter-{{ $.Values.version }}.0.zip + bin/elasticsearch-plugin install --batch https://github.com/vvanholl/elasticsearch-prometheus-exporter/releases/download/{{ $.Values.version }}.0/prometheus-exporter-{{ $.Values.version }}.0.zip; {{- end }} {{- end }} containers: diff --git a/charts/kubezero-logging/templates/secrets.yaml b/charts/kubezero-logging/templates/secrets.yaml index eb7d395..7dd7920 100644 --- a/charts/kubezero-logging/templates/secrets.yaml +++ b/charts/kubezero-logging/templates/secrets.yaml @@ -5,7 +5,7 @@ metadata: labels: common.k8s.elastic.co/type: elasticsearch elasticsearch.k8s.elastic.co/cluster-name: {{ template "kubezero-lib.fullname" $ }} - name: {{ template "kubezero-lib.fullname" $ }}-elastic-user + name: {{ template "kubezero-lib.fullname" $ }}-es-elastic-user namespace: {{ .Release.Namespace }} labels: {{ include "kubezero-lib.labels" . | indent 4 }} diff --git a/charts/kubezero-logging/values-test.yaml b/charts/kubezero-logging/values-test.yaml index b67a8f9..077be87 100644 --- a/charts/kubezero-logging/values-test.yaml +++ b/charts/kubezero-logging/values-test.yaml @@ -10,10 +10,11 @@ version: 7.6.0 es: nodeSets: - - count: 2 + - name: default-zone-0 + count: 2 storage: - size: 16Gi - class: local-sc-xfs + size: 512Gi + class: ebs-sc-gp2-xfs zone: us-west-2a s3Snapshot: enabled: true @@ -23,7 +24,6 @@ es: prometheus: true kibana: - replicas: 1 istio: enabled: true url: kibana.example.com diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index a0304fb..b4d00bb 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -23,7 +23,7 @@ es: prometheus: false kibana: - #replicas: 1 + count: 1 #servicename: kibana.example.com istio: enabled: false diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index ef98338..4fbf73b 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero ArgoCD Application - Root App of Apps chart of KubeZero type: application -version: 0.4.2 +version: 0.4.3 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero/templates/logging.yaml b/charts/kubezero/templates/logging.yaml new file mode 100644 index 0000000..d696c42 --- /dev/null +++ b/charts/kubezero/templates/logging.yaml @@ -0,0 +1,13 @@ +{{- if index .Values "logging" "enabled" }} +{{ template "kubezero-app.app" dict "root" . "name" "logging" "type" "helm" "namespace" "logging"}} + syncPolicy: + automated: + prune: true +--- +apiVersion: v1 +kind: Namespace +metadata: + name: logging + annotations: + iam.amazonaws.com/permitted: ".*ElasticSearchSnapshots.*" +{{- end }} diff --git a/charts/kubezero/values-all.yaml b/charts/kubezero/values-all.yaml index 70a0a95..e99e14c 100644 --- a/charts/kubezero/values-all.yaml +++ b/charts/kubezero/values-all.yaml @@ -33,3 +33,9 @@ aws-efs-csi-driver: istio: enabled: true + +metrics: + enabled: true + +logging: + enabled: true diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 2c486f1..87248f7 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -38,3 +38,6 @@ istio: metrics: enabled: false + +logging: + enabled: false diff --git a/deploy/deploy.sh b/deploy/deploy.sh index d20ee6e..2e8963c 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -41,7 +41,7 @@ EOF fi # Deploy initial argo-cad - helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set cert-manager.not_ready=true --set istio.enabled=false --set metrics.enabled=false > generated-values.yaml + helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set cert-manager.not_ready=true --set istio.enabled=false --set metrics.enabled=false --set logging.enabled=false > generated-values.yaml helm install -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml # Wait for argocd-server to be running kubectl rollout status deployment -n argocd kubezero-argocd-server @@ -55,7 +55,7 @@ EOF if [ -f cert-manager-backup.yaml ]; then kubectl apply -f cert-manager-backup.yaml else - helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set istio.enabled=false --set metrics.enabled=false > generated-values.yaml + helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set istio.enabled=false --set metrics.enabled=false --set logging.enabled=false > generated-values.yaml helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml wait_for kubectl get Issuer -n kube-system kubezero-local-ca-issuer 2>/dev/null 1>&2 wait_for kubectl get ClusterIssuer letsencrypt-dns-prod 2>/dev/null 1>&2 diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 2b1409a..47752d3 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -9,7 +9,7 @@ kubezero: values: network: {{ default "vxlan" .Values.calico.network }} mtu: {{ default "8941" .Values.calico.mtu }} - prometheus: {{ .Values.prometheus.enabled }} + prometheus: {{ .Values.metrics.enabled }} cert-manager: enabled: {{ index .Values "cert-manager" "enabled" }} values: @@ -59,7 +59,7 @@ kubezero: replicas: {{ ternary 2 1 .Values.HighAvailableControlplane }} prometheus: servicemonitor: - enabled: {{ .Values.prometheus.enabled }} + enabled: {{ .Values.metrics.enabled }} {{- end }} {{- if not .Values.kiam.not_ready }} @@ -123,6 +123,10 @@ kubezero: {{- end }} {{- end }} + logging: + enabled: {{ .Values.logging.enabled }} + values: + argo-cd: {{- with index .Values "argo-cd" "server" }} server: diff --git a/deploy/values.yaml b/deploy/values.yaml index ea9abff..65cb531 100644 --- a/deploy/values.yaml +++ b/deploy/values.yaml @@ -33,8 +33,7 @@ metrics: prometheus: enabled: false -# Deprecated once metrics goes live -prometheus: +logging: enabled: false argo-cd: -- 2.40.1 From f11e98d69e843f826fb30aa81ec426745317a836 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 14 Aug 2020 17:05:25 +0100 Subject: [PATCH 59/96] Bugfix release for Calico, README updates --- charts/kubezero-aws-ebs-csi-driver/README.md | 4 +-- charts/kubezero-aws-efs-csi-driver/README.md | 4 +-- charts/kubezero-calico/Chart.yaml | 4 +-- charts/kubezero-calico/README.md | 4 +-- charts/kubezero-calico/templates/calico.yaml | 12 +++---- .../templates/servicemonitor.yaml | 2 +- charts/kubezero-cert-manager/README.md | 4 +-- charts/kubezero-istio/README.md | 5 +-- charts/kubezero-kiam/README.md | 4 +-- .../README.md | 4 +-- charts/kubezero-logging/README.md | 32 +++++++++++++------ charts/kubezero-metrics/README.md | 3 +- charts/kubezero/README.md | 5 +-- deploy/deploy.sh | 10 +++--- deploy/templates/values.yaml | 1 - 15 files changed, 54 insertions(+), 44 deletions(-) diff --git a/charts/kubezero-aws-ebs-csi-driver/README.md b/charts/kubezero-aws-ebs-csi-driver/README.md index 0ed9f44..ea60547 100644 --- a/charts/kubezero-aws-ebs-csi-driver/README.md +++ b/charts/kubezero-aws-ebs-csi-driver/README.md @@ -2,7 +2,7 @@ kubezero-aws-ebs-csi-driver =========================== KubeZero Umbrella Chart for aws-ebs-csi-driver -Current chart version is `0.2.0` +Current chart version is `0.3.0` Source code can be found [here](https://kubezero.com) @@ -10,7 +10,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| -| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## IAM Role If you use kiam or kube2iam and restrict access on nodes running this controller please adjust: diff --git a/charts/kubezero-aws-efs-csi-driver/README.md b/charts/kubezero-aws-efs-csi-driver/README.md index 6bcab80..c506516 100644 --- a/charts/kubezero-aws-efs-csi-driver/README.md +++ b/charts/kubezero-aws-efs-csi-driver/README.md @@ -2,7 +2,7 @@ kubezero-aws-efs-csi-driver =========================== KubeZero Umbrella Chart for aws-efs-csi-driver -Current chart version is `0.1.0` +Current chart version is `0.1.1` Source code can be found [here](https://kubezero.com) @@ -10,7 +10,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| -| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Storage Class Optionally creates the *efs-cs* storage class. diff --git a/charts/kubezero-calico/Chart.yaml b/charts/kubezero-calico/Chart.yaml index afc4384..1bd20e0 100644 --- a/charts/kubezero-calico/Chart.yaml +++ b/charts/kubezero-calico/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-calico description: KubeZero Umbrella Chart for Calico type: application -version: 0.1.8 -appVersion: 3.15 +version: 0.1.9 +appVersion: 3.15.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-calico/README.md b/charts/kubezero-calico/README.md index 9d495c8..3fea3a1 100644 --- a/charts/kubezero-calico/README.md +++ b/charts/kubezero-calico/README.md @@ -2,7 +2,7 @@ kubezero-calico =============== KubeZero Umbrella Chart for Calico -Current chart version is `0.1.7` +Current chart version is `0.1.9` Source code can be found [here](https://kubezero.com) @@ -10,7 +10,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| -| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## KubeZero default configuration diff --git a/charts/kubezero-calico/templates/calico.yaml b/charts/kubezero-calico/templates/calico.yaml index 307ba50..63c4efc 100644 --- a/charts/kubezero-calico/templates/calico.yaml +++ b/charts/kubezero-calico/templates/calico.yaml @@ -322,10 +322,6 @@ spec: spec: nodeSelector: kubernetes.io/os: linux - {{- if .Values.migration }} - # Only run Calico on nodes that have been migrated. - projectcalico.org/node-network-during-migration: calico - {{- end }} hostNetwork: true tolerations: # Make sure calico-node gets scheduled on all nodes. @@ -345,7 +341,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: calico/cni:v3.15.0 + image: calico/cni:v3.15.1 command: ["/install-cni.sh"] env: # Name of the CNI config file to create. @@ -381,7 +377,7 @@ spec: # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes # to communicate with Felix over the Policy Sync API. - name: flexvol-driver - image: calico/pod2daemon-flexvol:v3.15.0 + image: calico/pod2daemon-flexvol:v3.15.1 volumeMounts: - name: flexvol-driver-host mountPath: /host/driver @@ -392,7 +388,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: calico/node:v3.15.0 + image: calico/node:v3.15.1 env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE @@ -594,7 +590,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: calico/kube-controllers:v3.15.0 + image: calico/kube-controllers:v3.15.1 env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS diff --git a/charts/kubezero-calico/templates/servicemonitor.yaml b/charts/kubezero-calico/templates/servicemonitor.yaml index 56c8f42..130d1bf 100644 --- a/charts/kubezero-calico/templates/servicemonitor.yaml +++ b/charts/kubezero-calico/templates/servicemonitor.yaml @@ -5,7 +5,7 @@ metadata: name: calico-node labels: k8s-app: calico-node - prometheus: kube-prometheus + release: metrics spec: jobLabel: k8s-app selector: diff --git a/charts/kubezero-cert-manager/README.md b/charts/kubezero-cert-manager/README.md index e341dfd..b80cfcb 100644 --- a/charts/kubezero-cert-manager/README.md +++ b/charts/kubezero-cert-manager/README.md @@ -2,7 +2,7 @@ kubezero-cert-manager ===================== KubeZero Umbrella Chart for cert-manager -Current chart version is `0.3.5` +Current chart version is `0.3.6` Source code can be found [here](https://kubezero.com) @@ -11,7 +11,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| | https://charts.jetstack.io | cert-manager | 0.15.1 | -| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## AWS - IAM Role If you use kiam or kube2iam and restrict access on nodes running cert-manager please adjust: diff --git a/charts/kubezero-istio/README.md b/charts/kubezero-istio/README.md index 6b7ba4e..fa4e298 100644 --- a/charts/kubezero-istio/README.md +++ b/charts/kubezero-istio/README.md @@ -5,7 +5,7 @@ KubeZero Umbrella Chart for Istio Installs Istio Operator and KubeZero Istio profile -Current chart version is `0.2.1` +Current chart version is `0.2.3` Source code can be found [here](https://kubezero.com) @@ -24,7 +24,8 @@ Source code can be found [here](https://kubezero.com) | Key | Type | Default | Description | |-----|------|---------|-------------| | ingress.autoscaleEnabled | bool | `false` | | -| ingress.private | bool | `true` | | +| ingress.private.enabled | bool | `true` | | +| ingress.private.nodeSelector | string | `"31080_31443_30671_30672_31224"` | | | ingress.replicaCount | int | `2` | | | ingress.type | string | `"NodePort"` | | | istio-operator.hub | string | `"docker.io/istio"` | | diff --git a/charts/kubezero-kiam/README.md b/charts/kubezero-kiam/README.md index 9d4ca94..d45bbd7 100644 --- a/charts/kubezero-kiam/README.md +++ b/charts/kubezero-kiam/README.md @@ -2,7 +2,7 @@ kubezero-kiam ============= KubeZero Umbrella Chart for Kiam -Current chart version is `0.2.6` +Current chart version is `0.2.7` Source code can be found [here](https://kubezero.com) @@ -11,7 +11,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| | https://uswitch.github.io/kiam-helm-charts/charts/ | kiam | 5.8.1 | -| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## KubeZero default configuration We run agents on the controllers as well, so we force eg. ebs csi controllers and others to assume roles etc. diff --git a/charts/kubezero-local-volume-provisioner/README.md b/charts/kubezero-local-volume-provisioner/README.md index b307058..836ff85 100644 --- a/charts/kubezero-local-volume-provisioner/README.md +++ b/charts/kubezero-local-volume-provisioner/README.md @@ -4,7 +4,7 @@ KubeZero Umbrella Chart for local-static-provisioner Provides persistent volumes backed by local volumes, eg. additional SSDs or spindles. -Current chart version is `0.0.1` +Current chart version is `0.1.0` Source code can be found [here](https://kubezero.com) @@ -12,7 +12,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| -| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## KubeZero default configuration diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md index ad980cc..0635ffe 100644 --- a/charts/kubezero-logging/README.md +++ b/charts/kubezero-logging/README.md @@ -2,7 +2,7 @@ kubezero-logging ================ KubeZero Umbrella Chart for complete EFK stack -Current chart version is `0.0.1` +Current chart version is `0.0.2` Source code can be found [here](https://kubezero.com) @@ -16,30 +16,42 @@ Source code can be found [here](https://kubezero.com) ### ECK - Operator mapped to controller nodes -### ElasticSearch +### ES + +- SSL disabled ( Todo: provide cluster certs and setup Kibana/Fluentd to use https incl. client certs ) - Installed Plugins: - - repository-s3 - - elasticsearch-prometheus-exporter + - repository-s3 + - elasticsearch-prometheus-exporter - [Cross AZ Zone awareness](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-advanced-node-scheduling.html#k8s-availability-zone-awareness) is implemented via nodeSets +### Kibana + +- increased timeout to ES to 3 minutes + + ## Manual tasks ATM - Install index template - setup Kibana - create `logstash-*` Index Pattern + ## Chart Values | Key | Type | Default | Description | |-----|------|---------|-------------| -| es.replicas | int | `2` | | -| es.storage.class | string | `"local-sc-xfs"` | | -| es.storage.size | string | `"16Gi"` | | -| fullnameOverride | string | `"logging"` | | -| kibana.replicas | int | `1` | | -| version | string | `"7.6.0"` | | +| es.elastic_password | string | `""` | | +| es.nodeSets | list | `[]` | | +| es.prometheus | bool | `false` | | +| es.s3Snapshot.enabled | bool | `false` | | +| es.s3Snapshot.iamrole | string | `""` | | +| kibana.count | int | `1` | | +| kibana.istio.enabled | bool | `false` | | +| kibana.istio.gateway | string | `"istio-system/ingressgateway"` | | +| kibana.istio.url | string | `""` | | +| version | string | `"7.8.1"` | | ## Resources: diff --git a/charts/kubezero-metrics/README.md b/charts/kubezero-metrics/README.md index 62b7040..4275afb 100644 --- a/charts/kubezero-metrics/README.md +++ b/charts/kubezero-metrics/README.md @@ -2,7 +2,7 @@ kubezero-metrics ================ KubeZero Umbrella Chart for prometheus-operator -Current chart version is `0.1.2` +Current chart version is `0.1.3` Source code can be found [here](https://kubezero.com) @@ -87,6 +87,7 @@ Source code can be found [here](https://kubezero.com) | prometheus-operator.prometheusOperator.enabled | bool | `true` | | | prometheus-operator.prometheusOperator.manageCrds | bool | `false` | | | prometheus-operator.prometheusOperator.namespaces.additional[0] | string | `"kube-system"` | | +| prometheus-operator.prometheusOperator.namespaces.additional[1] | string | `"logging"` | | | prometheus-operator.prometheusOperator.namespaces.releaseNamespace | bool | `true` | | | prometheus-operator.prometheusOperator.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | prometheus-operator.prometheusOperator.tlsProxy.enabled | bool | `false` | | diff --git a/charts/kubezero/README.md b/charts/kubezero/README.md index 87b4b43..b41419d 100644 --- a/charts/kubezero/README.md +++ b/charts/kubezero/README.md @@ -2,7 +2,7 @@ kubezero ======== KubeZero ArgoCD Application - Root App of Apps chart of KubeZero -Current chart version is `0.4.1` +Current chart version is `0.4.3` Source code can be found [here](https://kubezero.com) @@ -10,7 +10,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| -| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Chart Values @@ -27,5 +27,6 @@ Source code can be found [here](https://kubezero.com) | istio.enabled | bool | `false` | | | kiam.enabled | bool | `false` | | | local-volume-provisioner.enabled | bool | `false` | | +| logging.enabled | bool | `false` | | | metrics.enabled | bool | `false` | | | platform | string | `"aws"` | | diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 2e8963c..2961135 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -56,7 +56,7 @@ EOF kubectl apply -f cert-manager-backup.yaml else helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set istio.enabled=false --set metrics.enabled=false --set logging.enabled=false > generated-values.yaml - helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml + helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd -f generated-values.yaml wait_for kubectl get Issuer -n kube-system kubezero-local-ca-issuer 2>/dev/null 1>&2 wait_for kubectl get ClusterIssuer letsencrypt-dns-prod 2>/dev/null 1>&2 kubectl wait --for=condition=Ready -n kube-system Issuer/kubezero-local-ca-issuer @@ -64,12 +64,12 @@ EOF fi # Now that we have the cert-manager webhook, get the kiam certs in place but do NOT deploy kiam yet - helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set kiam.enabled=false > generated-values.yaml - helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml + helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set kiam.enabled=false --set istio.enabled=false --set metrics.enabled=false --set logging.enabled=false > generated-values.yaml + helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd -f generated-values.yaml # Now lets make sure kiam is working - helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true > generated-values.yaml - helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml + helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set istio.enabled=false --set metrics.enabled=false --set logging.enabled=false > generated-values.yaml + helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd -f generated-values.yaml wait_for kubectl get daemonset -n kube-system kiam-agent 2>/dev/null 1>&2 kubectl rollout status daemonset -n kube-system kiam-agent diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 47752d3..86b4763 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -5,7 +5,6 @@ kubezero: {{- end }} calico: enabled: {{ .Values.calico.enabled }} - type: {{ default "kustomize" .Values.calico.type }} values: network: {{ default "vxlan" .Values.calico.network }} mtu: {{ default "8941" .Values.calico.mtu }} -- 2.40.1 From 30f6432e59404f7c6e60f47b92124a7a87accd82 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 14 Aug 2020 17:10:25 +0100 Subject: [PATCH 60/96] Bugfix for prometheus service for calico --- charts/kubezero-calico/templates/service.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/kubezero-calico/templates/service.yaml b/charts/kubezero-calico/templates/service.yaml index 8d9b0a5..26ab6cb 100644 --- a/charts/kubezero-calico/templates/service.yaml +++ b/charts/kubezero-calico/templates/service.yaml @@ -1,4 +1,5 @@ {{- if .Values.prometheus }} +apiVersion: v1 kind: Service metadata: labels: -- 2.40.1 From a9cdc7109ebef0a8c5155eccae159676e885f27f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 14 Aug 2020 17:12:06 +0100 Subject: [PATCH 61/96] Add elastic-system ns to kubezero --- charts/kubezero-argo-cd/Chart.yaml | 2 +- charts/kubezero-argo-cd/templates/project.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-argo-cd/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml index 45a1a1a..fe1bc6d 100644 --- a/charts/kubezero-argo-cd/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application name: kubezero-argo-cd -version: 0.3.9 +version: 0.3.10 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-argo-cd/templates/project.yaml b/charts/kubezero-argo-cd/templates/project.yaml index 16faaba..8bf499f 100644 --- a/charts/kubezero-argo-cd/templates/project.yaml +++ b/charts/kubezero-argo-cd/templates/project.yaml @@ -26,6 +26,8 @@ spec: server: https://kubernetes.default.svc - namespace: monitoring server: https://kubernetes.default.svc + - namespace: elastic-system + server: https://kubernetes.default.svc - namespace: logging server: https://kubernetes.default.svc -- 2.40.1 From 55b0f0239452a9582202f9d712c8d238690b95f9 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 14 Aug 2020 17:39:05 +0100 Subject: [PATCH 62/96] Add proper label for kiam servicemonitors --- charts/kubezero-kiam/Chart.yaml | 2 +- charts/kubezero-kiam/README.md | 4 +++- charts/kubezero-kiam/values.yaml | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index 7bc63b8..eba7e46 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.2.7 +version: 0.2.8 appVersion: 3.6 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-kiam/README.md b/charts/kubezero-kiam/README.md index d45bbd7..223cc87 100644 --- a/charts/kubezero-kiam/README.md +++ b/charts/kubezero-kiam/README.md @@ -2,7 +2,7 @@ kubezero-kiam ============= KubeZero Umbrella Chart for Kiam -Current chart version is `0.2.7` +Current chart version is `0.2.8` Source code can be found [here](https://kubezero.com) @@ -41,6 +41,7 @@ By default all access to the meta-data service is blocked, expect for: | kiam.agent.image.tag | string | `"v3.6"` | | | kiam.agent.log.level | string | `"warn"` | | | kiam.agent.prometheus.servicemonitor.enabled | bool | `false` | | +| kiam.agent.prometheus.servicemonitor.labels.release | string | `"metrics"` | | | kiam.agent.sslCertHostPath | string | `"/etc/ssl/certs"` | | | kiam.agent.tlsCerts.caFileName | string | `"ca.crt"` | | | kiam.agent.tlsCerts.certFileName | string | `"tls.crt"` | | @@ -57,6 +58,7 @@ By default all access to the meta-data service is blocked, expect for: | kiam.server.log.level | string | `"warn"` | | | kiam.server.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | kiam.server.prometheus.servicemonitor.enabled | bool | `false` | | +| kiam.server.prometheus.servicemonitor.labels.release | string | `"metrics"` | | | kiam.server.service.port | int | `6444` | | | kiam.server.service.targetPort | int | `6444` | | | kiam.server.sslCertHostPath | string | `"/etc/ssl/certs"` | | diff --git a/charts/kubezero-kiam/values.yaml b/charts/kubezero-kiam/values.yaml index 4839a30..d8d4ac6 100644 --- a/charts/kubezero-kiam/values.yaml +++ b/charts/kubezero-kiam/values.yaml @@ -26,6 +26,8 @@ kiam: prometheus: servicemonitor: enabled: false + labels: + release: metrics log: level: warn @@ -51,6 +53,8 @@ kiam: prometheus: servicemonitor: enabled: false + labels: + release: metrics log: level: warn # extraEnv: -- 2.40.1 From 15605d0cefa332e5e5bc05fbf8ee7d3cda251f71 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 14 Aug 2020 22:31:34 +0100 Subject: [PATCH 63/96] Adjust scrape internal for kiam to match others, and servicemonitor for agents --- charts/kubezero-kiam/README.md.gotmpl | 2 +- charts/kubezero-kiam/values.yaml | 2 ++ deploy/templates/values.yaml | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-kiam/README.md.gotmpl b/charts/kubezero-kiam/README.md.gotmpl index 6382e4e..175c341 100644 --- a/charts/kubezero-kiam/README.md.gotmpl +++ b/charts/kubezero-kiam/README.md.gotmpl @@ -35,5 +35,5 @@ By default all access to the meta-data service is blocked, expect for: ## Resources - https://github.com/uswitch/kiam - https://www.bluematador.com/blog/iam-access-in-kubernetes-kube2iam-vs-kiam - +- [Grafana Dashboard](https://raw.githubusercontent.com/uswitch/kiam/master/docs/dashboard-prom.json) ![Kiam overview](./kiam_architecure.png) diff --git a/charts/kubezero-kiam/values.yaml b/charts/kubezero-kiam/values.yaml index d8d4ac6..ad05e57 100644 --- a/charts/kubezero-kiam/values.yaml +++ b/charts/kubezero-kiam/values.yaml @@ -26,6 +26,7 @@ kiam: prometheus: servicemonitor: enabled: false + interval: 30s labels: release: metrics log: @@ -53,6 +54,7 @@ kiam: prometheus: servicemonitor: enabled: false + interval: 30s labels: release: metrics log: diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 86b4763..8fa5a96 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -59,6 +59,10 @@ kubezero: prometheus: servicemonitor: enabled: {{ .Values.metrics.enabled }} + agent: + prometheus: + servicemonitor: + enabled: {{ .Values.metrics.enabled }} {{- end }} {{- if not .Values.kiam.not_ready }} -- 2.40.1 From 0e2e8502edef94f1f1644fd52451a2c206aa110b Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 14 Aug 2020 23:02:30 +0100 Subject: [PATCH 64/96] More logging fixes --- charts/kubezero-logging/templates/elasticsearch.yaml | 7 +++---- charts/kubezero-logging/values.yaml | 6 +----- deploy/templates/values.yaml | 3 +++ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/charts/kubezero-logging/templates/elasticsearch.yaml b/charts/kubezero-logging/templates/elasticsearch.yaml index f99ba51..5798c5e 100644 --- a/charts/kubezero-logging/templates/elasticsearch.yaml +++ b/charts/kubezero-logging/templates/elasticsearch.yaml @@ -56,10 +56,9 @@ spec: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - elasticsearch.k8s.elastic.co/cluster-name: {{ template "kubezero-lib.fullname" $ }} + - labelSelector: + matchLabels: + elasticsearch.k8s.elastic.co/cluster-name: {{ template "kubezero-lib.fullname" $ }} topologyKey: kubernetes.io/hostname {{- if .zone }} nodeAffinity: diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index b4d00bb..6d6ae9a 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -1,8 +1,4 @@ -# Default values for zdt-logging. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -# This is for backwards compatibility with older zdt-logging setup +# use this for backwards compatability # fullnameOverride: "" # Version for ElasticSearch and Kibana have to match so we define it at top-level diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 8fa5a96..600df34 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -129,6 +129,9 @@ kubezero: logging: enabled: {{ .Values.logging.enabled }} values: + {{- with .Values.logging }} + {{- toYaml . | nindent 6 }} + {{- end }} argo-cd: {{- with index .Values "argo-cd" "server" }} -- 2.40.1 From 5595fff159f5ed358f76685374b35df8e00ad6e6 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sat, 15 Aug 2020 13:24:14 +0100 Subject: [PATCH 65/96] Fix optional prometheus support --- charts/kubezero-logging/delete_zdt-logging.sh | 5 +++++ charts/kubezero-logging/templates/elasticsearch.yaml | 2 ++ 2 files changed, 7 insertions(+) create mode 100755 charts/kubezero-logging/delete_zdt-logging.sh diff --git a/charts/kubezero-logging/delete_zdt-logging.sh b/charts/kubezero-logging/delete_zdt-logging.sh new file mode 100755 index 0000000..0931e5d --- /dev/null +++ b/charts/kubezero-logging/delete_zdt-logging.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# We only need to delete the service monitor and virtual service, others will be taken over by the new chart and we dont loose data +kubectl delete -n logging VirtualService kibana-logging +kubectl delete -n logging ServiceMonitor es-logging diff --git a/charts/kubezero-logging/templates/elasticsearch.yaml b/charts/kubezero-logging/templates/elasticsearch.yaml index 5798c5e..a9dfd27 100644 --- a/charts/kubezero-logging/templates/elasticsearch.yaml +++ b/charts/kubezero-logging/templates/elasticsearch.yaml @@ -16,7 +16,9 @@ spec: node.data: true node.ingest: true node.ml: false + {{- if $.Values.es.prometheus }} prometheus.indices: false + {{- end }} {{- if .zone }} node.attr.zone: {{ .zone }} cluster.routing.allocation.awareness.attributes: zone -- 2.40.1 From 1ae1aac29422dd56f9ad05ebe31a20749df910ec Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sat, 15 Aug 2020 14:25:07 +0100 Subject: [PATCH 66/96] More logging fixes --- .../kubezero-logging/templates/secrets.yaml | 2 +- charts/kubezero-logging/values-test.yaml | 3 ++- charts/kubezero-logging/values.yaml | 3 ++- deploy/templates/values.yaml | 24 +++++++++++++++++-- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/charts/kubezero-logging/templates/secrets.yaml b/charts/kubezero-logging/templates/secrets.yaml index 7dd7920..7ba0fbb 100644 --- a/charts/kubezero-logging/templates/secrets.yaml +++ b/charts/kubezero-logging/templates/secrets.yaml @@ -11,4 +11,4 @@ metadata: {{ include "kubezero-lib.labels" . | indent 4 }} data: username: {{ "elastic" | b64enc | quote }} - elastic: {{ .Values.es.elastic_password | b64enc | quote }} + elastic: {{ .Values.elastic_password | b64enc | quote }} diff --git a/charts/kubezero-logging/values-test.yaml b/charts/kubezero-logging/values-test.yaml index 077be87..f4327e0 100644 --- a/charts/kubezero-logging/values-test.yaml +++ b/charts/kubezero-logging/values-test.yaml @@ -8,6 +8,8 @@ fullnameOverride: logging # Version for ElasticSearch and Kibana have to match so we define it at top-level version: 7.6.0 +elastic_password: "dsfsfs" # super_secret_elastic_password + es: nodeSets: - name: default-zone-0 @@ -19,7 +21,6 @@ es: s3Snapshot: enabled: true iamrole: "dfsf" # INSERT_CLOUDFORMATION_OUTPUT_ElasticSearchSnapshots - elastic_password: "dsfsfs" # super_secret_elastic_password prometheus: true diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 6d6ae9a..b0f1645 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -4,6 +4,8 @@ # Version for ElasticSearch and Kibana have to match so we define it at top-level version: 7.8.1 +elastic_password: "" # super_secret_elastic_password + es: nodeSets: [] #- count: 2 @@ -14,7 +16,6 @@ es: s3Snapshot: enabled: false iamrole: "" # INSERT_CLOUDFORMATION_OUTPUT_ElasticSearchSnapshots - elastic_password: "" # super_secret_elastic_password prometheus: false diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 600df34..fdc4d0b 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -129,8 +129,28 @@ kubezero: logging: enabled: {{ .Values.logging.enabled }} values: - {{- with .Values.logging }} - {{- toYaml . | nindent 6 }} + elastic_password: {{ .Values.logging.elastic_password }} + + {{- if .Values.logging.version }} + version: {{ .Values.logging.version }} + + {{- if .Values.logging.fullnameOverride }} + {{- end }} + fullnameOverride: {{ .Values.logging.fullnameOverride }} + {{- end }} + es: + {{- if .Values.logging.es.nodeSets }} + nodeSets: + {{- with .Values.logging.es.nodeSets }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + prometheus: {{ .Values.metrics.enabled }} + {{- if .Values.logging.kibana }} + kibana: + {{- with .Values.logging.kibana }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} argo-cd: -- 2.40.1 From a6eab7d24b6bca8571e55fdbc60741f7c76a3c79 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sat, 15 Aug 2020 14:33:41 +0100 Subject: [PATCH 67/96] Add label for servicemonitor --- charts/kubezero-logging/templates/servicemonitor.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/kubezero-logging/templates/servicemonitor.yaml b/charts/kubezero-logging/templates/servicemonitor.yaml index 208b31c..007acc8 100644 --- a/charts/kubezero-logging/templates/servicemonitor.yaml +++ b/charts/kubezero-logging/templates/servicemonitor.yaml @@ -6,6 +6,7 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{ include "kubezero-lib.labels" . | indent 4 }} + release: metrics spec: endpoints: - basicAuth: -- 2.40.1 From aac2e235f8ae4a8e78518a20f02870b6a05410f6 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sat, 15 Aug 2020 14:45:43 +0100 Subject: [PATCH 68/96] Exlude stateuful service objects to prevent double scrapes --- charts/kubezero-logging/templates/servicemonitor.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/kubezero-logging/templates/servicemonitor.yaml b/charts/kubezero-logging/templates/servicemonitor.yaml index 007acc8..0ca5595 100644 --- a/charts/kubezero-logging/templates/servicemonitor.yaml +++ b/charts/kubezero-logging/templates/servicemonitor.yaml @@ -19,6 +19,9 @@ spec: port: http path: /_prometheus/metrics selector: + matchExpressions: + key: elasticsearch.k8s.elastic.co/statefulset-name + operator: DoesNotExist matchLabels: common.k8s.elastic.co/type: elasticsearch elasticsearch.k8s.elastic.co/cluster-name: {{ template "kubezero-lib.fullname" $ }} -- 2.40.1 From 03bab16aa878bc0ff6a43527a1304a8a06f73493 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sat, 15 Aug 2020 14:49:30 +0100 Subject: [PATCH 69/96] Exlude stateuful service objects to prevent double scrapes --- charts/kubezero-logging/templates/servicemonitor.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-logging/templates/servicemonitor.yaml b/charts/kubezero-logging/templates/servicemonitor.yaml index 0ca5595..8c809f5 100644 --- a/charts/kubezero-logging/templates/servicemonitor.yaml +++ b/charts/kubezero-logging/templates/servicemonitor.yaml @@ -20,7 +20,7 @@ spec: path: /_prometheus/metrics selector: matchExpressions: - key: elasticsearch.k8s.elastic.co/statefulset-name + - key: elasticsearch.k8s.elastic.co/statefulset-name operator: DoesNotExist matchLabels: common.k8s.elastic.co/type: elasticsearch -- 2.40.1 From 4b734dc1bc38c4e9bd364bfd9c6e7c2cffe9630f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sat, 15 Aug 2020 15:59:57 +0100 Subject: [PATCH 70/96] Add cert-manager state handling for argo --- charts/kubezero-argo-cd/Chart.yaml | 2 +- charts/kubezero-argo-cd/values.yaml | 25 +++++++++++++++++++++++++ charts/kubezero-logging/Chart.yaml | 2 +- charts/kubezero/templates/logging.yaml | 16 ++++++++++++++++ deploy/templates/values.yaml | 11 ++++++++++- 5 files changed, 53 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-argo-cd/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml index fe1bc6d..324f89d 100644 --- a/charts/kubezero-argo-cd/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application name: kubezero-argo-cd -version: 0.3.10 +version: 0.3.11 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-argo-cd/values.yaml b/charts/kubezero-argo-cd/values.yaml index c02fed2..5adc1a1 100644 --- a/charts/kubezero-argo-cd/values.yaml +++ b/charts/kubezero-argo-cd/values.yaml @@ -44,6 +44,31 @@ argo-cd: # argo-cd.server.config.url -- ArgoCD hostname to be exposed via Istio url: argocd.example.com + resource.customizations: | + cert-manager.io/Certificate: + # Lua script for customizing the health status assessment + health.lua: | + hs = {} + if obj.status ~= nil then + if obj.status.conditions ~= nil then + for i, condition in ipairs(obj.status.conditions) do + if condition.type == "Ready" and condition.status == "False" then + hs.status = "Degraded" + hs.message = condition.message + return hs + end + if condition.type == "Ready" and condition.status == "True" then + hs.status = "Healthy" + hs.message = condition.message + return hs + end + end + end + end + hs.status = "Progressing" + hs.message = "Waiting for certificate" + return hs + # Rename former https port to grpc, works with istio + insecure service: servicePortHttpsName: grpc diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index 87fc2f0..099b623 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.0.2 +version: 0.1.0 appVersion: 1.2.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero/templates/logging.yaml b/charts/kubezero/templates/logging.yaml index d696c42..90cf116 100644 --- a/charts/kubezero/templates/logging.yaml +++ b/charts/kubezero/templates/logging.yaml @@ -3,6 +3,22 @@ syncPolicy: automated: prune: true + + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + jsonPointers: + - /webhooks/0/clientConfig/caBundle + - /webhooks/1/clientConfig/caBundle + - /webhooks/2/clientConfig/caBundle + - /webhooks/3/clientConfig/caBundle + - /webhooks/4/clientConfig/caBundle + - /webhooks/5/clientConfig/caBundle + - /webhooks/6/clientConfig/caBundle + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + jsonPointers: + - /status --- apiVersion: v1 kind: Namespace diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index fdc4d0b..9bb2f9f 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -133,11 +133,12 @@ kubezero: {{- if .Values.logging.version }} version: {{ .Values.logging.version }} + {{- end }} {{- if .Values.logging.fullnameOverride }} - {{- end }} fullnameOverride: {{ .Values.logging.fullnameOverride }} {{- end }} + es: {{- if .Values.logging.es.nodeSets }} nodeSets: @@ -146,6 +147,14 @@ kubezero: {{- end }} {{- end }} prometheus: {{ .Values.metrics.enabled }} + + {{- if .Values.logging.es.s3Snapshot }} + s3Snapshot: + {{- with .Values.logging.es.s3Snapshot }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.logging.kibana }} kibana: {{- with .Values.logging.kibana }} -- 2.40.1 From e8afc6ddbb25e85ef07b9669a1205afe02f8e8e8 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sat, 15 Aug 2020 23:37:45 +0100 Subject: [PATCH 71/96] EBS-CSI version bump, reduce ArgoCD concurrency to reduce load spikes, sync from 180s to 300s --- charts/kubezero-argo-cd/Chart.yaml | 2 +- charts/kubezero-argo-cd/README.md | 9 +++- charts/kubezero-argo-cd/README.md.gotmpl | 13 +++++ charts/kubezero-argo-cd/values.yaml | 49 ++++++++++--------- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 2 +- charts/kubezero-aws-ebs-csi-driver/README.md | 2 +- .../charts/aws-ebs-csi-driver/Chart.yaml | 22 ++++----- .../charts/aws-ebs-csi-driver/values.yaml | 2 +- charts/kubezero-kiam/README.md | 4 +- charts/kubezero-logging/README.md | 4 +- 10 files changed, 68 insertions(+), 41 deletions(-) create mode 100644 charts/kubezero-argo-cd/README.md.gotmpl diff --git a/charts/kubezero-argo-cd/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml index 324f89d..820dd55 100644 --- a/charts/kubezero-argo-cd/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application name: kubezero-argo-cd -version: 0.3.11 +version: 0.3.13 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-argo-cd/README.md b/charts/kubezero-argo-cd/README.md index 199bde5..b15d5b0 100644 --- a/charts/kubezero-argo-cd/README.md +++ b/charts/kubezero-argo-cd/README.md @@ -2,7 +2,7 @@ kubezero-argo-cd ================ KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application -Current chart version is `0.3.9` +Current chart version is `0.3.13` Source code can be found [here](https://kubezero.com) @@ -17,6 +17,9 @@ Source code can be found [here](https://kubezero.com) | Key | Type | Default | Description | |-----|------|---------|-------------| +| argo-cd.controller.args.appResyncPeriod | string | `"300"` | | +| argo-cd.controller.args.operationProcessors | string | `"1"` | | +| argo-cd.controller.args.statusProcessors | string | `"2"` | | | argo-cd.controller.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | argo-cd.controller.tolerations[0].effect | string | `"NoSchedule"` | | | argo-cd.controller.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | @@ -31,6 +34,7 @@ Source code can be found [here](https://kubezero.com) | argo-cd.repoServer.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | argo-cd.repoServer.tolerations[0].effect | string | `"NoSchedule"` | | | argo-cd.repoServer.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| argo-cd.server.config."resource.customizations" | string | `"cert-manager.io/Certificate:\n # Lua script for customizing the health status assessment\n health.lua: |\n hs = {}\n if obj.status ~= nil then\n if obj.status.conditions ~= nil then\n for i, condition in ipairs(obj.status.conditions) do\n if condition.type == \"Ready\" and condition.status == \"False\" then\n hs.status = \"Degraded\"\n hs.message = condition.message\n return hs\n end\n if condition.type == \"Ready\" and condition.status == \"True\" then\n hs.status = \"Healthy\"\n hs.message = condition.message\n return hs\n end\n end\n end\n end\n hs.status = \"Progressing\"\n hs.message = \"Waiting for certificate\"\n return hs\n"` | | | argo-cd.server.config.url | string | `"argocd.example.com"` | ArgoCD hostname to be exposed via Istio | | argo-cd.server.extraArgs[0] | string | `"--insecure"` | | | argo-cd.server.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | @@ -41,3 +45,6 @@ Source code can be found [here](https://kubezero.com) | kubezero.global.defaultSource.pathPrefix | string | `""` | optional path prefix within repoURL to support eg. remote subtrees | | kubezero.global.defaultSource.repoURL | string | `"https://github.com/zero-down-time/kubezero"` | default repository for argocd applications | | kubezero.global.defaultSource.targetRevision | string | `"HEAD"` | default tracking of repoURL | + +## Resources +- https://argoproj.github.io/argo-cd/operator-manual/metrics/ diff --git a/charts/kubezero-argo-cd/README.md.gotmpl b/charts/kubezero-argo-cd/README.md.gotmpl new file mode 100644 index 0000000..0f6b9f5 --- /dev/null +++ b/charts/kubezero-argo-cd/README.md.gotmpl @@ -0,0 +1,13 @@ +{{ template "chart.header" . }} +{{ template "chart.description" . }} + +{{ template "chart.versionLine" . }} + +{{ template "chart.sourceLinkLine" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} + +## Resources +- https://argoproj.github.io/argo-cd/operator-manual/metrics/ diff --git a/charts/kubezero-argo-cd/values.yaml b/charts/kubezero-argo-cd/values.yaml index 5adc1a1..e24d20b 100644 --- a/charts/kubezero-argo-cd/values.yaml +++ b/charts/kubezero-argo-cd/values.yaml @@ -26,6 +26,11 @@ argo-cd: # Run Argo on the controllers controller: + args: + statusProcessors: "2" + operationProcessors: "1" + appResyncPeriod: "300" + nodeSelector: node-role.kubernetes.io/master: "" tolerations: @@ -44,30 +49,30 @@ argo-cd: # argo-cd.server.config.url -- ArgoCD hostname to be exposed via Istio url: argocd.example.com - resource.customizations: | - cert-manager.io/Certificate: - # Lua script for customizing the health status assessment - health.lua: | - hs = {} - if obj.status ~= nil then - if obj.status.conditions ~= nil then - for i, condition in ipairs(obj.status.conditions) do - if condition.type == "Ready" and condition.status == "False" then - hs.status = "Degraded" - hs.message = condition.message - return hs - end - if condition.type == "Ready" and condition.status == "True" then - hs.status = "Healthy" - hs.message = condition.message - return hs + resource.customizations: | + cert-manager.io/Certificate: + # Lua script for customizing the health status assessment + health.lua: | + hs = {} + if obj.status ~= nil then + if obj.status.conditions ~= nil then + for i, condition in ipairs(obj.status.conditions) do + if condition.type == "Ready" and condition.status == "False" then + hs.status = "Degraded" + hs.message = condition.message + return hs + end + if condition.type == "Ready" and condition.status == "True" then + hs.status = "Healthy" + hs.message = condition.message + return hs + end + end end end - end - end - hs.status = "Progressing" - hs.message = "Waiting for certificate" - return hs + hs.status = "Progressing" + hs.message = "Waiting for certificate" + return hs # Rename former https port to grpc, works with istio + insecure service: diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 51053ae..9326a28 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.3.0 +version: 0.3.1 appVersion: 0.6.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-aws-ebs-csi-driver/README.md b/charts/kubezero-aws-ebs-csi-driver/README.md index ea60547..7dac424 100644 --- a/charts/kubezero-aws-ebs-csi-driver/README.md +++ b/charts/kubezero-aws-ebs-csi-driver/README.md @@ -2,7 +2,7 @@ kubezero-aws-ebs-csi-driver =========================== KubeZero Umbrella Chart for aws-ebs-csi-driver -Current chart version is `0.3.0` +Current chart version is `0.3.1` Source code can be found [here](https://kubezero.com) diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml index 336d34f..77c838e 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml @@ -1,16 +1,16 @@ apiVersion: v1 -appVersion: "0.6.0" -name: aws-ebs-csi-driver +appVersion: 0.6.0 description: A Helm chart for AWS EBS CSI Driver -version: 0.5.0 -kubeVersion: ">=1.13.0-0" home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver -sources: - - https://github.com/kubernetes-sigs/aws-ebs-csi-driver keywords: - - aws - - ebs - - csi +- aws +- ebs +- csi +kubeVersion: '>=1.13.0-0' maintainers: - - name: leakingtapan - email: chengpan@amazon.com +- email: chengpan@amazon.com + name: leakingtapan +name: aws-ebs-csi-driver +sources: +- https://github.com/kubernetes-sigs/aws-ebs-csi-driver +version: 0.5.0 diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml index 4528b83..fd74daf 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml @@ -6,7 +6,7 @@ replicaCount: 2 image: repository: amazon/aws-ebs-csi-driver - tag: "v0.5.0" + tag: "v0.6.0" pullPolicy: IfNotPresent sidecars: diff --git a/charts/kubezero-kiam/README.md b/charts/kubezero-kiam/README.md index 223cc87..d973e8b 100644 --- a/charts/kubezero-kiam/README.md +++ b/charts/kubezero-kiam/README.md @@ -41,6 +41,7 @@ By default all access to the meta-data service is blocked, expect for: | kiam.agent.image.tag | string | `"v3.6"` | | | kiam.agent.log.level | string | `"warn"` | | | kiam.agent.prometheus.servicemonitor.enabled | bool | `false` | | +| kiam.agent.prometheus.servicemonitor.interval | string | `"30s"` | | | kiam.agent.prometheus.servicemonitor.labels.release | string | `"metrics"` | | | kiam.agent.sslCertHostPath | string | `"/etc/ssl/certs"` | | | kiam.agent.tlsCerts.caFileName | string | `"ca.crt"` | | @@ -58,6 +59,7 @@ By default all access to the meta-data service is blocked, expect for: | kiam.server.log.level | string | `"warn"` | | | kiam.server.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | kiam.server.prometheus.servicemonitor.enabled | bool | `false` | | +| kiam.server.prometheus.servicemonitor.interval | string | `"30s"` | | | kiam.server.prometheus.servicemonitor.labels.release | string | `"metrics"` | | | kiam.server.service.port | int | `6444` | | | kiam.server.service.targetPort | int | `6444` | | @@ -79,5 +81,5 @@ By default all access to the meta-data service is blocked, expect for: ## Resources - https://github.com/uswitch/kiam - https://www.bluematador.com/blog/iam-access-in-kubernetes-kube2iam-vs-kiam - +- [Grafana Dashboard](https://raw.githubusercontent.com/uswitch/kiam/master/docs/dashboard-prom.json) ![Kiam overview](./kiam_architecure.png) diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md index 0635ffe..7e427ad 100644 --- a/charts/kubezero-logging/README.md +++ b/charts/kubezero-logging/README.md @@ -2,7 +2,7 @@ kubezero-logging ================ KubeZero Umbrella Chart for complete EFK stack -Current chart version is `0.0.2` +Current chart version is `0.1.0` Source code can be found [here](https://kubezero.com) @@ -42,7 +42,7 @@ Source code can be found [here](https://kubezero.com) | Key | Type | Default | Description | |-----|------|---------|-------------| -| es.elastic_password | string | `""` | | +| elastic_password | string | `""` | | | es.nodeSets | list | `[]` | | | es.prometheus | bool | `false` | | | es.s3Snapshot.enabled | bool | `false` | | -- 2.40.1 From 0db65bd060041247f1eea7cefee6569652dec308 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sun, 16 Aug 2020 15:49:57 +0100 Subject: [PATCH 72/96] Wire up prometheus metrics for argo-cd --- charts/kubezero-argo-cd/Chart.yaml | 2 +- charts/kubezero-argo-cd/README.md | 14 +++++++++++++- charts/kubezero-argo-cd/values.yaml | 26 +++++++++++++++++++++++++- deploy/templates/values.yaml | 9 +++++++-- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/charts/kubezero-argo-cd/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml index 820dd55..618d4b0 100644 --- a/charts/kubezero-argo-cd/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application name: kubezero-argo-cd -version: 0.3.13 +version: 0.4.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-argo-cd/README.md b/charts/kubezero-argo-cd/README.md index b15d5b0..0ba38ee 100644 --- a/charts/kubezero-argo-cd/README.md +++ b/charts/kubezero-argo-cd/README.md @@ -2,7 +2,7 @@ kubezero-argo-cd ================ KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application -Current chart version is `0.3.13` +Current chart version is `0.4.0` Source code can be found [here](https://kubezero.com) @@ -20,6 +20,10 @@ Source code can be found [here](https://kubezero.com) | argo-cd.controller.args.appResyncPeriod | string | `"300"` | | | argo-cd.controller.args.operationProcessors | string | `"1"` | | | argo-cd.controller.args.statusProcessors | string | `"2"` | | +| argo-cd.controller.metrics.enabled | bool | `false` | | +| argo-cd.controller.metrics.serviceMonitor.additionalLabels.release | string | `"metrics"` | | +| argo-cd.controller.metrics.serviceMonitor.enabled | bool | `true` | | +| argo-cd.controller.metrics.serviceMonitor.namespace | string | `"monitoring"` | | | argo-cd.controller.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | argo-cd.controller.tolerations[0].effect | string | `"NoSchedule"` | | | argo-cd.controller.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | @@ -31,12 +35,20 @@ Source code can be found [here](https://kubezero.com) | argo-cd.redis.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | argo-cd.redis.tolerations[0].effect | string | `"NoSchedule"` | | | argo-cd.redis.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| argo-cd.repoServer.metrics.enabled | bool | `false` | | +| argo-cd.repoServer.metrics.serviceMonitor.additionalLabels.release | string | `"metrics"` | | +| argo-cd.repoServer.metrics.serviceMonitor.enabled | bool | `true` | | +| argo-cd.repoServer.metrics.serviceMonitor.namespace | string | `"monitoring"` | | | argo-cd.repoServer.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | argo-cd.repoServer.tolerations[0].effect | string | `"NoSchedule"` | | | argo-cd.repoServer.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | argo-cd.server.config."resource.customizations" | string | `"cert-manager.io/Certificate:\n # Lua script for customizing the health status assessment\n health.lua: |\n hs = {}\n if obj.status ~= nil then\n if obj.status.conditions ~= nil then\n for i, condition in ipairs(obj.status.conditions) do\n if condition.type == \"Ready\" and condition.status == \"False\" then\n hs.status = \"Degraded\"\n hs.message = condition.message\n return hs\n end\n if condition.type == \"Ready\" and condition.status == \"True\" then\n hs.status = \"Healthy\"\n hs.message = condition.message\n return hs\n end\n end\n end\n end\n hs.status = \"Progressing\"\n hs.message = \"Waiting for certificate\"\n return hs\n"` | | | argo-cd.server.config.url | string | `"argocd.example.com"` | ArgoCD hostname to be exposed via Istio | | argo-cd.server.extraArgs[0] | string | `"--insecure"` | | +| argo-cd.server.metrics.enabled | bool | `false` | | +| argo-cd.server.metrics.serviceMonitor.additionalLabels.release | string | `"metrics"` | | +| argo-cd.server.metrics.serviceMonitor.enabled | bool | `true` | | +| argo-cd.server.metrics.serviceMonitor.namespace | string | `"monitoring"` | | | argo-cd.server.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | argo-cd.server.service.servicePortHttpsName | string | `"grpc"` | | | argo-cd.server.tolerations[0].effect | string | `"NoSchedule"` | | diff --git a/charts/kubezero-argo-cd/values.yaml b/charts/kubezero-argo-cd/values.yaml index e24d20b..c36fc9d 100644 --- a/charts/kubezero-argo-cd/values.yaml +++ b/charts/kubezero-argo-cd/values.yaml @@ -24,13 +24,21 @@ argo-cd: # argocdServerAdminPassword: "$2a$10$ivKzaXVxMqdeDSfS3nqi1Od3iDbnL7oXrixzDfZFRHlXHnAG6LydG" # argocdServerAdminPasswordMtime: "2020-04-24T15:33:09BST" - # Run Argo on the controllers controller: args: statusProcessors: "2" operationProcessors: "1" appResyncPeriod: "300" + metrics: + enabled: false + serviceMonitor: + enabled: true + namespace: monitoring + additionalLabels: + release: metrics + + # controller to masters nodeSelector: node-role.kubernetes.io/master: "" tolerations: @@ -38,6 +46,14 @@ argo-cd: effect: NoSchedule repoServer: + metrics: + enabled: false + serviceMonitor: + enabled: true + namespace: monitoring + additionalLabels: + release: metrics + nodeSelector: node-role.kubernetes.io/master: "" tolerations: @@ -78,6 +94,14 @@ argo-cd: service: servicePortHttpsName: grpc + metrics: + enabled: false + serviceMonitor: + enabled: true + namespace: monitoring + additionalLabels: + release: metrics + extraArgs: - --insecure nodeSelector: diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 9bb2f9f..15b9dac 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -163,10 +163,15 @@ kubezero: {{- end }} argo-cd: - {{- with index .Values "argo-cd" "server" }} + controller: + metrics: {{ .Values.metrics.enabled }} + repoServer: + metrics: {{ .Values.metrics.enabled }} server: + metrics: {{ .Values.metrics.enabled }} + {{- with index .Values "argo-cd" "server" }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- with index .Values "argo-cd" "configs" }} configs: {{- toYaml . | nindent 4 }} -- 2.40.1 From 1398484af85298ffa8789d042ca6f71b7d80b15b Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sun, 16 Aug 2020 19:25:07 +0100 Subject: [PATCH 73/96] Make argocd metrics work --- charts/kubezero-argo-cd/README.md.gotmpl | 1 + deploy/templates/values.yaml | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-argo-cd/README.md.gotmpl b/charts/kubezero-argo-cd/README.md.gotmpl index 0f6b9f5..d0cf107 100644 --- a/charts/kubezero-argo-cd/README.md.gotmpl +++ b/charts/kubezero-argo-cd/README.md.gotmpl @@ -11,3 +11,4 @@ ## Resources - https://argoproj.github.io/argo-cd/operator-manual/metrics/ +- https://raw.githubusercontent.com/argoproj/argo-cd/master/examples/dashboard.json diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 15b9dac..f93b393 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -164,11 +164,14 @@ kubezero: argo-cd: controller: - metrics: {{ .Values.metrics.enabled }} + metrics: + enabled: {{ .Values.metrics.enabled }} repoServer: - metrics: {{ .Values.metrics.enabled }} + metrics: + enabled: {{ .Values.metrics.enabled }} server: - metrics: {{ .Values.metrics.enabled }} + metrics: + enabled: {{ .Values.metrics.enabled }} {{- with index .Values "argo-cd" "server" }} {{- toYaml . | nindent 4 }} {{- end }} -- 2.40.1 From fba3e8bfc4237650743580d7f89495337c64b94d Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 17 Aug 2020 13:12:07 +0100 Subject: [PATCH 74/96] Make old ECK resources optional --- charts/kubezero-logging/templates/eck-operator.yaml | 2 ++ charts/kubezero-logging/templates/kibana.yaml | 3 ++- charts/kubezero-logging/templates/secrets.yaml | 2 ++ charts/kubezero-logging/values-no-eck.yaml | 6 ++++++ 4 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 charts/kubezero-logging/values-no-eck.yaml diff --git a/charts/kubezero-logging/templates/eck-operator.yaml b/charts/kubezero-logging/templates/eck-operator.yaml index 54bb221..055d1c3 100644 --- a/charts/kubezero-logging/templates/eck-operator.yaml +++ b/charts/kubezero-logging/templates/eck-operator.yaml @@ -1,3 +1,4 @@ +{{- if .Values.es.nodeSets }} apiVersion: v1 kind: Namespace metadata: @@ -3056,3 +3057,4 @@ spec: secret: defaultMode: 420 secretName: elastic-webhook-server-cert +{{- end }} diff --git a/charts/kubezero-logging/templates/kibana.yaml b/charts/kubezero-logging/templates/kibana.yaml index f80539a..1a24230 100644 --- a/charts/kubezero-logging/templates/kibana.yaml +++ b/charts/kubezero-logging/templates/kibana.yaml @@ -1,4 +1,5 @@ -{{- if .Values.kibana }} +{{- if .Values.es.nodeSets }} +# Only deploy Kibana if we have local ES cluster apiVersion: kibana.k8s.elastic.co/v1 kind: Kibana metadata: diff --git a/charts/kubezero-logging/templates/secrets.yaml b/charts/kubezero-logging/templates/secrets.yaml index 7ba0fbb..6fe9151 100644 --- a/charts/kubezero-logging/templates/secrets.yaml +++ b/charts/kubezero-logging/templates/secrets.yaml @@ -1,3 +1,4 @@ +{{- if .Values.es.nodeSets }} apiVersion: v1 kind: Secret type: Opaque @@ -12,3 +13,4 @@ metadata: data: username: {{ "elastic" | b64enc | quote }} elastic: {{ .Values.elastic_password | b64enc | quote }} +{{- end }} diff --git a/charts/kubezero-logging/values-no-eck.yaml b/charts/kubezero-logging/values-no-eck.yaml new file mode 100644 index 0000000..ef87eed --- /dev/null +++ b/charts/kubezero-logging/values-no-eck.yaml @@ -0,0 +1,6 @@ +# Default values for zdt-logging. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +kibana: + count: 0 -- 2.40.1 From 12abcacdd927626cfab7b58150cd016d4bb33b70 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 18 Aug 2020 11:34:34 +0100 Subject: [PATCH 75/96] Add fluentd to logging --- charts/kubezero-logging/Chart.yaml | 6 +- .../templates/{ => eck}/eck-operator.yaml | 0 .../templates/{ => eck}/elasticsearch.yaml | 0 .../{ => eck}/istio-virtualservice.yaml | 0 .../templates/{ => eck}/kibana.yaml | 0 .../templates/{ => eck}/secrets.yaml | 0 .../templates/{ => eck}/servicemonitor.yaml | 0 .../fluentd/fluentd-certificate.yaml | 16 ++ .../templates/fluentd/istio-service.yaml | 28 +++ .../{values-test.yaml => values-all.yaml} | 3 + charts/kubezero-logging/values-fluentd.yaml | 11 ++ charts/kubezero-logging/values-no-eck.yaml | 6 - charts/kubezero-logging/values-remote-es.yaml | 0 charts/kubezero-logging/values.yaml | 165 ++++++++++++++++++ deploy/templates/values.yaml | 15 ++ deploy/values.yaml | 2 + 16 files changed, 245 insertions(+), 7 deletions(-) rename charts/kubezero-logging/templates/{ => eck}/eck-operator.yaml (100%) rename charts/kubezero-logging/templates/{ => eck}/elasticsearch.yaml (100%) rename charts/kubezero-logging/templates/{ => eck}/istio-virtualservice.yaml (100%) rename charts/kubezero-logging/templates/{ => eck}/kibana.yaml (100%) rename charts/kubezero-logging/templates/{ => eck}/secrets.yaml (100%) rename charts/kubezero-logging/templates/{ => eck}/servicemonitor.yaml (100%) create mode 100644 charts/kubezero-logging/templates/fluentd/fluentd-certificate.yaml create mode 100644 charts/kubezero-logging/templates/fluentd/istio-service.yaml rename charts/kubezero-logging/{values-test.yaml => values-all.yaml} (96%) create mode 100644 charts/kubezero-logging/values-fluentd.yaml delete mode 100644 charts/kubezero-logging/values-no-eck.yaml create mode 100644 charts/kubezero-logging/values-remote-es.yaml diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index 099b623..0750821 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.1.0 +version: 0.2.0 appVersion: 1.2.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -18,4 +18,8 @@ dependencies: - name: kubezero-lib version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ + - name: fluentd + version: 2.5.1 + repository: https://kubernetes-charts.storage.googleapis.com/ + condition: fluentd.enabled kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-logging/templates/eck-operator.yaml b/charts/kubezero-logging/templates/eck/eck-operator.yaml similarity index 100% rename from charts/kubezero-logging/templates/eck-operator.yaml rename to charts/kubezero-logging/templates/eck/eck-operator.yaml diff --git a/charts/kubezero-logging/templates/elasticsearch.yaml b/charts/kubezero-logging/templates/eck/elasticsearch.yaml similarity index 100% rename from charts/kubezero-logging/templates/elasticsearch.yaml rename to charts/kubezero-logging/templates/eck/elasticsearch.yaml diff --git a/charts/kubezero-logging/templates/istio-virtualservice.yaml b/charts/kubezero-logging/templates/eck/istio-virtualservice.yaml similarity index 100% rename from charts/kubezero-logging/templates/istio-virtualservice.yaml rename to charts/kubezero-logging/templates/eck/istio-virtualservice.yaml diff --git a/charts/kubezero-logging/templates/kibana.yaml b/charts/kubezero-logging/templates/eck/kibana.yaml similarity index 100% rename from charts/kubezero-logging/templates/kibana.yaml rename to charts/kubezero-logging/templates/eck/kibana.yaml diff --git a/charts/kubezero-logging/templates/secrets.yaml b/charts/kubezero-logging/templates/eck/secrets.yaml similarity index 100% rename from charts/kubezero-logging/templates/secrets.yaml rename to charts/kubezero-logging/templates/eck/secrets.yaml diff --git a/charts/kubezero-logging/templates/servicemonitor.yaml b/charts/kubezero-logging/templates/eck/servicemonitor.yaml similarity index 100% rename from charts/kubezero-logging/templates/servicemonitor.yaml rename to charts/kubezero-logging/templates/eck/servicemonitor.yaml diff --git a/charts/kubezero-logging/templates/fluentd/fluentd-certificate.yaml b/charts/kubezero-logging/templates/fluentd/fluentd-certificate.yaml new file mode 100644 index 0000000..15d2517 --- /dev/null +++ b/charts/kubezero-logging/templates/fluentd/fluentd-certificate.yaml @@ -0,0 +1,16 @@ +{{- if .Values.fluentd.enabled }} +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: fluentd-ingress-cert + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + secretName: fluentd-certificate + issuerRef: + name: letsencrypt-dns-prod + kind: ClusterIssuer + dnsNames: + - "{{ .Values.fluentd.url }}" +{{- end }} diff --git a/charts/kubezero-logging/templates/fluentd/istio-service.yaml b/charts/kubezero-logging/templates/fluentd/istio-service.yaml new file mode 100644 index 0000000..048b8e9 --- /dev/null +++ b/charts/kubezero-logging/templates/fluentd/istio-service.yaml @@ -0,0 +1,28 @@ +{{- if .Values.fluentd.istio.enabled }} +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: fluentd + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + gateways: + - {{ .Values.fluentd.istio.gateway }} + hosts: + - {{ .Values.fluentd.url }} + tcp: + - match: + - port: 24224 + route: + - destination: + host: fluentd + port: + number: 24224 + http: + - route: + - destination: + host: fluentd + port: + number: 9880 +{{- end }} diff --git a/charts/kubezero-logging/values-test.yaml b/charts/kubezero-logging/values-all.yaml similarity index 96% rename from charts/kubezero-logging/values-test.yaml rename to charts/kubezero-logging/values-all.yaml index f4327e0..c87d0ad 100644 --- a/charts/kubezero-logging/values-test.yaml +++ b/charts/kubezero-logging/values-all.yaml @@ -29,3 +29,6 @@ kibana: enabled: true url: kibana.example.com gateway: istio-system/private-ingressgateway + +fluentd: + enabled: true diff --git a/charts/kubezero-logging/values-fluentd.yaml b/charts/kubezero-logging/values-fluentd.yaml new file mode 100644 index 0000000..07535e7 --- /dev/null +++ b/charts/kubezero-logging/values-fluentd.yaml @@ -0,0 +1,11 @@ +fluentd: + enabled: true + + metrics: + enabled: true + + url: fluentd.example.com + istio: + enabled: true + gateway: istio-system/private-ingressgateway + diff --git a/charts/kubezero-logging/values-no-eck.yaml b/charts/kubezero-logging/values-no-eck.yaml deleted file mode 100644 index ef87eed..0000000 --- a/charts/kubezero-logging/values-no-eck.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Default values for zdt-logging. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -kibana: - count: 0 diff --git a/charts/kubezero-logging/values-remote-es.yaml b/charts/kubezero-logging/values-remote-es.yaml new file mode 100644 index 0000000..e69de29 diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index b0f1645..b445b96 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -26,3 +26,168 @@ kibana: enabled: false gateway: "istio-system/ingressgateway" url: "" # kibana.example.com + +fluentd: + enabled: false + #image: + #repository: quay.io/fluentd_elasticsearch/fluentd + #tag: v2.9.0 + istio: + enabled: false + + # we wont persistent buffering + useStatefulSet: true + replicaCount: 2 + + plugins: + enabled: true + pluginsList: + - fluent-plugin-detect-exceptions + # - fluent-plugin-s3 + # - fluent-plugin-grok-parser + + persistence: + enabled: true + storageClass: "ebs-sc-gp2-xfs" + accessMode: ReadWriteOnce + size: 4Gi + + service: + ports: + - name: tcp-forward + protocol: TCP + containerPort: 24224 + - name: http-fluentd + protocol: TCP + containerPort: 9880 + + metrics: + enabled: false + serviceMonitor: + enabled: true + additionalLabels: + release: metrics + namespace: monitoring + + output: + host: logging-es-http + + env: + OUTPUT_USER: elastic + OUTPUT_SSL_VERIFY: "false" + + extraEnvVars: + - name: OUTPUT_PASSWORD + valueFrom: + secretKeyRef: + name: logging-es-elastic-user + key: elastic + - name: FLUENTD_SHARED_KEY + valueFrom: + secretKeyRef: + name: fluentd-config + key: shared_key + + extraVolumes: + - name: fluentd-certs + secret: + secretName: fluentd-certificate + extraVolumeMounts: + - name: fluentd-certs + mountPath: /mnt/fluentd-certs + readOnly: true + + configMaps: + forward-input.conf: | + + @type forward + port 24224 + bind 0.0.0.0 + skip_invalid_event true + + cert_path /mnt/fluentd-certs/tls.crt + private_key_path /mnt/fluentd-certs/tls.key + + + self_hostname "#{ENV['HOSTNAME']}" + shared_key "#{ENV['FLUENTD_SHARED_KEY']}" + + + + output.conf: | + + @id elasticsearch + @type elasticsearch + @log_level info + include_tag_key true + id_key id + remove_keys id + + # This pipeline incl. eg. GeoIP + pipeline fluentd + + host "#{ENV['OUTPUT_HOST']}" + port "#{ENV['OUTPUT_PORT']}" + scheme "#{ENV['OUTPUT_SCHEME']}" + ssl_version "#{ENV['OUTPUT_SSL_VERSION']}" + ssl_verify "#{ENV['OUTPUT_SSL_VERIFY']}" + user "#{ENV['OUTPUT_USER']}" + password "#{ENV['OUTPUT_PASSWORD']}" + + logstash_format true + reload_connections false + reconnect_on_error true + reload_on_failure true + request_timeout 15s + + + @type file + path /var/log/fluentd-buffers/kubernetes.system.buffer + flush_mode interval + flush_thread_count 2 + flush_interval 5s + flush_at_shutdown true + retry_type exponential_backoff + retry_timeout 60m + retry_max_interval 30 + chunk_limit_size "#{ENV['OUTPUT_BUFFER_CHUNK_LIMIT']}" + queue_limit_length "#{ENV['OUTPUT_BUFFER_QUEUE_LIMIT']}" + overflow_action drop_oldest_chunk + + + +# filter.conf: | +# +# @type parser +# key_name message +# reserve_data true +# reserve_time true +# +# @type grok +# +# # SSH +# +# pattern %{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:system.auth.user} from %{IPORHOST:system.auth.ip} port %{NUMBER:system.auth.port} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})? +# +# +# pattern %{DATA:system.auth.ssh.event} user %{DATA:system.auth.user} from %{IPORHOST:system.auth.ip} +# +# +# # sudo +# +# pattern \s*%{DATA:system.auth.user} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command} +# +# +# # Users +# +# pattern new group: name=%{DATA:system.auth.groupadd.name}, GID=%{NUMBER:system.auth.groupadd.gid} +# +# +# pattern new user: name=%{DATA:system.auth.useradd.name}, UID=%{NUMBER:system.auth.useradd.uid}, GID=%{NUMBER:system.auth.useradd.gid}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$ +# +# +# +# pattern %{GREEDYDATA:message} +# +# +# diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index f93b393..8ec4919 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -139,6 +139,7 @@ kubezero: fullnameOverride: {{ .Values.logging.fullnameOverride }} {{- end }} + {{- if .Values.logging.es }} es: {{- if .Values.logging.es.nodeSets }} nodeSets: @@ -154,6 +155,7 @@ kubezero: {{- toYaml . | nindent 10 }} {{- end }} {{- end }} + {{- end }} {{- if .Values.logging.kibana }} kibana: @@ -162,6 +164,19 @@ kubezero: {{- end }} {{- end }} + fluentd: + enabled: {{ .Values.logging.fluentd.enabled }} + metrics: + enabled: {{ .Values.metrics.enabled }} + url: {{ .Values.logging.fluentd.url }} + {{- if and .Values.logging.fluentd.istio .Values.istio.enabled }} + istio: + {{- with .Values.logging.fluentd.istio }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + + argo-cd: controller: metrics: diff --git a/deploy/values.yaml b/deploy/values.yaml index 65cb531..2a291a8 100644 --- a/deploy/values.yaml +++ b/deploy/values.yaml @@ -35,6 +35,8 @@ metrics: logging: enabled: false + fluentd: + enabled: false argo-cd: server: {} -- 2.40.1 From 777a0d7f9441ea83e6ce52b0bae73c166601ef26 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 18 Aug 2020 11:41:09 +0100 Subject: [PATCH 76/96] Disable statefulset for fluentd being broken upstream --- charts/kubezero-logging/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index b445b96..9aa6ab1 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -35,8 +35,8 @@ fluentd: istio: enabled: false - # we wont persistent buffering - useStatefulSet: true + # broken as of 2.5.1 ;-( + # useStatefulSet: true replicaCount: 2 plugins: -- 2.40.1 From e6e0aa103b817e5243e25e9b12e386e0f48aaae1 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 18 Aug 2020 11:58:37 +0100 Subject: [PATCH 77/96] Add missing fluentd secrets --- .../kubezero-logging/templates/fluentd/secret.yaml | 12 ++++++++++++ charts/kubezero-logging/values.yaml | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 charts/kubezero-logging/templates/fluentd/secret.yaml diff --git a/charts/kubezero-logging/templates/fluentd/secret.yaml b/charts/kubezero-logging/templates/fluentd/secret.yaml new file mode 100644 index 0000000..761cc92 --- /dev/null +++ b/charts/kubezero-logging/templates/fluentd/secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.fluentd.enabled }} +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: {{ template "kubezero-lib.fullname" $ }}-fluentd-secret + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +data: + shared_key: {{ .Values.fluentd.shared_key | b64enc | quote }} +{{- end }} diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 9aa6ab1..8041db0 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -72,6 +72,8 @@ fluentd: output: host: logging-es-http + shared_key: "cloudbender" + env: OUTPUT_USER: elastic OUTPUT_SSL_VERIFY: "false" @@ -85,7 +87,7 @@ fluentd: - name: FLUENTD_SHARED_KEY valueFrom: secretKeyRef: - name: fluentd-config + name: logging-fluentd-secret key: shared_key extraVolumes: -- 2.40.1 From 4a6cbfbbcf75dbe7150b1cfe41312493a76db128 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 18 Aug 2020 12:08:49 +0100 Subject: [PATCH 78/96] Disable persistence by default --- charts/kubezero-logging/values.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 8041db0..bb13763 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -46,11 +46,11 @@ fluentd: # - fluent-plugin-s3 # - fluent-plugin-grok-parser - persistence: - enabled: true - storageClass: "ebs-sc-gp2-xfs" - accessMode: ReadWriteOnce - size: 4Gi + #persistence: + # enabled: true + # storageClass: "ebs-sc-gp2-xfs" + # accessMode: ReadWriteOnce + # size: 4Gi service: ports: -- 2.40.1 From 279dde5ee28feac6069e9a8f36e00b39c5ec0b22 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 18 Aug 2020 12:36:56 +0100 Subject: [PATCH 79/96] Revert ot quay image, disable plugins --- charts/kubezero-logging/values.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index bb13763..946c2ab 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -29,9 +29,9 @@ kibana: fluentd: enabled: false - #image: - #repository: quay.io/fluentd_elasticsearch/fluentd - #tag: v2.9.0 + image: + repository: quay.io/fluentd_elasticsearch/fluentd + tag: v2.9.0 istio: enabled: false @@ -40,11 +40,11 @@ fluentd: replicaCount: 2 plugins: - enabled: true + enabled: false pluginsList: - - fluent-plugin-detect-exceptions - # - fluent-plugin-s3 - # - fluent-plugin-grok-parser + #- fluent-plugin-detect-exceptions + #- fluent-plugin-s3 + #- fluent-plugin-grok-parser #persistence: # enabled: true -- 2.40.1 From fbc203a2c9d4592424efed7044aa9c78cb9b4a40 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 18 Aug 2020 12:45:15 +0100 Subject: [PATCH 80/96] Fix istio to service mapping --- .../kubezero-logging/templates/fluentd/istio-service.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-logging/templates/fluentd/istio-service.yaml b/charts/kubezero-logging/templates/fluentd/istio-service.yaml index 048b8e9..4dfe010 100644 --- a/charts/kubezero-logging/templates/fluentd/istio-service.yaml +++ b/charts/kubezero-logging/templates/fluentd/istio-service.yaml @@ -2,7 +2,7 @@ apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: - name: fluentd + name: {{ template "kubezero-lib.fullname" $ }}-fluentd namespace: {{ .Release.Namespace }} labels: {{ include "kubezero-lib.labels" . | indent 4 }} @@ -16,13 +16,13 @@ spec: - port: 24224 route: - destination: - host: fluentd + host: {{ template "kubezero-lib.fullname" $ }}-fluentd port: number: 24224 http: - route: - destination: - host: fluentd + host: {{ template "kubezero-lib.fullname" $ }}-fluentd port: number: 9880 {{- end }} -- 2.40.1 From e7823037035928844bd446cfaed58d226df9cf38 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 18 Aug 2020 13:13:30 +0100 Subject: [PATCH 81/96] Revert to default images --- charts/kubezero-logging/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 946c2ab..1a537d4 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -29,9 +29,9 @@ kibana: fluentd: enabled: false - image: - repository: quay.io/fluentd_elasticsearch/fluentd - tag: v2.9.0 + #image: + # repository: quay.io/fluentd_elasticsearch/fluentd + # tag: v2.9.0 istio: enabled: false -- 2.40.1 From d49ff51379a895515d70b5ecf4d1016120702768 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 20 Aug 2020 11:40:08 +0100 Subject: [PATCH 82/96] Disable default syncPolicy, use values instead --- charts/kubezero-argo-cd/Chart.yaml | 2 +- charts/kubezero-argo-cd/README.md | 3 +- charts/kubezero-argo-cd/templates/app.yaml | 9 ++--- charts/kubezero-argo-cd/values.yaml | 5 +++ charts/kubezero-logging/README.md | 35 ++++++++++++++++++- charts/kubezero/Chart.yaml | 2 +- charts/kubezero/README.md | 6 ++-- charts/kubezero/templates/_app.yaml | 5 +++ .../templates/aws-ebs-csi-driver.yaml | 3 -- .../templates/aws-efs-csi-driver.yaml | 3 -- charts/kubezero/templates/calico.yaml | 3 -- charts/kubezero/templates/cert-manager.yaml | 5 +-- charts/kubezero/templates/istio.yaml | 3 -- charts/kubezero/templates/kiam.yaml | 3 -- .../templates/local-volume-provisioner.yaml | 3 -- charts/kubezero/templates/logging.yaml | 3 -- charts/kubezero/templates/metrics.yaml | 3 -- charts/kubezero/values-calico.yaml | 17 +++++++++ charts/kubezero/values.yaml | 4 +-- 19 files changed, 76 insertions(+), 41 deletions(-) create mode 100644 charts/kubezero/values-calico.yaml diff --git a/charts/kubezero-argo-cd/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml index 618d4b0..06a2bc7 100644 --- a/charts/kubezero-argo-cd/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application name: kubezero-argo-cd -version: 0.4.0 +version: 0.4.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-argo-cd/README.md b/charts/kubezero-argo-cd/README.md index 0ba38ee..1a2278d 100644 --- a/charts/kubezero-argo-cd/README.md +++ b/charts/kubezero-argo-cd/README.md @@ -2,7 +2,7 @@ kubezero-argo-cd ================ KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application -Current chart version is `0.4.0` +Current chart version is `0.4.1` Source code can be found [here](https://kubezero.com) @@ -60,3 +60,4 @@ Source code can be found [here](https://kubezero.com) ## Resources - https://argoproj.github.io/argo-cd/operator-manual/metrics/ +- https://raw.githubusercontent.com/argoproj/argo-cd/master/examples/dashboard.json diff --git a/charts/kubezero-argo-cd/templates/app.yaml b/charts/kubezero-argo-cd/templates/app.yaml index f997988..4bbff87 100644 --- a/charts/kubezero-argo-cd/templates/app.yaml +++ b/charts/kubezero-argo-cd/templates/app.yaml @@ -14,12 +14,13 @@ spec: helm: values: | -{{- toYaml .Values.kubezero | nindent 8 }} + {{- toYaml .Values.kubezero | nindent 8 }} destination: server: {{ .Values.kubezero.global.defaultDestination.server }} namespace: argocd + + {{- if .Values.kubezero.global.syncPolicy }} syncPolicy: - automated: - prune: true - selfHeal: false + {{- toYaml .Values.kubezero.global.syncPolicy | nindent 4 }} + {{- end }} diff --git a/charts/kubezero-argo-cd/values.yaml b/charts/kubezero-argo-cd/values.yaml index c36fc9d..be0f389 100644 --- a/charts/kubezero-argo-cd/values.yaml +++ b/charts/kubezero-argo-cd/values.yaml @@ -15,6 +15,11 @@ kubezero: # kubezero.global.defaultSource.pathPrefix -- optional path prefix within repoURL to support eg. remote subtrees pathPrefix: '' + # syncPolicy, details see: https://argoproj.github.io/argo-cd/user-guide/auto_sync + #syncPolicy: + # automated: + # prune: true + argo-cd: installCRDs: false diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md index 7e427ad..03dff66 100644 --- a/charts/kubezero-logging/README.md +++ b/charts/kubezero-logging/README.md @@ -2,7 +2,7 @@ kubezero-logging ================ KubeZero Umbrella Chart for complete EFK stack -Current chart version is `0.1.0` +Current chart version is `0.2.0` Source code can be found [here](https://kubezero.com) @@ -10,6 +10,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| +| https://kubernetes-charts.storage.googleapis.com/ | fluentd | 2.5.1 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Changes from upstream @@ -47,6 +48,38 @@ Source code can be found [here](https://kubezero.com) | es.prometheus | bool | `false` | | | es.s3Snapshot.enabled | bool | `false` | | | es.s3Snapshot.iamrole | string | `""` | | +| fluentd.configMaps."forward-input.conf" | string | `"\n @type forward\n port 24224\n bind 0.0.0.0\n skip_invalid_event true\n \n cert_path /mnt/fluentd-certs/tls.crt\n private_key_path /mnt/fluentd-certs/tls.key\n \n \n self_hostname \"#{ENV['HOSTNAME']}\"\n shared_key \"#{ENV['FLUENTD_SHARED_KEY']}\"\n \n\n"` | | +| fluentd.configMaps."output.conf" | string | `"\n @id elasticsearch\n @type elasticsearch\n @log_level info\n include_tag_key true\n id_key id\n remove_keys id\n\n # This pipeline incl. eg. GeoIP\n pipeline fluentd\n\n host \"#{ENV['OUTPUT_HOST']}\"\n port \"#{ENV['OUTPUT_PORT']}\"\n scheme \"#{ENV['OUTPUT_SCHEME']}\"\n ssl_version \"#{ENV['OUTPUT_SSL_VERSION']}\"\n ssl_verify \"#{ENV['OUTPUT_SSL_VERIFY']}\"\n user \"#{ENV['OUTPUT_USER']}\"\n password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n logstash_format true\n reload_connections false\n reconnect_on_error true\n reload_on_failure true\n request_timeout 15s\n\n \n @type file\n path /var/log/fluentd-buffers/kubernetes.system.buffer\n flush_mode interval\n flush_thread_count 2\n flush_interval 5s\n flush_at_shutdown true\n retry_type exponential_backoff\n retry_timeout 60m\n retry_max_interval 30\n chunk_limit_size \"#{ENV['OUTPUT_BUFFER_CHUNK_LIMIT']}\"\n queue_limit_length \"#{ENV['OUTPUT_BUFFER_QUEUE_LIMIT']}\"\n overflow_action drop_oldest_chunk\n \n\n"` | | +| fluentd.enabled | bool | `false` | | +| fluentd.env.OUTPUT_SSL_VERIFY | string | `"false"` | | +| fluentd.env.OUTPUT_USER | string | `"elastic"` | | +| fluentd.extraEnvVars[0].name | string | `"OUTPUT_PASSWORD"` | | +| fluentd.extraEnvVars[0].valueFrom.secretKeyRef.key | string | `"elastic"` | | +| fluentd.extraEnvVars[0].valueFrom.secretKeyRef.name | string | `"logging-es-elastic-user"` | | +| fluentd.extraEnvVars[1].name | string | `"FLUENTD_SHARED_KEY"` | | +| fluentd.extraEnvVars[1].valueFrom.secretKeyRef.key | string | `"shared_key"` | | +| fluentd.extraEnvVars[1].valueFrom.secretKeyRef.name | string | `"logging-fluentd-secret"` | | +| fluentd.extraVolumeMounts[0].mountPath | string | `"/mnt/fluentd-certs"` | | +| fluentd.extraVolumeMounts[0].name | string | `"fluentd-certs"` | | +| fluentd.extraVolumeMounts[0].readOnly | bool | `true` | | +| fluentd.extraVolumes[0].name | string | `"fluentd-certs"` | | +| fluentd.extraVolumes[0].secret.secretName | string | `"fluentd-certificate"` | | +| fluentd.istio.enabled | bool | `false` | | +| fluentd.metrics.enabled | bool | `false` | | +| fluentd.metrics.serviceMonitor.additionalLabels.release | string | `"metrics"` | | +| fluentd.metrics.serviceMonitor.enabled | bool | `true` | | +| fluentd.metrics.serviceMonitor.namespace | string | `"monitoring"` | | +| fluentd.output.host | string | `"logging-es-http"` | | +| fluentd.plugins.enabled | bool | `false` | | +| fluentd.plugins.pluginsList | string | `nil` | | +| fluentd.replicaCount | int | `2` | | +| fluentd.service.ports[0].containerPort | int | `24224` | | +| fluentd.service.ports[0].name | string | `"tcp-forward"` | | +| fluentd.service.ports[0].protocol | string | `"TCP"` | | +| fluentd.service.ports[1].containerPort | int | `9880` | | +| fluentd.service.ports[1].name | string | `"http-fluentd"` | | +| fluentd.service.ports[1].protocol | string | `"TCP"` | | +| fluentd.shared_key | string | `"cloudbender"` | | | kibana.count | int | `1` | | | kibana.istio.enabled | bool | `false` | | | kibana.istio.gateway | string | `"istio-system/ingressgateway"` | | diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index 4fbf73b..7662099 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero ArgoCD Application - Root App of Apps chart of KubeZero type: application -version: 0.4.3 +version: 0.4.4 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero/README.md b/charts/kubezero/README.md index b41419d..d99761c 100644 --- a/charts/kubezero/README.md +++ b/charts/kubezero/README.md @@ -2,7 +2,7 @@ kubezero ======== KubeZero ArgoCD Application - Root App of Apps chart of KubeZero -Current chart version is `0.4.3` +Current chart version is `0.4.4` Source code can be found [here](https://kubezero.com) @@ -18,8 +18,8 @@ Source code can be found [here](https://kubezero.com) |-----|------|---------|-------------| | aws-ebs-csi-driver.enabled | bool | `false` | | | aws-efs-csi-driver.enabled | bool | `false` | | -| calico.enabled | bool | `true` | | -| cert-manager.enabled | bool | `true` | | +| calico.enabled | bool | `false` | | +| cert-manager.enabled | bool | `false` | | | global.defaultDestination.server | string | `"https://kubernetes.default.svc"` | | | global.defaultSource.pathPrefix | string | `""` | | | global.defaultSource.repoURL | string | `"https://github.com/zero-down-time/kubezero"` | | diff --git a/charts/kubezero/templates/_app.yaml b/charts/kubezero/templates/_app.yaml index 453f52c..4c9ef8f 100644 --- a/charts/kubezero/templates/_app.yaml +++ b/charts/kubezero/templates/_app.yaml @@ -31,4 +31,9 @@ spec: destination: server: {{ .root.Values.global.defaultDestination.server }} namespace: {{ default "kube-system" .namespace }} + + {{- if .root.Values.global.syncPolicy }} + syncPolicy: + {{- toYaml .root.Values.global.syncPolicy | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/kubezero/templates/aws-ebs-csi-driver.yaml b/charts/kubezero/templates/aws-ebs-csi-driver.yaml index 8fb49a0..cf4e108 100644 --- a/charts/kubezero/templates/aws-ebs-csi-driver.yaml +++ b/charts/kubezero/templates/aws-ebs-csi-driver.yaml @@ -1,6 +1,3 @@ {{- if index .Values "aws-ebs-csi-driver" "enabled" }} {{ template "kubezero-app.app" dict "root" . "name" "aws-ebs-csi-driver" "type" "helm" }} - syncPolicy: - automated: - prune: true {{- end }} diff --git a/charts/kubezero/templates/aws-efs-csi-driver.yaml b/charts/kubezero/templates/aws-efs-csi-driver.yaml index a50005b..13a43ca 100644 --- a/charts/kubezero/templates/aws-efs-csi-driver.yaml +++ b/charts/kubezero/templates/aws-efs-csi-driver.yaml @@ -1,6 +1,3 @@ {{- if index .Values "aws-efs-csi-driver" "enabled" }} {{ template "kubezero-app.app" dict "root" . "name" "aws-efs-csi-driver" "type" "helm" }} - syncPolicy: - automated: - prune: true {{- end }} diff --git a/charts/kubezero/templates/calico.yaml b/charts/kubezero/templates/calico.yaml index 91eebd5..156da24 100644 --- a/charts/kubezero/templates/calico.yaml +++ b/charts/kubezero/templates/calico.yaml @@ -1,8 +1,5 @@ {{- if .Values.calico.enabled }} {{ template "kubezero-app.app" dict "root" . "name" "calico" "type" "helm" "retain" true }} - syncPolicy: - automated: - prune: true ignoreDifferences: - group: apiextensions.k8s.io diff --git a/charts/kubezero/templates/cert-manager.yaml b/charts/kubezero/templates/cert-manager.yaml index 847d232..e155019 100644 --- a/charts/kubezero/templates/cert-manager.yaml +++ b/charts/kubezero/templates/cert-manager.yaml @@ -1,9 +1,6 @@ {{- if index .Values "cert-manager" "enabled" }} {{ template "kubezero-app.app" dict "root" . "name" "cert-manager" "type" "helm" "namespace" "cert-manager" }} - syncPolicy: - automated: - prune: true - selfHeal: true + --- apiVersion: v1 kind: Namespace diff --git a/charts/kubezero/templates/istio.yaml b/charts/kubezero/templates/istio.yaml index 9dbd753..7ac9cb1 100644 --- a/charts/kubezero/templates/istio.yaml +++ b/charts/kubezero/templates/istio.yaml @@ -1,8 +1,5 @@ {{- if .Values.istio.enabled }} {{ template "kubezero-app.app" dict "root" . "name" "istio" "type" "helm" }} - syncPolicy: - automated: - prune: true ignoreDifferences: - group: apiextensions.k8s.io diff --git a/charts/kubezero/templates/kiam.yaml b/charts/kubezero/templates/kiam.yaml index a9c7347..78c34af 100644 --- a/charts/kubezero/templates/kiam.yaml +++ b/charts/kubezero/templates/kiam.yaml @@ -1,6 +1,3 @@ {{- if index .Values "kiam" "enabled" }} {{ template "kubezero-app.app" dict "root" . "name" "kiam" "type" "helm" }} - syncPolicy: - automated: - prune: true {{- end }} diff --git a/charts/kubezero/templates/local-volume-provisioner.yaml b/charts/kubezero/templates/local-volume-provisioner.yaml index 0db5f77..9313cfe 100644 --- a/charts/kubezero/templates/local-volume-provisioner.yaml +++ b/charts/kubezero/templates/local-volume-provisioner.yaml @@ -1,6 +1,3 @@ {{- if index .Values "local-volume-provisioner" "enabled" }} {{ template "kubezero-app.app" dict "root" . "name" "local-volume-provisioner" "type" "helm" }} - syncPolicy: - automated: - prune: true {{- end }} diff --git a/charts/kubezero/templates/logging.yaml b/charts/kubezero/templates/logging.yaml index 90cf116..0f40e65 100644 --- a/charts/kubezero/templates/logging.yaml +++ b/charts/kubezero/templates/logging.yaml @@ -1,8 +1,5 @@ {{- if index .Values "logging" "enabled" }} {{ template "kubezero-app.app" dict "root" . "name" "logging" "type" "helm" "namespace" "logging"}} - syncPolicy: - automated: - prune: true ignoreDifferences: - group: admissionregistration.k8s.io diff --git a/charts/kubezero/templates/metrics.yaml b/charts/kubezero/templates/metrics.yaml index 59c7ee3..6fba80a 100644 --- a/charts/kubezero/templates/metrics.yaml +++ b/charts/kubezero/templates/metrics.yaml @@ -1,8 +1,5 @@ {{- if index .Values "metrics" "enabled" }} {{ template "kubezero-app.app" dict "root" . "name" "metrics" "type" "helm" "namespace" "monitoring"}} - syncPolicy: - automated: - prune: true --- apiVersion: v1 kind: Namespace diff --git a/charts/kubezero/values-calico.yaml b/charts/kubezero/values-calico.yaml new file mode 100644 index 0000000..618c328 --- /dev/null +++ b/charts/kubezero/values-calico.yaml @@ -0,0 +1,17 @@ +global: + defaultDestination: + server: https://kubernetes.default.svc + + # This repoURL is used a base for all the repoURLs applications + # Setting this to a eg. private git repo incl. the use of pathPrefix allows kubezero to be + # integrated into any repository as a git subtree if for example public internet access is unavailable + defaultSource: + # defaultSource.repoURL -- default repository for argocd applications + repoURL: https://github.com/zero-down-time/kubezero + # defaultSource.targetRevision -- default tracking of repoURL + targetRevision: HEAD + # defaultSource.pathPrefix -- optional path prefix within repoURL to support eg. remote subtrees + pathPrefix: '' + +calico: + enabled: true diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 87248f7..ef4ee59 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -16,10 +16,10 @@ global: platform: aws calico: - enabled: true + enabled: false cert-manager: - enabled: true + enabled: false local-volume-provisioner: enabled: false -- 2.40.1 From c5e0187475e5c80c08033cad02fff5e3b97e906d Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 20 Aug 2020 15:55:49 +0100 Subject: [PATCH 83/96] Set Istio idle timeout --- charts/kubezero-istio/templates/istio-private-ingress.yaml | 2 ++ scripts/publish.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-istio/templates/istio-private-ingress.yaml b/charts/kubezero-istio/templates/istio-private-ingress.yaml index 08be1e7..0375308 100644 --- a/charts/kubezero-istio/templates/istio-private-ingress.yaml +++ b/charts/kubezero-istio/templates/istio-private-ingress.yaml @@ -34,6 +34,8 @@ spec: value: '"1"' - name: ISTIO_META_ROUTER_MODE value: standard + - name: ISTIO_META_IDLE_TIMEOUT + value: "60s" {{- if eq .Values.ingress.type "NodePort" }} nodeSelector: node.kubernetes.io/ingress.private: "{{ .Values.ingress.private.nodeSelector }}" diff --git a/scripts/publish.sh b/scripts/publish.sh index fd57f2c..b97d9c6 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -4,7 +4,7 @@ set -eux # all credits go to the argoproj Helm guys https://github.com/argoproj/argo-helm SRCROOT="$(cd "$(dirname "$0")/.." && pwd)" -GIT_PUSH=${GIT_PUSH:-false} +GIT_PUSH=${GIT_PUSH:-true} rm -rf $SRCROOT/output && git clone -b gh-pages ssh://git@git.zero-downtime.net:22000/ZeroDownTime/KubeZero.git $SRCROOT/output # Reset all -- 2.40.1 From 225526869e2eeeaa47d09ef7aaab891b2d73a718 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 20 Aug 2020 16:12:41 +0100 Subject: [PATCH 84/96] Set Istio idle timeout to 1h --- charts/kubezero-istio/templates/istio-private-ingress.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-istio/templates/istio-private-ingress.yaml b/charts/kubezero-istio/templates/istio-private-ingress.yaml index 0375308..ffc6288 100644 --- a/charts/kubezero-istio/templates/istio-private-ingress.yaml +++ b/charts/kubezero-istio/templates/istio-private-ingress.yaml @@ -35,7 +35,7 @@ spec: - name: ISTIO_META_ROUTER_MODE value: standard - name: ISTIO_META_IDLE_TIMEOUT - value: "60s" + value: "3600s" {{- if eq .Values.ingress.type "NodePort" }} nodeSelector: node.kubernetes.io/ingress.private: "{{ .Values.ingress.private.nodeSelector }}" -- 2.40.1 From 89d765dc538fe2476af19d353b7613c44fb6ad5c Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 20 Aug 2020 16:55:47 +0100 Subject: [PATCH 85/96] Add graceful shutdown to Ingress gateway, might need istio 1.7 to actually work though --- charts/kubezero-istio/templates/istio-private-ingress.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/kubezero-istio/templates/istio-private-ingress.yaml b/charts/kubezero-istio/templates/istio-private-ingress.yaml index ffc6288..f064c6b 100644 --- a/charts/kubezero-istio/templates/istio-private-ingress.yaml +++ b/charts/kubezero-istio/templates/istio-private-ingress.yaml @@ -30,6 +30,8 @@ spec: name: istio-private-ingressgateway {{- end }} env: + - name: TERMINATION_DRAIN_DURATION_SECONDS + value: "60" - name: ISTIO_META_HTTP10 value: '"1"' - name: ISTIO_META_ROUTER_MODE -- 2.40.1 From bdc9687bc312579725dc2da30eacfad0ce7eda09 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 20 Aug 2020 17:38:18 +0100 Subject: [PATCH 86/96] Apply graceful shutdown fixes interim like Contour --- .../templates/istio-private-ingress.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-istio/templates/istio-private-ingress.yaml b/charts/kubezero-istio/templates/istio-private-ingress.yaml index f064c6b..40b753e 100644 --- a/charts/kubezero-istio/templates/istio-private-ingress.yaml +++ b/charts/kubezero-istio/templates/istio-private-ingress.yaml @@ -30,8 +30,9 @@ spec: name: istio-private-ingressgateway {{- end }} env: - - name: TERMINATION_DRAIN_DURATION_SECONDS - value: "60" + # https://github.com/istio/istio/issues/26524 + #- name: TERMINATION_DRAIN_DURATION_SECONDS + # value: "60" - name: ISTIO_META_HTTP10 value: '"1"' - name: ISTIO_META_ROUTER_MODE @@ -53,6 +54,16 @@ spec: rollingUpdate: maxSurge: 100% maxUnavailable: 25% + overlays: + - apiVersion: apps/v1 + kind: Deployment + name: istio-private-ingressgateway + patches: + - path: spec.template.spec.containers.[name:istio-proxy].lifecycle + value: {"preStop": {"exec": {"command": ["sh", "-c", "curl -X POST http://localhost:15000/healthcheck/fail && sleep 30"]}}} + - path: spec.template.spec.terminationGracePeriodSeconds + value: 90 + values: gateways: istio-ingressgateway: -- 2.40.1 From 5dac264e17ae1db81e40aa41725206dc1b91edfa Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 20 Aug 2020 18:32:01 +0100 Subject: [PATCH 87/96] Also apply improved healthcehck handling and draining to public ingress --- charts/kubezero-istio/templates/istio.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/charts/kubezero-istio/templates/istio.yaml b/charts/kubezero-istio/templates/istio.yaml index bea5e87..25a26f4 100644 --- a/charts/kubezero-istio/templates/istio.yaml +++ b/charts/kubezero-istio/templates/istio.yaml @@ -34,10 +34,15 @@ spec: name: istio-ingressgateway {{- end }} env: + # https://github.com/istio/istio/issues/26524 + #- name: TERMINATION_DRAIN_DURATION_SECONDS + # value: "60" - name: ISTIO_META_HTTP10 value: '"1"' - name: ISTIO_META_ROUTER_MODE value: standard + - name: ISTIO_META_IDLE_TIMEOUT + value: "3600s" {{- if eq .Values.ingress.type "NodePort" }} nodeSelector: node.kubernetes.io/ingress.public: "30080_30443" @@ -53,6 +58,16 @@ spec: rollingUpdate: maxSurge: 100% maxUnavailable: 25% + overlays: + - apiVersion: apps/v1 + kind: Deployment + name: istio-ingressgateway + patches: + - path: spec.template.spec.containers.[name:istio-proxy].lifecycle + value: {"preStop": {"exec": {"command": ["sh", "-c", "curl -X POST http://localhost:15000/healthcheck/fail && sleep 30"]}}} + - path: spec.template.spec.terminationGracePeriodSeconds + value: 90 + name: istio-ingressgateway pilot: enabled: true -- 2.40.1 From 3cfa3512e66eaf6748a3aca51a89435e9e7f1297 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 21 Aug 2020 14:17:47 +0100 Subject: [PATCH 88/96] Switch istio ingress to http healthchecks, more tuning --- .../templates/ingress-gateway.yaml | 2 ++ .../templates/istio-private-ingress.yaml | 29 +++++++++++-------- charts/kubezero-istio/templates/istio.yaml | 9 ++++-- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/charts/kubezero-istio/templates/ingress-gateway.yaml b/charts/kubezero-istio/templates/ingress-gateway.yaml index f183be0..63b9de4 100644 --- a/charts/kubezero-istio/templates/ingress-gateway.yaml +++ b/charts/kubezero-istio/templates/ingress-gateway.yaml @@ -29,6 +29,7 @@ spec: serverCertificate: /etc/istio/ingressgateway-certs/tls.crt credentialName: public-ingress-cert +{{- if .Values.ingress.private.enabled }} --- apiVersion: networking.istio.io/v1alpha3 kind: Gateway @@ -78,3 +79,4 @@ spec: protocol: TCP hosts: - "*" +{{- end }} diff --git a/charts/kubezero-istio/templates/istio-private-ingress.yaml b/charts/kubezero-istio/templates/istio-private-ingress.yaml index 40b753e..7966539 100644 --- a/charts/kubezero-istio/templates/istio-private-ingress.yaml +++ b/charts/kubezero-istio/templates/istio-private-ingress.yaml @@ -80,6 +80,11 @@ spec: values: istio-private-ingressgateway type: {{ default "NodePort" .Values.ingress.type }} ports: + - name: http-status + port: 15021 + {{- if eq .Values.ingress.type "NodePort" }} + nodePort: 31021 + {{- end }} - name: http2 port: 80 {{- if eq .Values.ingress.type "NodePort" }} @@ -90,31 +95,31 @@ spec: {{- if eq .Values.ingress.type "NodePort" }} nodePort: 31443 {{- end }} - - name: amqp - port: 5672 - {{- if eq .Values.ingress.type "NodePort" }} - nodePort: 30672 - {{- end }} - - name: amqps - port: 5671 - {{- if eq .Values.ingress.type "NodePort" }} - nodePort: 30671 - {{- end }} - name: fluentd-forward port: 24224 {{- if eq .Values.ingress.type "NodePort" }} nodePort: 31224 {{- end }} + - name: amqps + port: 5671 + {{- if eq .Values.ingress.type "NodePort" }} + nodePort: 31671 + {{- end }} + - name: amqp + port: 5672 + {{- if eq .Values.ingress.type "NodePort" }} + nodePort: 31672 + {{- end }} sds: enabled: true image: node-agent-k8s resources: limits: - cpu: 2000m + #cpu: 2000m memory: 1024Mi requests: cpu: 100m - memory: 128Mi + memory: 64Mi secretVolumes: - mountPath: /etc/istio/ingressgateway-certs name: ingressgateway-certs diff --git a/charts/kubezero-istio/templates/istio.yaml b/charts/kubezero-istio/templates/istio.yaml index 25a26f4..50831ae 100644 --- a/charts/kubezero-istio/templates/istio.yaml +++ b/charts/kubezero-istio/templates/istio.yaml @@ -49,11 +49,11 @@ spec: {{- end }} resources: limits: - cpu: 2000m + # cpu: 2000m memory: 1024Mi requests: cpu: 100m - memory: 128Mi + memory: 64Mi strategy: rollingUpdate: maxSurge: 100% @@ -112,6 +112,11 @@ spec: values: istio-ingressgateway type: {{ default "NodePort" .Values.ingress.type }} ports: + - name: http-status + port: 15021 + {{- if eq .Values.ingress.type "NodePort" }} + nodePort: 30021 + {{- end }} - name: http2 port: 80 {{- if eq .Values.ingress.type "NodePort" }} -- 2.40.1 From 47fa52369478cab680b44843725c48867c2182fb Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 21 Aug 2020 20:39:55 +0100 Subject: [PATCH 89/96] Refactor argo apps factory --- charts/kubezero/Chart.yaml | 2 +- charts/kubezero/templates/_app.tpl | 37 ++++++++++++++++++ charts/kubezero/templates/_app.yaml | 39 ------------------- .../templates/aws-ebs-csi-driver.yaml | 2 +- .../templates/aws-efs-csi-driver.yaml | 2 +- charts/kubezero/templates/calico.yaml | 2 +- charts/kubezero/templates/cert-manager.yaml | 2 +- charts/kubezero/templates/istio.yaml | 2 +- charts/kubezero/templates/kiam.yaml | 2 +- .../templates/local-volume-provisioner.yaml | 2 +- charts/kubezero/templates/logging.yaml | 2 +- charts/kubezero/templates/metrics.yaml | 2 +- charts/kubezero/values.yaml | 4 ++ 13 files changed, 51 insertions(+), 49 deletions(-) create mode 100644 charts/kubezero/templates/_app.tpl delete mode 100644 charts/kubezero/templates/_app.yaml diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index 7662099..a39534d 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero ArgoCD Application - Root App of Apps chart of KubeZero type: application -version: 0.4.4 +version: 0.4.5 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero/templates/_app.tpl b/charts/kubezero/templates/_app.tpl new file mode 100644 index 0000000..166550d --- /dev/null +++ b/charts/kubezero/templates/_app.tpl @@ -0,0 +1,37 @@ +{{- define "kubezero-app.app" }} +{{- $name := regexReplaceAll "kubezero/templates/([a-z-]*)..*" .Template.Name "${1}" }} +{{- $my_values := index .Values $name "values" }} + +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ $name }} + namespace: argocd + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} + {{- if not ( index .Values $name "retain" ) }} + finalizers: + - resources-finalizer.argocd.argoproj.io + {{- end }} +spec: + project: kubezero + + source: + repoURL: {{ .Values.global.defaultSource.repoURL }} + targetRevision: {{ .Values.global.defaultSource.targetRevision }} + path: {{ .Values.global.defaultSource.pathPrefix}}charts/kubezero-{{ $name }} + {{- if $my_values }} + helm: + values: | +{{- toYaml $my_values | nindent 8 }} + {{- end }} + + destination: + server: {{ .Values.global.defaultDestination.server }} + namespace: {{ default "kube-system" ( index .Values $name "namespace" ) }} + + {{- if .Values.global.syncPolicy }} + syncPolicy: + {{- toYaml .Values.global.syncPolicy | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/kubezero/templates/_app.yaml b/charts/kubezero/templates/_app.yaml deleted file mode 100644 index 4c9ef8f..0000000 --- a/charts/kubezero/templates/_app.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- define "kubezero-app.app" }} -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: {{ .name | lower }} - namespace: argocd - labels: -{{ include "kubezero-lib.labels" .root | indent 4 }} - {{- if not .retain }} - finalizers: - - resources-finalizer.argocd.argoproj.io - {{- end }} -spec: - project: kubezero - - source: - repoURL: {{ .root.Values.global.defaultSource.repoURL }} - targetRevision: {{ .root.Values.global.defaultSource.targetRevision }} - {{- if eq .type "helm" }} - {{- $my_values := index .root.Values .name "values" }} - path: {{ .root.Values.global.defaultSource.pathPrefix}}charts/kubezero-{{ .name }} - {{- if $my_values }} - helm: - values: | -{{- toYaml $my_values | nindent 8 }} - {{- end }} - {{- else }} - path: {{ .root.Values.global.defaultSource.pathPrefix }}artifacts/kubezero-{{ .name }} - {{- end }} - - destination: - server: {{ .root.Values.global.defaultDestination.server }} - namespace: {{ default "kube-system" .namespace }} - - {{- if .root.Values.global.syncPolicy }} - syncPolicy: - {{- toYaml .root.Values.global.syncPolicy | nindent 4 }} - {{- end }} -{{- end }} diff --git a/charts/kubezero/templates/aws-ebs-csi-driver.yaml b/charts/kubezero/templates/aws-ebs-csi-driver.yaml index cf4e108..66d7215 100644 --- a/charts/kubezero/templates/aws-ebs-csi-driver.yaml +++ b/charts/kubezero/templates/aws-ebs-csi-driver.yaml @@ -1,3 +1,3 @@ {{- if index .Values "aws-ebs-csi-driver" "enabled" }} -{{ template "kubezero-app.app" dict "root" . "name" "aws-ebs-csi-driver" "type" "helm" }} +{{ include "kubezero-app.app" . }} {{- end }} diff --git a/charts/kubezero/templates/aws-efs-csi-driver.yaml b/charts/kubezero/templates/aws-efs-csi-driver.yaml index 13a43ca..f3675d3 100644 --- a/charts/kubezero/templates/aws-efs-csi-driver.yaml +++ b/charts/kubezero/templates/aws-efs-csi-driver.yaml @@ -1,3 +1,3 @@ {{- if index .Values "aws-efs-csi-driver" "enabled" }} -{{ template "kubezero-app.app" dict "root" . "name" "aws-efs-csi-driver" "type" "helm" }} +{{ include "kubezero-app.app" . }} {{- end }} diff --git a/charts/kubezero/templates/calico.yaml b/charts/kubezero/templates/calico.yaml index 156da24..fa61d2a 100644 --- a/charts/kubezero/templates/calico.yaml +++ b/charts/kubezero/templates/calico.yaml @@ -1,5 +1,5 @@ {{- if .Values.calico.enabled }} -{{ template "kubezero-app.app" dict "root" . "name" "calico" "type" "helm" "retain" true }} +{{ include "kubezero-app.app" . }} ignoreDifferences: - group: apiextensions.k8s.io diff --git a/charts/kubezero/templates/cert-manager.yaml b/charts/kubezero/templates/cert-manager.yaml index e155019..d7bd21b 100644 --- a/charts/kubezero/templates/cert-manager.yaml +++ b/charts/kubezero/templates/cert-manager.yaml @@ -1,5 +1,5 @@ {{- if index .Values "cert-manager" "enabled" }} -{{ template "kubezero-app.app" dict "root" . "name" "cert-manager" "type" "helm" "namespace" "cert-manager" }} +{{ include "kubezero-app.app" . }} --- apiVersion: v1 diff --git a/charts/kubezero/templates/istio.yaml b/charts/kubezero/templates/istio.yaml index 7ac9cb1..f2cc09a 100644 --- a/charts/kubezero/templates/istio.yaml +++ b/charts/kubezero/templates/istio.yaml @@ -1,5 +1,5 @@ {{- if .Values.istio.enabled }} -{{ template "kubezero-app.app" dict "root" . "name" "istio" "type" "helm" }} +{{ include "kubezero-app.app" . }} ignoreDifferences: - group: apiextensions.k8s.io diff --git a/charts/kubezero/templates/kiam.yaml b/charts/kubezero/templates/kiam.yaml index 78c34af..a23ed62 100644 --- a/charts/kubezero/templates/kiam.yaml +++ b/charts/kubezero/templates/kiam.yaml @@ -1,3 +1,3 @@ {{- if index .Values "kiam" "enabled" }} -{{ template "kubezero-app.app" dict "root" . "name" "kiam" "type" "helm" }} +{{ include "kubezero-app.app" . }} {{- end }} diff --git a/charts/kubezero/templates/local-volume-provisioner.yaml b/charts/kubezero/templates/local-volume-provisioner.yaml index 9313cfe..5961f72 100644 --- a/charts/kubezero/templates/local-volume-provisioner.yaml +++ b/charts/kubezero/templates/local-volume-provisioner.yaml @@ -1,3 +1,3 @@ {{- if index .Values "local-volume-provisioner" "enabled" }} -{{ template "kubezero-app.app" dict "root" . "name" "local-volume-provisioner" "type" "helm" }} +{{ include "kubezero-app.app" . }} {{- end }} diff --git a/charts/kubezero/templates/logging.yaml b/charts/kubezero/templates/logging.yaml index 0f40e65..ef8f5e4 100644 --- a/charts/kubezero/templates/logging.yaml +++ b/charts/kubezero/templates/logging.yaml @@ -1,5 +1,5 @@ {{- if index .Values "logging" "enabled" }} -{{ template "kubezero-app.app" dict "root" . "name" "logging" "type" "helm" "namespace" "logging"}} +{{ include "kubezero-app.app" . }} ignoreDifferences: - group: admissionregistration.k8s.io diff --git a/charts/kubezero/templates/metrics.yaml b/charts/kubezero/templates/metrics.yaml index 6fba80a..2bef83f 100644 --- a/charts/kubezero/templates/metrics.yaml +++ b/charts/kubezero/templates/metrics.yaml @@ -1,5 +1,5 @@ {{- if index .Values "metrics" "enabled" }} -{{ template "kubezero-app.app" dict "root" . "name" "metrics" "type" "helm" "namespace" "monitoring"}} +{{ include "kubezero-app.app" . }} --- apiVersion: v1 kind: Namespace diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index ef4ee59..be2b3ec 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -17,9 +17,11 @@ platform: aws calico: enabled: false + retain: true cert-manager: enabled: false + namespace: cert-manager local-volume-provisioner: enabled: false @@ -38,6 +40,8 @@ istio: metrics: enabled: false + namespace: monitoring logging: enabled: false + namespace: logging -- 2.40.1 From be346b592f49c265e1a36852402b4e343f492b58 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sat, 22 Aug 2020 18:27:18 +0100 Subject: [PATCH 90/96] Add fluent-bit support to kuberzero-logging, istio fixes --- charts/fluent-bit | 1 + .../templates/istio-private-ingress.yaml | 8 +- charts/kubezero-lib/templates/_helpers.tpl | 2 +- charts/kubezero-logging/Chart.yaml | 7 +- charts/kubezero-logging/values-fluentbit.yaml | 8 ++ charts/kubezero-logging/values.yaml | 108 +++++++++++++++++- deploy/templates/values.yaml | 12 ++ deploy/values.yaml | 2 + scripts/publish.sh | 5 +- 9 files changed, 144 insertions(+), 9 deletions(-) create mode 120000 charts/fluent-bit create mode 100644 charts/kubezero-logging/values-fluentbit.yaml diff --git a/charts/fluent-bit b/charts/fluent-bit new file mode 120000 index 0000000..cd73d77 --- /dev/null +++ b/charts/fluent-bit @@ -0,0 +1 @@ +../../helm-charts/charts/fluent-bit \ No newline at end of file diff --git a/charts/kubezero-istio/templates/istio-private-ingress.yaml b/charts/kubezero-istio/templates/istio-private-ingress.yaml index 7966539..bcb998c 100644 --- a/charts/kubezero-istio/templates/istio-private-ingress.yaml +++ b/charts/kubezero-istio/templates/istio-private-ingress.yaml @@ -45,11 +45,11 @@ spec: {{- end }} resources: limits: - cpu: 2000m + # cpu: 2000m memory: 1024Mi requests: cpu: 100m - memory: 128Mi + memory: 64Mi strategy: rollingUpdate: maxSurge: 100% @@ -115,11 +115,11 @@ spec: image: node-agent-k8s resources: limits: - #cpu: 2000m + cpu: 2000m memory: 1024Mi requests: cpu: 100m - memory: 64Mi + memory: 128Mi secretVolumes: - mountPath: /etc/istio/ingressgateway-certs name: ingressgateway-certs diff --git a/charts/kubezero-lib/templates/_helpers.tpl b/charts/kubezero-lib/templates/_helpers.tpl index f0de548..808f73b 100644 --- a/charts/kubezero-lib/templates/_helpers.tpl +++ b/charts/kubezero-lib/templates/_helpers.tpl @@ -2,7 +2,7 @@ Common set of labels */ -}} {{- define "kubezero-lib.labels" -}} -helm.sh/chart: {{ include "kubezero-lib.chart . }} +helm.sh/chart: {{ include "kubezero-lib.chart" . }} app.kubernetes.io/name: {{ include "kubezero-lib.name" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/part-of: kubezero diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index 0750821..71cdf36 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.2.0 +version: 0.3.1 appVersion: 1.2.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -22,4 +22,9 @@ dependencies: version: 2.5.1 repository: https://kubernetes-charts.storage.googleapis.com/ condition: fluentd.enabled + - name: fluent-bit + version: 0.6.3 + repository: https://zero-down-time.github.io/kubezero/ + # repository: https://fluent.github.io/helm-charts + condition: fluent-bit.enabled kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-logging/values-fluentbit.yaml b/charts/kubezero-logging/values-fluentbit.yaml new file mode 100644 index 0000000..298b39a --- /dev/null +++ b/charts/kubezero-logging/values-fluentbit.yaml @@ -0,0 +1,8 @@ +fluent-bit: + enabled: true + + metrics: + enabled: true + + url: fluentd.example.com + diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 1a537d4..6f34ef8 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -125,7 +125,7 @@ fluentd: id_key id remove_keys id - # This pipeline incl. eg. GeoIP + # KubeZero pipeline incl. GeoIP etc. pipeline fluentd host "#{ENV['OUTPUT_HOST']}" @@ -193,3 +193,109 @@ fluentd: # # # + + +fluent-bit: + enabled: true + test: + enabled: false + + config: + outputs: | + [OUTPUT] + Match * + Name forward + Host fluentd + Port 24224 + tls on + tls.verify off + Shared_Key cloudbender + + inputs: | + [INPUT] + Name tail + Path /var/log/containers/*.log + Parser cri + Tag kube.* + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + DB /var/log/flb_kube.db + DB.Sync Normal + + filters: | + [FILTER] + Name kubernetes + Match kube.* + Merge_Log On + Keep_Log Off + K8S-Logging.Parser On + K8S-Logging.Exclude On + + [FILTER] + Name lua + Match kube.* + script /fluent-bit/etc/functions.lua + call dedot + + customParsers: | + [PARSER] + # http://rubular.com/r/tjUt3Awgg4 + Name cri + Format regex + Regex ^(?