From 593c296e92e1fe6d70606a79313efbd207cede0e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 28 Jan 2022 17:22:12 +0100 Subject: [PATCH] feat: update aws-node-termination handler chart, first version of forseti --- charts/kubezero-addons/Chart.yaml | 2 +- .../aws-node-termination-handler/Chart.yaml | 38 +- .../aws-node-termination-handler/README.md | 266 +++++++------ .../example-values-imds-linux.yaml | 5 + .../example-values-imds-windows.yaml | 5 + .../example-values-queue.yaml | 13 + .../templates/NOTES.txt | 11 +- .../templates/_helpers.tpl | 73 ++-- .../templates/clusterrole.yaml | 4 + .../templates/clusterrolebinding.yaml | 14 +- .../templates/daemonset.linux.yaml | 356 ++++++++--------- .../templates/daemonset.windows.yaml | 322 ++++++++------- .../templates/deployment.yaml | 368 +++++++++-------- .../templates/pdb.yaml | 4 +- .../templates/podmonitor.yaml | 6 +- .../templates/psp.yaml | 2 +- .../templates/service.yaml | 2 +- .../templates/serviceaccount.yaml | 9 +- .../templates/servicemonitor.yaml | 6 +- .../aws-node-termination-handler/test.yaml | 175 -------- .../aws-node-termination-handler/values.yaml | 372 ++++++++++-------- charts/kubezero-addons/nth.patch | 65 +-- .../templates/awsController/deployment.yaml | 30 -- .../templates/awsController/rbac.yaml | 31 -- .../templates/forseti/deployment.yaml | 83 ++++ .../templates/forseti/rbac.yaml | 104 +++++ .../templates/forseti/service.yaml | 16 + charts/kubezero-addons/update.sh | 5 +- charts/kubezero-addons/values.yaml | 24 +- 29 files changed, 1211 insertions(+), 1200 deletions(-) create mode 100644 charts/kubezero-addons/charts/aws-node-termination-handler/example-values-imds-linux.yaml create mode 100644 charts/kubezero-addons/charts/aws-node-termination-handler/example-values-imds-windows.yaml create mode 100644 charts/kubezero-addons/charts/aws-node-termination-handler/example-values-queue.yaml delete mode 100644 charts/kubezero-addons/charts/aws-node-termination-handler/test.yaml delete mode 100644 charts/kubezero-addons/templates/awsController/deployment.yaml delete mode 100644 charts/kubezero-addons/templates/awsController/rbac.yaml create mode 100644 charts/kubezero-addons/templates/forseti/deployment.yaml create mode 100644 charts/kubezero-addons/templates/forseti/rbac.yaml create mode 100644 charts/kubezero-addons/templates/forseti/service.yaml diff --git a/charts/kubezero-addons/Chart.yaml b/charts/kubezero-addons/Chart.yaml index d6c810e..4766737 100644 --- a/charts/kubezero-addons/Chart.yaml +++ b/charts/kubezero-addons/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-addons description: KubeZero umbrella chart for various optional cluster addons type: application -version: 0.3.1 +version: 0.4.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml index 832f402..c7dd973 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml @@ -1,21 +1,25 @@ -apiVersion: v1 +apiVersion: v2 +name: aws-node-termination-handler +description: A Helm chart for the AWS Node Termination Handler. +type: application +version: 0.16.0 appVersion: 1.14.0 -description: A Helm chart for the AWS Node Termination Handler +kubeVersion: ">= 1.16-0" +keywords: + - aws + - eks + - ec2 + - node-termination + - spot home: https://github.com/aws/eks-charts icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png -keywords: -- eks -- ec2 -- node-termination -- spot -maintainers: -- email: bwagner5@users.noreply.github.com - name: Brandon Wagner - url: https://github.com/bwagner5 -- email: jillmon@users.noreply.github.com - name: Jillian Montalvo - url: https://github.com/jillmon -name: aws-node-termination-handler sources: -- https://github.com/aws/eks-charts -version: 0.16.0 + - https://github.com/aws/aws-node-termination-handler/ + - https://github.com/aws/eks-charts/ +maintainers: + - name: Brandon Wagner + url: https://github.com/bwagner5 + email: bwagner5@users.noreply.github.com + - name: Jillian Montalvo + url: https://github.com/jillmon + email: jillmon@users.noreply.github.com diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/README.md b/charts/kubezero-addons/charts/aws-node-termination-handler/README.md index 186109e..4876692 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/README.md +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/README.md @@ -1,172 +1,170 @@ # AWS Node Termination Handler -AWS Node Termination Handler Helm chart for Kubernetes. For more information on this project see the project repo at https://github.com/aws/aws-node-termination-handler. +AWS Node Termination Handler Helm chart for Kubernetes. For more information on this project see the project repo at [github.com/aws/aws-node-termination-handler](https://github.com/aws/aws-node-termination-handler). ## Prerequisites -* Kubernetes >= 1.14 +- _Kubernetes_ >= v1.16 ## Installing the Chart -Add the EKS repository to Helm: +Before you can install the chart you will need to add the `aws` repo to [Helm](https://helm.sh/). -```sh -helm repo add eks https://aws.github.io/eks-charts +```shell +helm repo add eks https://aws.github.io/eks-charts/ ``` -Install AWS Node Termination Handler: +After you've installed the repo you can install the chart, the following command will install the chart with the release name `aws-node-termination-handler` and the default configuration to the `kube-system` namespace. -To install the chart with the release name aws-node-termination-handler and default configuration: - -```sh -helm upgrade --install aws-node-termination-handler \ - --namespace kube-system \ - eks/aws-node-termination-handler +```shell +helm upgrade --install --namespace kube-system aws-node-termination-handler eks/aws-node-termination-handler ``` -To install into an EKS cluster where the Node Termination Handler is already installed, you can run: +To install the chart on an EKS cluster where the AWS Node Termination Handler is already installed, you can run the following command. -```sh -helm upgrade --install --recreate-pods --force \ - aws-node-termination-handler --namespace kube-system eks/aws-node-termination-handler +```shell +helm upgrade --install --namespace kube-system aws-node-termination-handler eks/aws-node-termination-handler --recreate-pods --force ``` -If you receive an error similar to `Error: release aws-node-termination-handler -failed: "aws-node-termination-handler" already exists`, simply rerun -the above command. +If you receive an error similar to the one below simply rerun the above command. -The [configuration](#configuration) section lists the parameters that can be configured during installation. +> Error: release aws-node-termination-handler failed: "aws-node-termination-handler" already exists -## Uninstalling the Chart +To uninstall the `aws-node-termination-handler` chart installation from the `kube-system` namespace run the following command. -To uninstall/delete the `aws-node-termination-handler` deployment: - -```sh -helm delete --purge aws-node-termination-handler +```shell +helm delete --namespace kube-system aws-node-termination-handler ``` -The command removes all the Kubernetes components associated with the chart and deletes the release. - ## Configuration -The following tables lists the configurable parameters of the chart and their default values. +The following tables lists the configurable parameters of the chart and their default values. These values are split up into the [common configuration](#common-configuration) shared by all AWS Node Termination Handler modes, [queue configuration](#queue-processor-mode-configuration) used when AWS Node Termination Handler is in in queue-processor mode, and [IMDS configuration](#imds-mode-configuration) used when AWS Node Termination Handler is in IMDS mode; for more information about the different modes see the project [README](https://github.com/aws/aws-node-termination-handler/blob/main/README.md). -### AWS Node Termination Handler Common Configuration +### Common Configuration -The configuration in this table applies to both queue-processor mode and IMDS mode. +The configuration in this table applies to all AWS Node Termination Handler modes. -Parameter | Description | Default ---- | --- | --- -`deleteLocalData` | Tells kubectl to continue even if there are pods using emptyDir (local data that will be deleted when the node is drained). | `true` -`gracePeriod` | (DEPRECATED: Renamed to podTerminationGracePeriod) The time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used, which defaults to 30 seconds if not specified. | `-1` -`podTerminationGracePeriod` | The time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used, which defaults to 30 seconds if not specified. | `-1` -`nodeTerminationGracePeriod` | Period of time in seconds given to each NODE to terminate gracefully. Node draining will be scheduled based on this value to optimize the amount of compute time, but still safely drain the node before an event. | `120` -`ignoreDaemonSets` | Causes kubectl to skip daemon set managed pods | `true` -`instanceMetadataURL` | The URL of EC2 instance metadata. This shouldn't need to be changed unless you are testing. | `http://169.254.169.254:80` -`webhookURL` | Posts event data to URL upon instance interruption action | `` -`webhookURLSecretName` | Pass Webhook URL as a secret. Secret Key: `webhookurl`, Value: `` | None -`webhookProxy` | Uses the specified HTTP(S) proxy for sending webhooks | `` -`webhookHeaders` | Replaces the default webhook headers. | `{"Content-type":"application/json"}` -`webhookTemplate` | Replaces the default webhook message template. | `{"text":"[NTH][Instance Interruption] EventID: {{ .EventID }} - Kind: {{ .Kind }} - Instance: {{ .InstanceID }} - Node: {{ .NodeName }} - Description: {{ .Description }} - Start Time: {{ .StartTime }}"}` -`webhookTemplateConfigMapName` | Pass Webhook template file as configmap | None -`webhookTemplateConfigMapKey` | Name of the template file stored in the configmap| None -`metadataTries` | The number of times to try requesting metadata. If you would like 2 retries, set metadata-tries to 3. | `3` -`cordonOnly` | If true, nodes will be cordoned but not drained when an interruption event occurs. | `false` -`taintNode` | If true, nodes will be tainted when an interruption event occurs. Currently used taint keys are `aws-node-termination-handler/scheduled-maintenance`, `aws-node-termination-handler/spot-itn`, `aws-node-termination-handler/asg-lifecycle-termination` and `aws-node-termination-handler/rebalance-recommendation`| `false` -`jsonLogging` | If true, use JSON-formatted logs instead of human readable logs. | `false` -`logLevel` | Sets the log level (INFO, DEBUG, or ERROR) | `INFO` -`enablePrometheusServer` | If true, start an http server exposing `/metrics` endpoint for prometheus. | `false` -`prometheusServerPort` | Replaces the default HTTP port for exposing prometheus metrics. | `9092` -`enableProbesServer` | If true, start an http server exposing `/healthz` endpoint for probes. | `false` -`probesServerPort` | Replaces the default HTTP port for exposing probes endpoint. | `8080` -`probesServerEndpoint` | Replaces the default endpoint for exposing probes endpoint. | `/healthz` -`emitKubernetesEvents` | If `true`, Kubernetes events will be emitted when interruption events are received and when actions are taken on Kubernetes nodes. In IMDS Processor mode a default set of annotations with all the node metadata gathered from IMDS will be attached to each event. More information [here](https://github.com/aws/aws-node-termination-handler/blob/main/docs/kubernetes_events.md) | `false` -`kubernetesExtraEventsAnnotations` | A comma-separated list of `key=value` extra annotations to attach to all emitted Kubernetes events. Example: `first=annotation,sample.annotation/number=two"` | None +| Parameter | Description | Default | +| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | +| `image.repository` | Image repository. | `public.ecr.aws/aws-ec2/aws-node-termination-handler` | +| `image.tag` | Image tag. | `v{{ .Chart.AppVersion}}` | +| `image.pullPolicy` | Image pull policy. | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets. | `[]` | +| `nameOverride` | Override the `name` of the chart. | `""` | +| `fullnameOverride` | Override the `fullname` of the chart. | `""` | +| `serviceAccount.create` | If `true`, create a new service account. | `true` | +| `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the full name template. | `nil` | +| `serviceAccount.annotations` | Annotations to add to the service account. | `{}` | +| `rbac.create` | If `true`, create the RBAC resources. | `true` | +| `rbac.pspEnabled` | If `true`, create a pod security policy resource. | `true` | +| `customLabels` | Labels to add to all resource metadata. | `{}` | +| `podLabels` | Labels to add to the pod. | `{}` | +| `podAnnotations` | Annotations to add to the pod. | `{}` | +| `podSecurityContext` | Security context for the pod. | _See values.yaml_ | +| `securityContext` | Security context for the _aws-node-termination-handler_ container. | _See values.yaml_ | +| `terminationGracePeriodSeconds` | The termination grace period for the pod. | `nil` | +| `resources` | Resource requests and limits for the _aws-node-termination-handler_ container. | `{}` | +| `nodeSelector` | Expressions to select a node by it's labels for pod assignment. In IMDS mode this has a higher priority than `daemonsetNodeSelector` (for backwards compatibility) but shouldn't be used. | `{}` | +| `affinity` | Affinity settings for pod assignment. In IMDS mode this has a higher priority than `daemonsetAffinity` (for backwards compatibility) but shouldn't be used. | `{}` | +| `tolerations` | Tolerations for pod assignment. In IMDS mode this has a higher priority than `daemonsetTolerations` (for backwards compatibility) but shouldn't be used. | `[]` | +| `extraEnv` | Additional environment variables for the _aws-node-termination-handler_ container. | `[]` | +| `probes` | The Kubernetes liveness probe configuration. | _See values.yaml_ | +| `logLevel` | Sets the log level (`info`,`debug`, or `error`) | `info` | +| `jsonLogging` | If `true`, use JSON-formatted logs instead of human readable logs. | `false` | +| `enablePrometheusServer` | If `true`, start an http server exposing `/metrics` endpoint for _Prometheus_. | `false` | +| `prometheusServerPort` | Replaces the default HTTP port for exposing _Prometheus_ metrics. | `9092` | +| `dryRun` | If `true`, only log if a node would be drained. | `false` | +| `cordonOnly` | If `true`, nodes will be cordoned but not drained when an interruption event occurs. | `false` | +| `taintNode` | If `true`, nodes will be tainted when an interruption event occurs. Currently used taint keys are `aws-node-termination-handler/scheduled-maintenance`, `aws-node-termination-handler/spot-itn`, `aws-node-termination-handler/asg-lifecycle-termination` and `aws-node-termination-handler/rebalance-recommendation`. | `false` | +| `deleteLocalData` | If `true`, continue even if there are pods using local data that will be deleted when the node is drained. | `true` | +| `ignoreDaemonSets` | If `true`, skip terminating daemon set managed pods. | `true` | +| `podTerminationGracePeriod` | The time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used, which defaults to 30 seconds if not specified for the pod. | `-1` | +| `nodeTerminationGracePeriod` | Period of time in seconds given to each node to terminate gracefully. Node draining will be scheduled based on this value to optimize the amount of compute time, but still safely drain the node before an event. | `120` | +| `emitKubernetesEvents` | If `true`, Kubernetes events will be emitted when interruption events are received and when actions are taken on Kubernetes nodes. In IMDS Processor mode a default set of annotations with all the node metadata gathered from IMDS will be attached to each event. More information [here](https://github.com/aws/aws-node-termination-handler/blob/main/docs/kubernetes_events.md). | `false` | +| `kubernetesEventsExtraAnnotations` | A comma-separated list of `key=value` extra annotations to attach to all emitted Kubernetes events (e.g. `first=annotation,sample.annotation/number=two"`). | `""` | +| `webhookURL` | Posts event data to URL upon instance interruption action. | `""` | +| `webhookURLSecretName` | Pass the webhook URL as a Secret using the key `webhookurl`. | `""` | +| `webhookHeaders` | Replace the default webhook headers (e.g. `{"Content-type":"application/json"}`). | `""` | +| `webhookProxy` | Uses the specified HTTP(S) proxy for sending webhook data. | `""` | +| `webhookTemplate` | Replaces the default webhook message template (e.g. `{"text":"[NTH][Instance Interruption] EventID: {{ .EventID }} - Kind: {{ .Kind }} - Instance: {{ .InstanceID }} - Node: {{ .NodeName }} - Description: {{ .Description }} - Start Time: {{ .StartTime }}"}`). | `""` | +| `webhookTemplateConfigMapName` | Pass the webhook template file as a configmap. | "``" | +| `webhookTemplateConfigMapKey` | Name of the Configmap key storing the template file. | `""` | +| `enableSqsTerminationDraining` | If `true`, this turns on queue-processor mode which drains nodes when an SQS termination event is received. | `false` | -### AWS Node Termination Handler - Queue-Processor Mode Configuration +### Queue-Processor Mode Configuration -Parameter | Description | Default ---- | --- | --- -`enableSqsTerminationDraining` | If true, this turns on queue-processor mode which drains nodes when an SQS termination event is received. | `false` -`queueURL` | Listens for messages on the specified SQS queue URL | None -`awsRegion` | If specified, use the AWS region for AWS API calls, else NTH will try to find the region through AWS_REGION env var, IMDS, or the specified queue URL | `` -`checkASGTagBeforeDraining` | If true, check that the instance is tagged with "aws-node-termination-handler/managed" as the key before draining the node | `true` -`managedAsgTag` | The tag to ensure is on a node if checkASGTagBeforeDraining is true | `aws-node-termination-handler/managed` -`workers` | The maximum amount of parallel event processors | `10` -`replicas` | The number of replicas in the NTH deployment when using queue-processor mode (NOTE: increasing replicas may cause duplicate webhooks since NTH pods are stateless) | `1` -`podDisruptionBudget` | Limit the disruption for controller pods, requires at least 2 controller replicas | `{}` -`serviceMonitor.create` | If `true`, create a ServiceMonitor (this requires enableSqsTerminationDraining and enablePrometheusServer to be set) | `false` -`serviceMonitor.interval` | Prometheus scrape interval | `30s` -`serviceMonitor.sampleLimit` | Number of scraped samples accepted | `5000` -`serviceMonitor.labels` | Additional ServiceMonitor metadata labels | `{}` -`serviceMonitor.namespace` | Override ServiceMonitor Helm release namespace | `{{ .Release.Namespace }}` +The configuration in this table applies to AWS Node Termination Handler in queue-processor mode. -### AWS Node Termination Handler - IMDS Mode Configuration +| Parameter | Description | Default | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------- | +| `replicas` | The number of replicas in the deployment when using queue-processor mode (NOTE: increasing replicas may cause duplicate webhooks since pods are stateless). | `1` | +| `strategy` | Specify the update strategy for the deployment. | `{}` | +| `podDisruptionBudget` | Limit the disruption for controller pods, requires at least 2 controller replicas. | `{}` | +| `serviceMonitor.create` | If `true`, create a ServiceMonitor. This requires `enablePrometheusServer: true`. | `false` | +| `serviceMonitor.namespace` | Override ServiceMonitor _Helm_ release namespace. | `nil` | +| `serviceMonitor.labels` | Additional ServiceMonitor metadata labels. | `{}` | +| `serviceMonitor.interval` | _Prometheus_ scrape interval. | `30s` | +| `serviceMonitor.sampleLimit` | Number of scraped samples accepted. | `5000` | +| `priorityClassName` | Name of the PriorityClass to use for the Deployment. | `system-cluster-critical` | +| `awsRegion` | If specified, use the AWS region for AWS API calls, else NTH will try to find the region through the `AWS_REGION` environment variable, IMDS, or the specified queue URL. | `""` | +| `queueURL` | Listens for messages on the specified SQS queue URL. | `""` | +| `workers` | The maximum amount of parallel event processors to handle concurrent events. | `10` | +| `checkASGTagBeforeDraining` | If `true`, check that the instance is tagged with the `managedAsgTag` before draining the node. | `true` | +| `managedAsgTag` | The node tag to check if `checkASGTagBeforeDraining` is `true`. | `aws-node-termination-handler/managed` | +| `assumeAsgTagPropagation` | If `true`, assume that ASG tags will be appear on the ASG's instances. | `false` | -Parameter | Description | Default ---- | --- | --- -`enableScheduledEventDraining` | [EXPERIMENTAL] If true, drain nodes before the maintenance window starts for an EC2 instance scheduled event | `false` -`enableSpotInterruptionDraining` | If true, drain nodes when the spot interruption termination notice is received | `true` -`enableRebalanceDraining` | If true, drain nodes when the rebalance recommendation notice is received | `false` -`enableRebalanceMonitoring` | If true, cordon nodes when the rebalance recommendation notice is received. If you'd like to drain the node in addition to cordoning, then also set `enableRebalanceDraining`. | `false` -`useHostNetwork` | If `true`, enables `hostNetwork` for the Linux DaemonSet. NOTE: setting this to `false` may cause issues accessing IMDSv2 if your account is not configured with an IP hop count of 2 | `true` -`podMonitor.create` | If `true`, create a PodMonitor (this requires enableSqsTerminationDraining to not be set and enablePrometheusServer to be set) | `false` -`podMonitor.interval` | Prometheus scrape interval | `30s` -`podMonitor.sampleLimit` | Number of scraped samples accepted | `5000` -`podMonitor.labels` | Additional PodMonitor metadata labels | `{}` -`podMonitor.namespace` | Override PodMonitor Helm release namespace | `{{ .Release.Namespace }}` +### IMDS Mode Configuration -### Kubernetes Configuration +The configuration in this table applies to AWS Node Termination Handler in IMDS mode. -Parameter | Description | Default ---- | --- | --- -`image.repository` | image repository | `public.ecr.aws/aws-ec2/aws-node-termination-handler` -`image.tag` | image tag | `` -`image.pullPolicy` | image pull policy | `IfNotPresent` -`image.pullSecrets` | image pull secrets (for private docker registries) | `[]` -`affinity` | node/pod affinities | None -`linuxAffinity` | Linux node/pod affinities | None -`windowsAffinity` | Windows node/pod affinities | None -`podAnnotations` | annotations to add to each pod | `{}` -`linuxPodAnnotations` | Linux annotations to add to each pod | `{}` -`windowsPodAnnotations` | Windows annotations to add to each pod | `{}` -`podLabels` | labels to add to each pod | `{}` -`linuxPodLabels` | labels to add to each Linux pod | `{}` -`windowsPodLabels` | labels to add to each Windows pod | `{}` -`priorityClassName` | Name of the priorityClass | `system-node-critical` -`resources` | Resources for the pods | `requests.cpu: 50m, requests.memory: 64Mi, limits.cpu: 100m, limits.memory: 128Mi` -`dnsPolicy` | DaemonSet DNS policy | Linux: `ClusterFirstWithHostNet`, Windows: `ClusterFirst` -`nodeSelector` | Tells the all daemon sets where to place the node-termination-handler pods. For example: `lifecycle: "Ec2Spot"`, `on-demand: "false"`, `aws.amazon.com/purchaseType: "spot"`, etc. Value must be a valid yaml expression. | `{}` -`linuxNodeSelector` | Tells the Linux daemon set where to place the node-termination-handler pods. For example: `lifecycle: "Ec2Spot"`, `on-demand: "false"`, `aws.amazon.com/purchaseType: "spot"`, etc. Value must be a valid yaml expression. | `{}` -`windowsNodeSelector` | Tells the Windows daemon set where to place the node-termination-handler pods. For example: `lifecycle: "Ec2Spot"`, `on-demand: "false"`, `aws.amazon.com/purchaseType: "spot"`, etc. Value must be a valid yaml expression. | `{}` -`tolerations` | list of node taints to tolerate | `[ {"operator": "Exists"} ]` -`rbac.create` | if `true`, create and use RBAC resources | `true` -`rbac.pspEnabled` | If `true`, create and use a restricted pod security policy | `true` -`serviceAccount.create` | If `true`, create a new service account | `true` -`serviceAccount.name` | Service account to be used | None -`serviceAccount.annotations` | Specifies the annotations for ServiceAccount | `{}` -`securityContext.runAsUserID` | User ID to run the container | `1000` -`securityContext.runAsGroupID` | Group ID to run the container | `1000` -`nodeSelectorTermsOs` | Operating System Node Selector Key | `kubernetes.io/os` -`nodeSelectorTermsArch` | CPU Architecture Node Selector Key | `kubernetes.io/arch` -`targetNodeOs` | Space separated list of node OS's to target, e.g. "linux", "windows", "linux windows". Note: Windows support is experimental. | `"linux"` -`updateStrategy` | Update strategy for the all DaemonSets (Linux and Windows) | `type=RollingUpdate,rollingUpdate.maxUnavailable=1` -`linuxUpdateStrategy` | Update strategy for the Linux DaemonSet | `type=RollingUpdate,rollingUpdate.maxUnavailable=1` -`windowsUpdateStrategy` | Update strategy for the Windows DaemonSet | `type=RollingUpdate,rollingUpdate.maxUnavailable=1` -`extraEnv` | Additional environment variables to inject into pod configuration | `[]` +| Parameter | Description | Default | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------- | +| `targetNodeOs` | Space separated list of node OS's to target (e.g. `"linux"`, `"windows"`, `"linux windows"`). Windows support is **EXPERIMENTAL**. | `"linux"` | +| `linuxPodLabels` | Labels to add to each Linux pod. | `{}` | +| `windowsPodLabels` | Labels to add to each Windows pod. | `{}` | +| `linuxPodAnnotations` | Annotations to add to each Linux pod. | `{}` | +| `windowsPodAnnotations` | Annotations to add to each Windows pod. | `{}` | +| `updateStrategy` | Update strategy for the all DaemonSets. | _See values.yaml_ | +| `daemonsetPriorityClassName` | Name of the PriorityClass to use for all DaemonSets. | `system-node-critical` | +| `podMonitor.create` | If `true`, create a PodMonitor. This requires `enablePrometheusServer: true`. | `false` | +| `podMonitor.namespace` | Override PodMonitor _Helm_ release namespace. | `nil` | +| `podMonitor.labels` | Additional PodMonitor metadata labels | `{}` | +| `podMonitor.interval` | _Prometheus_ scrape interval. | `30s` | +| `podMonitor.sampleLimit` | Number of scraped samples accepted. | `5000` | +| `useHostNetwork` | If `true`, enables `hostNetwork` for the Linux DaemonSet. NOTE: setting this to `false` may cause issues accessing IMDSv2 if your account is not configured with an IP hop count of 2 see [Metrics Endpoint Considerations](#metrics-endpoint-considerations) | `true` | +| `dnsPolicy` | If specified, this overrides `linuxDnsPolicy` and `windowsDnsPolicy` with a single policy. | `""` | +| `linuxDnsPolicy` | DNS policy for the Linux DaemonSet. | `""` | +| `windowsDnsPolicy` | DNS policy for the Windows DaemonSet. | `""` | +| `daemonsetNodeSelector` | Expressions to select a node by it's labels for DaemonSet pod assignment. For backwards compatibility the `nodeSelector` value has priority over this but shouldn't be used. | `{}` | +| `linuxNodeSelector` | Override `daemonsetNodeSelector` for the Linux DaemonSet. | `{}` | +| `windowsNodeSelector` | Override `daemonsetNodeSelector` for the Windows DaemonSet. | `{}` | +| `daemonsetAffinity` | Affinity settings for DaemonSet pod assignment. For backwards compatibility the `affinity` has priority over this but shouldn't be used. | `{}` | +| `linuxAffinity` | Override `daemonsetAffinity` for the Linux DaemonSet. | `{}` | +| `windowsAffinity` | Override `daemonsetAffinity` for the Windows DaemonSet. | `{}` | +| `daemonsetTolerations` | Tolerations for DaemonSet pod assignment. For backwards compatibility the `tolerations` has priority over this but shouldn't be used. | `[]` | +| `linuxTolerations` | Override `daemonsetTolerations` for the Linux DaemonSet. | `[]` | +| `windowsTolerations` | Override `daemonsetTolerations` for the Linux DaemonSet. | `[]` | +| `enableProbesServer` | If `true`, start an http server exposing `/healthz` endpoint for probes. | `false` | +| `metadataTries` | The number of times to try requesting metadata. | `3` | +| `enableSpotInterruptionDraining` | If `true`, drain nodes when the spot interruption termination notice is received. | `true` | +| `enableScheduledEventDraining` | If `true`, drain nodes before the maintenance window starts for an EC2 instance scheduled event. This is **EXPERIMENTAL**. | `false` | +| `enableRebalanceMonitoring` | If `true`, cordon nodes when the rebalance recommendation notice is received. If you'd like to drain the node in addition to cordoning, then also set `enableRebalanceDraining`. | `false` | +| `enableRebalanceDraining` | If `true`, drain nodes when the rebalance recommendation notice is received. | `false` | -### Testing Configuration (NOT RECOMMENDED FOR PROD DEPLOYMENTS) +### Testing Configuration -Parameter | Description | Default ---- | --- | --- -`procUptimeFile` | (Used for Testing) Specify the uptime file | `/proc/uptime` -`awsEndpoint` | (Used for testing) If specified, use the AWS endpoint to make API calls | None -`awsSecretAccessKey` | (Used for testing) Pass-thru env var | None -`awsAccessKeyID` | (Used for testing) Pass-thru env var | None -`dryRun` | If true, only log if a node would be drained | `false` +The configuration in this table applies to AWS Node Termination Handler testing and is **NOT RECOMMENDED** FOR PRODUCTION DEPLOYMENTS. -## Metrics endpoint consideration +| Parameter | Description | Default | +| --------------------- | --------------------------------------------------------------------------------- | -------------- | +| `awsEndpoint` | (Used for testing) If specified, use the provided AWS endpoint to make API calls. | `""` | +| `awsSecretAccessKey` | (Used for testing) Pass-thru environment variable. | `nil` | +| `awsAccessKeyID` | (Used for testing) Pass-thru environment variable. | `nil` | +| `instanceMetadataURL` | (Used for testing) If specified, use the provided metadata URL. | `""` | +| `procUptimeFile` | (Used for Testing) Specify the uptime file. | `/proc/uptime` | -NTH in IMDS mode runs as a DaemonSet w/ `host_networking=true` by default. If the prometheus server is enabled, nothing else will be able to bind to the configured port (by default `:9092`) in the root network namespace. Therefore, it will need to have a firewall/security group configured on the nodes to block access to the `/metrics` endpoint. +## Metrics Endpoint Considerations -You can switch NTH in IMDS mode to run w/ `host_networking=false`, but you will need to make sure that IMDSv1 is enabled or IMDSv2 IP hop count will need to be incremented to 2. https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html +AWS Node Termination HAndler in IMDS mode runs as a DaemonSet with `useHostNetwork: true` by default. If the Prometheus server is enabled with `enablePrometheusServer: true` nothing else will be able to bind to the configured port (by default `prometheusServerPort: 9092`) in the root network namespace. Therefore, it will need to have a firewall/security group configured on the nodes to block access to the `/metrics` endpoint. + +You can switch NTH in IMDS mode to run w/ `useHostNetwork: false`, but you will need to make sure that IMDSv1 is enabled or IMDSv2 IP hop count will need to be incremented to 2 (see the [IMDSv2 documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html). diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/example-values-imds-linux.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/example-values-imds-linux.yaml new file mode 100644 index 0000000..c0df26c --- /dev/null +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/example-values-imds-linux.yaml @@ -0,0 +1,5 @@ +enableSqsTerminationDraining: false + +targetNodeOs: linux + +enableProbesServer: true diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/example-values-imds-windows.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/example-values-imds-windows.yaml new file mode 100644 index 0000000..193978e --- /dev/null +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/example-values-imds-windows.yaml @@ -0,0 +1,5 @@ +enableSqsTerminationDraining: false + +targetNodeOs: windows + +enableProbesServer: true diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/example-values-queue.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/example-values-queue.yaml new file mode 100644 index 0000000..fd204ab --- /dev/null +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/example-values-queue.yaml @@ -0,0 +1,13 @@ +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::99999999:role/nth-role + +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + +enableSqsTerminationDraining: true diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/NOTES.txt b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/NOTES.txt index f2dd1ce..d0aaf70 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/NOTES.txt +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/NOTES.txt @@ -1,3 +1,8 @@ -{{ .Release.Name }} has been installed or updated. To check the status of pods, run: - -kubectl get pods --namespace {{ .Values.namespace }} +*********************************************************************** +* AWS Node Termination Handler * +*********************************************************************** + Chart version: {{ .Chart.Version }} + App version: {{ .Chart.AppVersion }} + Image tag: {{ include "aws-node-termination-handler.image" . }} + Mode : {{ if .Values.enableSqsTerminationDraining }}Queue Processor{{ else }}IMDS{{ end }} +*********************************************************************** diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/_helpers.tpl b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/_helpers.tpl index 249a9c9..45f06f4 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/_helpers.tpl +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/_helpers.tpl @@ -1,4 +1,5 @@ {{/* vim: set filetype=mustache: */}} + {{/* Expand the name of the chart. */}} @@ -28,20 +29,32 @@ If release name contains chart name it will be used as a full name. Equivalent to "aws-node-termination-handler.fullname" except that "-win" indicator is appended to the end. Name will not exceed 63 characters. */}} -{{- define "aws-node-termination-handler.fullname.windows" -}} +{{- define "aws-node-termination-handler.fullnameWindows" -}} {{- include "aws-node-termination-handler.fullname" . | trunc 59 | trimSuffix "-" | printf "%s-win" -}} {{- end -}} +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aws-node-termination-handler.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Common labels */}} {{- define "aws-node-termination-handler.labels" -}} -helm.sh/chart: {{ include "aws-node-termination-handler.chart" . }} {{ include "aws-node-termination-handler.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} +app.kubernetes.io/component: {{ .Release.Name }} +app.kubernetes.io/part-of: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ include "aws-node-termination-handler.chart" . }} +{{- with .Values.customLabels }} +{{ toYaml . }} +{{- end }} {{- end -}} {{/* @@ -53,10 +66,19 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} {{/* -Create chart name and version as used by the chart label. +Selector labels for the deployment */}} -{{- define "aws-node-termination-handler.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- define "aws-node-termination-handler.selectorLabelsDeployment" -}} +{{ include "aws-node-termination-handler.selectorLabels" . }} +app.kubernetes.io/component: deployment +{{- end -}} + +{{/* +Selector labels for the daemonset +*/}} +{{- define "aws-node-termination-handler.selectorLabelsDaemonset" -}} +{{ include "aws-node-termination-handler.selectorLabels" . }} +app.kubernetes.io/component: daemonset {{- end -}} {{/* @@ -71,36 +93,17 @@ Create the name of the service account to use {{- end -}} {{/* -Get the default node selector term prefix. +The image to use */}} -{{- define "aws-node-termination-handler.defaultNodeSelectorTermsPrefix" -}} -kubernetes.io -{{- end -}} +{{- define "aws-node-termination-handler.image" -}} +{{- printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} -{{/* -Get the default node selector OS term. -*/}} -{{- define "aws-node-termination-handler.defaultNodeSelectorTermsOs" -}} - {{- list (include "aws-node-termination-handler.defaultNodeSelectorTermsPrefix" .) "os" | join "/" -}} -{{- end -}} - -{{/* -Get the default node selector Arch term. -*/}} -{{- define "aws-node-termination-handler.defaultNodeSelectorTermsArch" -}} - {{- list (include "aws-node-termination-handler.defaultNodeSelectorTermsPrefix" .) "arch" | join "/" -}} -{{- end -}} - -{{/* -Get the node selector OS term. -*/}} -{{- define "aws-node-termination-handler.nodeSelectorTermsOs" -}} - {{- or .Values.nodeSelectorTermsOs (include "aws-node-termination-handler.defaultNodeSelectorTermsOs" .) -}} -{{- end -}} - -{{/* -Get the node selector Arch term. -*/}} -{{- define "aws-node-termination-handler.nodeSelectorTermsArch" -}} - {{- or .Values.nodeSelectorTermsArch (include "aws-node-termination-handler.defaultNodeSelectorTermsArch" .) -}} +{{/* Get PodDisruptionBudget API Version */}} +{{- define "aws-node-termination-handler.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} {{- end -}} diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/clusterrole.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/clusterrole.yaml index 8418ff3..43c2b03 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/clusterrole.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/clusterrole.yaml @@ -1,7 +1,10 @@ +{{- if .Values.rbac.create -}} kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "aws-node-termination-handler.fullname" . }} + labels: + {{- include "aws-node-termination-handler.labels" . | nindent 4 }} rules: - apiGroups: - "" @@ -46,3 +49,4 @@ rules: - create - patch {{- end }} +{{- end -}} diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/clusterrolebinding.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/clusterrolebinding.yaml index b5c2532..1058df1 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/clusterrolebinding.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/clusterrolebinding.yaml @@ -1,12 +1,16 @@ +{{- if .Values.rbac.create -}} kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "aws-node-termination-handler.fullname" . }} -subjects: -- kind: ServiceAccount - name: {{ template "aws-node-termination-handler.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} + labels: + {{- include "aws-node-termination-handler.labels" . | nindent 4 }} roleRef: + apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: {{ include "aws-node-termination-handler.fullname" . }} - apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: {{ template "aws-node-termination-handler.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.linux.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.linux.yaml index 0a09aa6..199879c 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.linux.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.linux.yaml @@ -1,226 +1,198 @@ -{{- if and (lower .Values.targetNodeOs | contains "linux") (not .Values.enableSqsTerminationDraining) -}} +{{- if and (not .Values.enableSqsTerminationDraining) (lower .Values.targetNodeOs | contains "linux") -}} apiVersion: apps/v1 kind: DaemonSet metadata: name: {{ include "aws-node-termination-handler.fullname" . }} - namespace: {{ .Release.Namespace }} labels: {{- include "aws-node-termination-handler.labels" . | nindent 4 }} spec: - {{- if (or .Values.updateStrategy .Values.linuxUpdateStrategy) }} + {{- with .Values.updateStrategy }} updateStrategy: - {{- with .Values.updateStrategy }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.linuxUpdateStrategy }} - {{- toYaml . | nindent 4 }} - {{- end }} + {{- toYaml . | nindent 4 }} {{- end }} selector: matchLabels: - {{- include "aws-node-termination-handler.selectorLabels" . | nindent 6 }} - {{ include "aws-node-termination-handler.nodeSelectorTermsOs" . }}: linux + {{- include "aws-node-termination-handler.selectorLabelsDaemonset" . | nindent 6 }} + kubernetes.io/os: linux template: metadata: - {{- if (or .Values.podAnnotations .Values.linuxPodAnnotations) }} - annotations: - {{- range $key, $value := (mergeOverwrite (dict) .Values.podAnnotations .Values.linuxPodAnnotations) }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} labels: - {{- include "aws-node-termination-handler.selectorLabels" . | nindent 8 }} + {{- include "aws-node-termination-handler.selectorLabelsDaemonset" . | nindent 8 }} + kubernetes.io/os: linux k8s-app: aws-node-termination-handler - {{ include "aws-node-termination-handler.nodeSelectorTermsOs" . }}: linux - {{- range $key, $value := (mergeOverwrite (dict) .Values.podLabels .Values.linuxPodLabels) }} - {{ $key }}: {{ $value | quote }} + {{- with (mergeOverwrite (dict) .Values.podLabels .Values.linuxPodLabels) }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if or .Values.podAnnotations .Values.linuxPodAnnotations }} + annotations: + {{- toYaml (mergeOverwrite (dict) .Values.podAnnotations .Values.linuxPodAnnotations) | nindent 8 }} {{- end }} spec: - volumes: - - name: "uptime" - hostPath: - path: {{ .Values.procUptimeFile | default "/proc/uptime" | quote }} - {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} - - name: "webhook-template" - configMap: - name: {{ .Values.webhookTemplateConfigMapName }} - {{- end }} - priorityClassName: {{ .Values.priorityClassName | quote }} - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: {{ include "aws-node-termination-handler.nodeSelectorTermsOs" . | quote }} - operator: In - values: - - linux - - key: {{ include "aws-node-termination-handler.nodeSelectorTermsArch" . | quote }} - operator: In - values: - - amd64 - - arm64 - - arm - - key: "eks.amazonaws.com/compute-type" - operator: NotIn - values: - - fargate - {{- with .Values.affinity }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.linuxAffinity }} - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ template "aws-node-termination-handler.serviceAccountName" . }} + {{- with .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "aws-node-termination-handler.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.daemonsetPriorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} hostNetwork: {{ .Values.useHostNetwork }} - dnsPolicy: {{ .Values.dnsPolicy | default "ClusterFirstWithHostNet" | quote }} + dnsPolicy: {{ default .Values.linuxDnsPolicy .Values.dnsPolicy }} containers: - - name: {{ include "aws-node-termination-handler.name" . }} - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} + - name: aws-node-termination-handler + {{- with .Values.securityContext }} securityContext: - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: {{ .Values.securityContext.runAsUserID }} - runAsGroup: {{ .Values.securityContext.runAsGroupID }} - allowPrivilegeEscalation: false - volumeMounts: - - name: "uptime" - mountPath: {{ .Values.procUptimeFile | default "/proc/uptime" | quote }} - readOnly: true - {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} - - name: "webhook-template" - mountPath: "/config/" - {{- end }} + {{- toYaml . | nindent 12 }} + {{- end }} + image: {{ include "aws-node-termination-handler.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: DELETE_LOCAL_DATA - value: {{ .Values.deleteLocalData | quote }} - - name: IGNORE_DAEMON_SETS - value: {{ .Values.ignoreDaemonSets | quote }} - - name: GRACE_PERIOD - value: {{ .Values.gracePeriod | quote }} - - name: POD_TERMINATION_GRACE_PERIOD - value: {{ .Values.podTerminationGracePeriod | quote }} - - name: INSTANCE_METADATA_URL - value: {{ .Values.instanceMetadataURL | quote }} - - name: NODE_TERMINATION_GRACE_PERIOD - value: {{ .Values.nodeTerminationGracePeriod | quote }} - - name: WEBHOOK_URL - {{- if .Values.webhookURLSecretName }} - valueFrom: - secretKeyRef: - name: {{ .Values.webhookURLSecretName }} - key: webhookurl - {{- else }} - value: {{ .Values.webhookURL | quote }} - {{- end }} - - name: WEBHOOK_HEADERS - value: {{ .Values.webhookHeaders | quote }} - {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} - - name: WEBHOOK_TEMPLATE_FILE - value: {{ print "/config/" .Values.webhookTemplateConfigMapKey | quote }} - {{- end }} - - name: WEBHOOK_TEMPLATE - value: {{ .Values.webhookTemplate | quote }} - - name: DRY_RUN - value: {{ .Values.dryRun | quote }} - - name: ENABLE_SPOT_INTERRUPTION_DRAINING - value: {{ .Values.enableSpotInterruptionDraining | quote }} - - name: ENABLE_SCHEDULED_EVENT_DRAINING - value: {{ .Values.enableScheduledEventDraining | quote }} - - name: ENABLE_REBALANCE_MONITORING - value: {{ .Values.enableRebalanceMonitoring | quote }} - - name: ENABLE_REBALANCE_DRAINING - value: {{ .Values.enableRebalanceDraining | quote }} - - name: CHECK_ASG_TAG_BEFORE_DRAINING - value: {{ .Values.checkASGTagBeforeDraining | quote }} - - name: MANAGED_ASG_TAG - value: {{ .Values.managedAsgTag | quote }} - - name: METADATA_TRIES - value: {{ .Values.metadataTries | quote }} - - name: CORDON_ONLY - value: {{ .Values.cordonOnly | quote }} - - name: TAINT_NODE - value: {{ .Values.taintNode | quote }} - - name: JSON_LOGGING - value: {{ .Values.jsonLogging | quote }} - - name: LOG_LEVEL - value: {{ .Values.logLevel | quote }} - - name: WEBHOOK_PROXY - value: {{ .Values.webhookProxy | quote }} - - name: UPTIME_FROM_FILE - value: {{ .Values.procUptimeFile | quote }} - - name: ENABLE_PROMETHEUS_SERVER - value: {{ .Values.enablePrometheusServer | quote }} - - name: PROMETHEUS_SERVER_PORT - value: {{ .Values.prometheusServerPort | quote }} - - name: ENABLE_PROBES_SERVER - value: {{ .Values.enableProbesServer | quote }} - - name: PROBES_SERVER_PORT - value: {{ .Values.probesServerPort | quote }} - - name: PROBES_SERVER_ENDPOINT - value: {{ .Values.probesServerEndpoint | quote }} - - name: EMIT_KUBERNETES_EVENTS - value: {{ .Values.emitKubernetesEvents | quote }} - - name: KUBERNETES_EVENTS_EXTRA_ANNOTATIONS - value: {{ .Values.kubernetesEventsExtraAnnotations | quote }} -{{- range $key, $value := .Values.extraEnv }} - - name: {{ $key }} - value: {{ $value | quote }} -{{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ENABLE_PROBES_SERVER + value: {{ .Values.enableProbesServer | quote }} + - name: PROBES_SERVER_PORT + value: {{ .Values.probes.httpGet.port | quote }} + - name: PROBES_SERVER_ENDPOINT + value: {{ .Values.probes.httpGet.path | quote }} + - name: LOG_LEVEL + value: {{ .Values.logLevel | quote }} + - name: JSON_LOGGING + value: {{ .Values.jsonLogging | quote }} + - name: ENABLE_PROMETHEUS_SERVER + value: {{ .Values.enablePrometheusServer | quote }} + - name: PROMETHEUS_SERVER_PORT + value: {{ .Values.prometheusServerPort | quote }} + {{- with .Values.instanceMetadataURL }} + - name: INSTANCE_METADATA_URL + value: {{ . | quote }} + {{- end }} + - name: METADATA_TRIES + value: {{ .Values.metadataTries | quote }} + - name: DRY_RUN + value: {{ .Values.dryRun | quote }} + - name: CORDON_ONLY + value: {{ .Values.cordonOnly | quote }} + - name: TAINT_NODE + value: {{ .Values.taintNode | quote }} + - name: DELETE_LOCAL_DATA + value: {{ .Values.deleteLocalData | quote }} + - name: IGNORE_DAEMON_SETS + value: {{ .Values.ignoreDaemonSets | quote }} + - name: POD_TERMINATION_GRACE_PERIOD + value: {{ .Values.podTerminationGracePeriod | quote }} + - name: NODE_TERMINATION_GRACE_PERIOD + value: {{ .Values.nodeTerminationGracePeriod | quote }} + - name: EMIT_KUBERNETES_EVENTS + value: {{ .Values.emitKubernetesEvents | quote }} + {{- with .Values.kubernetesEventsExtraAnnotations }} + - name: KUBERNETES_EVENTS_EXTRA_ANNOTATIONS + value: {{ . | quote }} + {{- end }} + {{- if or .Values.webhookURL .Values.webhookURLSecretName }} + - name: WEBHOOK_URL + {{- if .Values.webhookURLSecretName }} + valueFrom: + secretKeyRef: + name: {{ .Values.webhookURLSecretName }} + key: webhookurl + {{- else }} + value: {{ .Values.webhookURL | quote }} + {{- end }} + {{- end }} + {{- with .Values.webhookHeaders }} + - name: WEBHOOK_HEADERS + value: {{ . | quote }} + {{- end }} + {{- with .Values.webhookProxy }} + - name: WEBHOOK_PROXY + value: {{ . | quote }} + {{- end }} + {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + - name: WEBHOOK_TEMPLATE_FILE + value: {{ print "/config/" .Values.webhookTemplateConfigMapKey | quote }} + {{- else if .Values.webhookTemplate }} + - name: WEBHOOK_TEMPLATE + value: {{ .Values.webhookTemplate | quote }} + {{- end }} + - name: ENABLE_SPOT_INTERRUPTION_DRAINING + value: {{ .Values.enableSpotInterruptionDraining | quote }} + - name: ENABLE_SCHEDULED_EVENT_DRAINING + value: {{ .Values.enableScheduledEventDraining | quote }} + - name: ENABLE_REBALANCE_MONITORING + value: {{ .Values.enableRebalanceMonitoring | quote }} + - name: ENABLE_REBALANCE_DRAINING + value: {{ .Values.enableRebalanceDraining | quote }} + - name: ENABLE_SQS_TERMINATION_DRAINING + value: "false" + - name: UPTIME_FROM_FILE + value: {{ .Values.procUptimeFile | quote }} {{- if or .Values.enablePrometheusServer .Values.enableProbesServer }} ports: + {{- if .Values.enableProbesServer }} + - name: liveness-probe + protocol: TCP + containerPort: {{ .Values.probes.httpGet.port }} {{- end }} {{- if .Values.enablePrometheusServer }} - - containerPort: {{ .Values.prometheusServerPort }} - {{- if .Values.useHostNetwork }} - hostPort: {{ .Values.prometheusServerPort }} - {{- end }} - name: http-metrics - protocol: TCP + - name: http-metrics + protocol: TCP + containerPort: {{ .Values.prometheusServerPort }} {{- end }} - {{- if .Values.enableProbesServer }} - - containerPort: {{ .Values.probesServerPort }} - {{- if .Values.useHostNetwork }} - hostPort: {{ .Values.probesServerPort }} - {{- end }} - name: liveness-probe - protocol: TCP {{- end }} {{- if .Values.enableProbesServer }} livenessProbe: {{- toYaml .Values.probes | nindent 12 }} {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: uptime + mountPath: {{ .Values.procUptimeFile }} + readOnly: true + {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + - name: webhook-template + mountPath: /config/ + {{- end }} + volumes: + - name: uptime + hostPath: + path: {{ .Values.procUptimeFile | default "/proc/uptime" }} + {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + - name: webhook-template + configMap: + name: {{ .Values.webhookTemplateConfigMapName }} + {{- end }} nodeSelector: - {{ include "aws-node-termination-handler.nodeSelectorTermsOs" . }}: linux - {{- with .Values.nodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.linuxNodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- if .Values.image.pullSecrets }} - imagePullSecrets: - {{- range .Values.image.pullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: + kubernetes.io/os: linux + {{- with default .Values.daemonsetNodeSelector (default .Values.nodeSelector .Values.linuxNodeSelector) }} {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} + {{- if or .Values.daemonsetAffinity (or .Values.affinity .Values.linuxAffinity) }} + affinity: + {{- toYaml (default .Values.daemonsetAffinity (default .Values.affinity .Values.linuxAffinity)) | nindent 8 }} + {{- end }} + {{- if or .Values.daemonsetTolerations (or .Values.tolerations .Values.linuxTolerations) }} + tolerations: + {{- toYaml (default .Values.daemonsetTolerations (default .Values.tolerations .Values.linuxTolerations )) | nindent 8 }} + {{- end }} {{- end -}} diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.windows.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.windows.yaml index d5dfa6f..ea7f833 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.windows.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.windows.yaml @@ -1,196 +1,192 @@ -{{- if and (lower .Values.targetNodeOs | contains "windows") (not .Values.enableSqsTerminationDraining) -}} +{{- if and (not .Values.enableSqsTerminationDraining) (lower .Values.targetNodeOs | contains "windows") -}} apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ include "aws-node-termination-handler.fullname.windows" . }} - namespace: {{ .Release.Namespace }} + name: {{ include "aws-node-termination-handler.fullnameWindows" . }} labels: {{- include "aws-node-termination-handler.labels" . | nindent 4 }} spec: - {{- if (or .Values.updateStrategy .Values.windowsUpdateStrategy) }} + {{- with .Values.updateStrategy }} updateStrategy: - {{- with .Values.updateStrategy }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.windowsUpdateStrategy }} - {{- toYaml . | nindent 4 }} - {{- end }} + {{- toYaml . | nindent 4 }} {{- end }} selector: matchLabels: - {{- include "aws-node-termination-handler.selectorLabels" . | nindent 6 }} - {{ include "aws-node-termination-handler.nodeSelectorTermsOs" . }}: windows + {{- include "aws-node-termination-handler.selectorLabelsDaemonset" . | nindent 6 }} + kubernetes.io/os: windows template: metadata: - {{- if (or .Values.podAnnotations .Values.windowsPodAnnotations) }} - annotations: - {{- range $key, $value := (mergeOverwrite (dict) .Values.podAnnotations .Values.windowsPodAnnotations) }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} labels: - {{- include "aws-node-termination-handler.selectorLabels" . | nindent 8 }} + {{- include "aws-node-termination-handler.selectorLabelsDaemonset" . | nindent 8 }} + kubernetes.io/os: windows k8s-app: aws-node-termination-handler - {{ include "aws-node-termination-handler.nodeSelectorTermsOs" . }}: windows - {{- range $key, $value := (mergeOverwrite (dict) .Values.podLabels .Values.windowsPodLabels) }} - {{ $key }}: {{ $value | quote }} + {{- with (mergeOverwrite (dict) .Values.podLabels .Values.windowsPodLabels) }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if or .Values.podAnnotations .Values.windowsPodAnnotations }} + annotations: + {{- toYaml (mergeOverwrite (dict) .Values.podAnnotations .Values.windowsPodAnnotations) | nindent 8 }} {{- end }} spec: - {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} - volumes: - - name: "webhook-template" - configMap: - name: {{ .Values.webhookTemplateConfigMapName }} + {{- with .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} {{- end }} - priorityClassName: {{ .Values.priorityClassName | quote }} - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: {{ include "aws-node-termination-handler.nodeSelectorTermsOs" . | quote }} - operator: In - values: - - windows - - key: {{ include "aws-node-termination-handler.nodeSelectorTermsArch" . | quote }} - operator: In - values: - - amd64 - {{- with .Values.affinity }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.windowsAffinity }} - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ template "aws-node-termination-handler.serviceAccountName" . }} - dnsPolicy: {{ .Values.dnsPolicy | default "ClusterFirst" | quote }} + serviceAccountName: {{ include "aws-node-termination-handler.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.daemonsetPriorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + hostNetwork: false + dnsPolicy: {{ default .Values.windowsDnsPolicy .Values.dnsPolicy }} containers: - - name: {{ include "aws-node-termination-handler.name" . }} - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + - name: aws-node-termination-handler + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: {{ include "aws-node-termination-handler.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} - volumeMounts: - - name: "webhook-template" - mountPath: "/config/" - {{- end }} env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: DELETE_LOCAL_DATA - value: {{ .Values.deleteLocalData | quote }} - - name: IGNORE_DAEMON_SETS - value: {{ .Values.ignoreDaemonSets | quote }} - - name: GRACE_PERIOD - value: {{ .Values.gracePeriod | quote }} - - name: POD_TERMINATION_GRACE_PERIOD - value: {{ .Values.podTerminationGracePeriod | quote }} - - name: INSTANCE_METADATA_URL - value: {{ .Values.instanceMetadataURL | quote }} - - name: NODE_TERMINATION_GRACE_PERIOD - value: {{ .Values.nodeTerminationGracePeriod | quote }} - - name: WEBHOOK_URL - value: {{ .Values.webhookURL | quote }} - - name: WEBHOOK_HEADERS - value: {{ .Values.webhookHeaders | quote }} - {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} - - name: WEBHOOK_TEMPLATE_FILE - value: {{ print "/config/" .Values.webhookTemplateConfigMapKey | quote }} - {{- end }} - - name: WEBHOOK_TEMPLATE - value: {{ .Values.webhookTemplate | quote }} - - name: DRY_RUN - value: {{ .Values.dryRun | quote }} - - name: ENABLE_SPOT_INTERRUPTION_DRAINING - value: {{ .Values.enableSpotInterruptionDraining | quote }} - - name: ENABLE_SCHEDULED_EVENT_DRAINING - value: {{ .Values.enableScheduledEventDraining | quote }} - - name: ENABLE_REBALANCE_MONITORING - value: {{ .Values.enableRebalanceMonitoring | quote }} - - name: ENABLE_REBALANCE_DRAINING - value: {{ .Values.enableRebalanceDraining | quote }} - - name: CHECK_ASG_TAG_BEFORE_DRAINING - value: {{ .Values.checkASGTagBeforeDraining | quote }} - - name: MANAGED_ASG_TAG - value: {{ .Values.managedAsgTag | quote }} - - name: METADATA_TRIES - value: {{ .Values.metadataTries | quote }} - - name: CORDON_ONLY - value: {{ .Values.cordonOnly | quote }} - - name: TAINT_NODE - value: {{ .Values.taintNode | quote }} - - name: JSON_LOGGING - value: {{ .Values.jsonLogging | quote }} - - name: LOG_LEVEL - value: {{ .Values.logLevel | quote }} - - name: WEBHOOK_PROXY - value: {{ .Values.webhookProxy | quote }} - - name: UPTIME_FROM_FILE - value: {{ .Values.procUptimeFile | quote }} - - name: ENABLE_PROMETHEUS_SERVER - value: {{ .Values.enablePrometheusServer | quote }} - - name: PROMETHEUS_SERVER_PORT - value: {{ .Values.prometheusServerPort | quote }} - - name: ENABLE_PROBES_SERVER - value: {{ .Values.enableProbesServer | quote }} - - name: PROBES_SERVER_PORT - value: {{ .Values.probesServerPort | quote }} - - name: PROBES_SERVER_ENDPOINT - value: {{ .Values.probesServerEndpoint | quote }} - - name: EMIT_KUBERNETES_EVENTS - value: {{ .Values.emitKubernetesEvents | quote }} - - name: KUBERNETES_EVENTS_EXTRA_ANNOTATIONS - value: {{ .Values.kubernetesEventsExtraAnnotations | quote }} -{{- range $key, $value := .Values.extraEnv }} - - name: {{ $key }} - value: {{ $value | quote }} -{{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ENABLE_PROBES_SERVER + value: {{ .Values.enableProbesServer | quote }} + - name: PROBES_SERVER_PORT + value: {{ .Values.probes.httpGet.port | quote }} + - name: PROBES_SERVER_ENDPOINT + value: {{ .Values.probes.httpGet.path | quote }} + - name: LOG_LEVEL + value: {{ .Values.logLevel | quote }} + - name: JSON_LOGGING + value: {{ .Values.jsonLogging | quote }} + - name: ENABLE_PROMETHEUS_SERVER + value: {{ .Values.enablePrometheusServer | quote }} + - name: PROMETHEUS_SERVER_PORT + value: {{ .Values.prometheusServerPort | quote }} + {{- with .Values.instanceMetadataURL }} + - name: INSTANCE_METADATA_URL + value: {{ . | quote }} + {{- end }} + - name: METADATA_TRIES + value: {{ .Values.metadataTries | quote }} + - name: DRY_RUN + value: {{ .Values.dryRun | quote }} + - name: CORDON_ONLY + value: {{ .Values.cordonOnly | quote }} + - name: TAINT_NODE + value: {{ .Values.taintNode | quote }} + - name: DELETE_LOCAL_DATA + value: {{ .Values.deleteLocalData | quote }} + - name: IGNORE_DAEMON_SETS + value: {{ .Values.ignoreDaemonSets | quote }} + - name: POD_TERMINATION_GRACE_PERIOD + value: {{ .Values.podTerminationGracePeriod | quote }} + - name: NODE_TERMINATION_GRACE_PERIOD + value: {{ .Values.nodeTerminationGracePeriod | quote }} + - name: EMIT_KUBERNETES_EVENTS + value: {{ .Values.emitKubernetesEvents | quote }} + {{- with .Values.kubernetesEventsExtraAnnotations }} + - name: KUBERNETES_EVENTS_EXTRA_ANNOTATIONS + value: {{ . | quote }} + {{- end }} + {{- if or .Values.webhookURL .Values.webhookURLSecretName }} + - name: WEBHOOK_URL + {{- if .Values.webhookURLSecretName }} + valueFrom: + secretKeyRef: + name: {{ .Values.webhookURLSecretName }} + key: webhookurl + {{- else }} + value: {{ .Values.webhookURL | quote }} + {{- end }} + {{- end }} + {{- with .Values.webhookHeaders }} + - name: WEBHOOK_HEADERS + value: {{ . | quote }} + {{- end }} + {{- with .Values.webhookProxy }} + - name: WEBHOOK_PROXY + value: {{ . | quote }} + {{- end }} + {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + - name: WEBHOOK_TEMPLATE_FILE + value: {{ print "/config/" .Values.webhookTemplateConfigMapKey | quote }} + {{- else if .Values.webhookTemplate }} + - name: WEBHOOK_TEMPLATE + value: {{ .Values.webhookTemplate | quote }} + {{- end }} + - name: ENABLE_SPOT_INTERRUPTION_DRAINING + value: {{ .Values.enableSpotInterruptionDraining | quote }} + - name: ENABLE_SCHEDULED_EVENT_DRAINING + value: {{ .Values.enableScheduledEventDraining | quote }} + - name: ENABLE_REBALANCE_MONITORING + value: {{ .Values.enableRebalanceMonitoring | quote }} + - name: ENABLE_REBALANCE_DRAINING + value: {{ .Values.enableRebalanceDraining | quote }} + - name: ENABLE_SQS_TERMINATION_DRAINING + value: "false" {{- if or .Values.enablePrometheusServer .Values.enableProbesServer }} ports: + {{- if .Values.enableProbesServer }} + - name: liveness-probe + protocol: TCP + containerPort: {{ .Values.probes.httpGet.port }} + hostPort: {{ .Values.probes.httpGet.port }} {{- end }} {{- if .Values.enablePrometheusServer }} - - containerPort: {{ .Values.prometheusServerPort }} - hostPort: {{ .Values.prometheusServerPort }} - name: http-metrics - protocol: TCP + - name: http-metrics + protocol: TCP + containerPort: {{ .Values.prometheusServerPort }} + hostPort: {{ .Values.prometheusServerPort }} {{- end }} - {{- if .Values.enableProbesServer }} - - containerPort: {{ .Values.probesServerPort }} - hostPort: {{ .Values.probesServerPort }} - name: liveness-probe - protocol: TCP {{- end }} {{- if .Values.enableProbesServer }} livenessProbe: {{- toYaml .Values.probes | nindent 12 }} {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + volumeMounts: + - name: webhook-template + mountPath: /config/ + {{- end }} + {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + volumes: + - name: webhook-template + configMap: + name: {{ .Values.webhookTemplateConfigMapName }} + {{- end }} nodeSelector: - {{ include "aws-node-termination-handler.nodeSelectorTermsOs" . }}: windows - {{- with .Values.nodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.windowsNodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- if .Values.image.pullSecrets }} - imagePullSecrets: - {{- range .Values.image.pullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: + kubernetes.io/os: windows + {{- with default .Values.daemonsetNodeSelector (default .Values.nodeSelector .Values.windowsNodeSelector) }} {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} + {{- if or .Values.daemonsetAffinity (or .Values.affinity .Values.windowsAffinity) }} + affinity: + {{- toYaml (default .Values.daemonsetAffinity (default .Values.affinity .Values.windowsAffinity )) | nindent 8 }} + {{- end }} + {{- if or .Values.daemonsetTolerations (or .Values.tolerations .Values.windowsTolerations) }} + tolerations: + {{- toYaml (default .Values.daemonsetTolerations (default .Values.tolerations .Values.windowsTolerations )) | nindent 8 }} + {{- end }} {{- end -}} diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/deployment.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/deployment.yaml index 292a299..d29d92f 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/deployment.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/deployment.yaml @@ -3,35 +3,187 @@ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "aws-node-termination-handler.fullname" . }} - namespace: {{ .Release.Namespace }} labels: {{- include "aws-node-termination-handler.labels" . | nindent 4 }} spec: replicas: {{ .Values.replicas }} + {{- with .Values.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} selector: matchLabels: - {{- include "aws-node-termination-handler.selectorLabels" . | nindent 6 }} - {{ include "aws-node-termination-handler.nodeSelectorTermsOs" . }}: linux + {{- include "aws-node-termination-handler.selectorLabelsDeployment" . | nindent 6 }} template: metadata: - annotations: - {{- range $key, $value := .Values.podAnnotations }} - {{ $key }}: {{ $value | quote }} - {{- end }} labels: - {{- include "aws-node-termination-handler.selectorLabels" . | nindent 8 }} + {{- include "aws-node-termination-handler.selectorLabelsDeployment" . | nindent 8 }} k8s-app: aws-node-termination-handler - {{ include "aws-node-termination-handler.nodeSelectorTermsOs" . }}: linux - {{- range $key, $value := .Values.podLabels }} - {{ $key }}: {{ $value | quote }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} {{- end }} spec: + {{- with .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "aws-node-termination-handler.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + containers: + - name: aws-node-termination-handler + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: {{ include "aws-node-termination-handler.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ENABLE_PROBES_SERVER + value: "true" + - name: PROBES_SERVER_PORT + value: {{ .Values.probes.httpGet.port | quote }} + - name: PROBES_SERVER_ENDPOINT + value: {{ .Values.probes.httpGet.path | quote }} + - name: LOG_LEVEL + value: {{ .Values.logLevel | quote }} + - name: JSON_LOGGING + value: {{ .Values.jsonLogging | quote }} + - name: ENABLE_PROMETHEUS_SERVER + value: {{ .Values.enablePrometheusServer | quote }} + - name: PROMETHEUS_SERVER_PORT + value: {{ .Values.prometheusServerPort | quote }} + - name: CHECK_ASG_TAG_BEFORE_DRAINING + value: {{ .Values.checkASGTagBeforeDraining | quote }} + - name: MANAGED_ASG_TAG + value: {{ .Values.managedAsgTag | quote }} + - name: ASSUME_ASG_TAG_PROPAGATION + value: {{ .Values.assumeAsgTagPropagation | quote }} + - name: DRY_RUN + value: {{ .Values.dryRun | quote }} + - name: CORDON_ONLY + value: {{ .Values.cordonOnly | quote }} + - name: TAINT_NODE + value: {{ .Values.taintNode | quote }} + - name: DELETE_LOCAL_DATA + value: {{ .Values.deleteLocalData | quote }} + - name: IGNORE_DAEMON_SETS + value: {{ .Values.ignoreDaemonSets | quote }} + - name: POD_TERMINATION_GRACE_PERIOD + value: {{ .Values.podTerminationGracePeriod | quote }} + - name: NODE_TERMINATION_GRACE_PERIOD + value: {{ .Values.nodeTerminationGracePeriod | quote }} + - name: EMIT_KUBERNETES_EVENTS + value: {{ .Values.emitKubernetesEvents | quote }} + {{- with .Values.kubernetesEventsExtraAnnotations }} + - name: KUBERNETES_EVENTS_EXTRA_ANNOTATIONS + value: {{ . | quote }} + {{- end }} + {{- if or .Values.webhookURL .Values.webhookURLSecretName }} + - name: WEBHOOK_URL + {{- if .Values.webhookURLSecretName }} + valueFrom: + secretKeyRef: + name: {{ .Values.webhookURLSecretName }} + key: webhookurl + {{- else }} + value: {{ .Values.webhookURL | quote }} + {{- end }} + {{- end }} + {{- with .Values.webhookHeaders }} + - name: WEBHOOK_HEADERS + value: {{ . | quote }} + {{- end }} + {{- with .Values.webhookProxy }} + - name: WEBHOOK_PROXY + value: {{ . | quote }} + {{- end }} + {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + - name: WEBHOOK_TEMPLATE_FILE + value: {{ print "/config/" .Values.webhookTemplateConfigMapKey | quote }} + {{- else if .Values.webhookTemplate }} + - name: WEBHOOK_TEMPLATE + value: {{ .Values.webhookTemplate | quote }} + {{- end }} + - name: ENABLE_SPOT_INTERRUPTION_DRAINING + value: "false" + - name: ENABLE_SCHEDULED_EVENT_DRAINING + value: "false" + - name: ENABLE_REBALANCE_MONITORING + value: "false" + - name: ENABLE_REBALANCE_DRAINING + value: "false" + - name: ENABLE_SQS_TERMINATION_DRAINING + value: "true" + {{- with .Values.awsRegion }} + - name: AWS_REGION + value: {{ . | quote }} + {{- end }} + {{- with .Values.awsEndpoint }} + - name: AWS_ENDPOINT + value: {{ . | quote }} + {{- end }} + {{- if and .Values.awsAccessKeyID .Values.awsSecretAccessKey }} + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.awsAccessKeyID | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.awsSecretAccessKey | quote }} + {{- end }} + - name: QUEUE_URL + value: {{ .Values.queueURL | quote }} + - name: WORKERS + value: {{ .Values.workers | quote }} + {{- with .Values.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: liveness-probe + protocol: TCP + containerPort: {{ .Values.probes.httpGet.port }} + {{- if .Values.enablePrometheusServer }} + - name: http-metrics + protocol: TCP + containerPort: {{ .Values.prometheusServerPort }} + {{- end }} + livenessProbe: + {{- toYaml .Values.probes | nindent 12 }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: aws-token + mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + readOnly: true + {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + - name: webhook-template + mountPath: /config/ + {{- end }} volumes: - {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} - - name: "webhook-template" - configMap: - name: {{ .Values.webhookTemplateConfigMapName }} - {{- end }} - name: aws-token projected: sources: @@ -39,180 +191,22 @@ spec: path: token expirationSeconds: 86400 audience: "sts.amazonaws.com" - priorityClassName: {{ .Values.priorityClassName | quote }} - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: {{ include "aws-node-termination-handler.nodeSelectorTermsOs" . | quote }} - operator: In - values: - - linux - - key: {{ include "aws-node-termination-handler.nodeSelectorTermsArch" . | quote }} - operator: In - values: - - amd64 - - arm64 - - arm - {{- with .Values.affinity }} - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ template "aws-node-termination-handler.serviceAccountName" . }} - hostNetwork: false - dnsPolicy: {{ .Values.dnsPolicy | quote }} - securityContext: - fsGroup: {{ .Values.securityContext.runAsGroupID }} - containers: - - name: {{ include "aws-node-termination-handler.name" . }} - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - securityContext: - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: {{ .Values.securityContext.runAsUserID }} - runAsGroup: {{ .Values.securityContext.runAsGroupID }} - allowPrivilegeEscalation: false - volumeMounts: - {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} - - name: "webhook-template" - mountPath: "/config/" - {{- end }} - - name: aws-token - mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" - readOnly: true - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: DELETE_LOCAL_DATA - value: {{ .Values.deleteLocalData | quote }} - - name: IGNORE_DAEMON_SETS - value: {{ .Values.ignoreDaemonSets | quote }} - - name: POD_TERMINATION_GRACE_PERIOD - value: {{ .Values.podTerminationGracePeriod | quote }} - - name: INSTANCE_METADATA_URL - value: {{ .Values.instanceMetadataURL | quote }} - - name: NODE_TERMINATION_GRACE_PERIOD - value: {{ .Values.nodeTerminationGracePeriod | quote }} - - name: WEBHOOK_URL - {{- if .Values.webhookURLSecretName }} - valueFrom: - secretKeyRef: - name: {{ .Values.webhookURLSecretName }} - key: webhookurl - {{- else }} - value: {{ .Values.webhookURL | quote }} - {{- end }} - - name: WEBHOOK_HEADERS - value: {{ .Values.webhookHeaders | quote }} - {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} - - name: WEBHOOK_TEMPLATE_FILE - value: {{ print "/config/" .Values.webhookTemplateConfigMapKey | quote }} - {{- end }} - - name: WEBHOOK_TEMPLATE - value: {{ .Values.webhookTemplate | quote }} - - name: DRY_RUN - value: {{ .Values.dryRun | quote }} - - name: METADATA_TRIES - value: {{ .Values.metadataTries | quote }} - - name: CORDON_ONLY - value: {{ .Values.cordonOnly | quote }} - - name: TAINT_NODE - value: {{ .Values.taintNode | quote }} - - name: JSON_LOGGING - value: {{ .Values.jsonLogging | quote }} - - name: LOG_LEVEL - value: {{ .Values.logLevel | quote }} - - name: WEBHOOK_PROXY - value: {{ .Values.webhookProxy | quote }} - - name: ENABLE_PROMETHEUS_SERVER - value: {{ .Values.enablePrometheusServer | quote }} - - name: ENABLE_PROBES_SERVER - value: {{ .Values.enableProbesServer | quote }} - - name: ENABLE_SPOT_INTERRUPTION_DRAINING - value: "false" - - name: ENABLE_SCHEDULED_EVENT_DRAINING - value: "false" - - name: ENABLE_REBALANCE_MONITORING - value: "false" - - name: ENABLE_REBALANCE_DRAINING - value: "false" - - name: ENABLE_SQS_TERMINATION_DRAINING - value: "true" - - name: QUEUE_URL - value: {{ .Values.queueURL | quote }} - - name: PROMETHEUS_SERVER_PORT - value: {{ .Values.prometheusServerPort | quote }} - - name: PROBES_SERVER_PORT - value: {{ .Values.probesServerPort | quote }} - - name: PROBES_SERVER_ENDPOINT - value: {{ .Values.probesServerEndpoint | quote }} - - name: AWS_REGION - value: {{ .Values.awsRegion | quote }} - - name: AWS_ENDPOINT - value: {{ .Values.awsEndpoint | quote }} - {{- if .Values.awsSecretAccessKey }} - - name: AWS_SECRET_ACCESS_KEY - value: {{ .Values.awsSecretAccessKey | quote }} - - name: AWS_ACCESS_KEY_ID - value: {{ .Values.awsAccessKeyID | quote }} - {{- end }} - - name: CHECK_ASG_TAG_BEFORE_DRAINING - value: {{ .Values.checkASGTagBeforeDraining | quote }} - - name: MANAGED_ASG_TAG - value: {{ .Values.managedAsgTag | quote }} - - name: WORKERS - value: {{ .Values.workers | quote }} - - name: EMIT_KUBERNETES_EVENTS - value: {{ .Values.emitKubernetesEvents | quote }} - - name: KUBERNETES_EVENTS_EXTRA_ANNOTATIONS - value: {{ .Values.kubernetesEventsExtraAnnotations | quote }} -{{- range $key, $value := .Values.extraEnv }} - - name: {{ $key }} - value: {{ $value | quote }} -{{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- if or .Values.enablePrometheusServer .Values.enableProbesServer }} - ports: - {{- end }} - {{- if .Values.enablePrometheusServer }} - - containerPort: {{ .Values.prometheusServerPort }} - name: http-metrics - protocol: TCP - {{- end }} - {{- if .Values.enableProbesServer }} - - containerPort: {{ .Values.probesServerPort }} - name: liveness-probe - protocol: TCP - {{- end }} - {{- if .Values.enableProbesServer }} - livenessProbe: - {{- toYaml .Values.probes | nindent 12 }} - {{- end }} + {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + - name: webhook-template + configMap: + name: {{ .Values.webhookTemplateConfigMapName }} + {{- end }} nodeSelector: - {{ include "aws-node-termination-handler.nodeSelectorTermsOs" . }}: linux - {{- with .Values.nodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- if .Values.image.pullSecrets }} - imagePullSecrets: - {{- range .Values.image.pullSecrets }} - - name: {{ . }} + kubernetes.io/os: linux + {{- with .Values.nodeSelector }} + {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} {{- end }} diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/pdb.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/pdb.yaml index 1c88ef5..a2564fc 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/pdb.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/pdb.yaml @@ -1,5 +1,5 @@ {{- if and .Values.enableSqsTerminationDraining (and .Values.podDisruptionBudget (gt (int .Values.replicas) 1)) }} -apiVersion: policy/v1beta1 +apiVersion: {{ include "aws-node-termination-handler.pdb.apiVersion" . }} kind: PodDisruptionBudget metadata: name: {{ include "aws-node-termination-handler.fullname" . }} @@ -8,6 +8,6 @@ metadata: spec: selector: matchLabels: - {{- include "aws-node-termination-handler.selectorLabels" . | nindent 6 }} + {{- include "aws-node-termination-handler.selectorLabelsDeployment" . | nindent 6 }} {{- toYaml .Values.podDisruptionBudget | nindent 2 }} {{- end }} diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/podmonitor.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/podmonitor.yaml index 1c497d6..bbcbd9b 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/podmonitor.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/podmonitor.yaml @@ -5,8 +5,6 @@ metadata: name: {{ template "aws-node-termination-handler.fullname" . }} {{- if .Values.podMonitor.namespace }} namespace: {{ .Values.podMonitor.namespace }} - {{- else }} - namespace: {{ .Release.Namespace }} {{- end }} labels: {{- include "aws-node-termination-handler.labels" . | nindent 4 }} @@ -14,7 +12,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - jobLabel: {{ include "aws-node-termination-handler.name" . }} + jobLabel: app.kubernetes.io/name namespaceSelector: matchNames: - {{ .Release.Namespace }} @@ -29,5 +27,5 @@ spec: {{- end }} selector: matchLabels: - {{- include "aws-node-termination-handler.selectorLabels" . | nindent 6 }} + {{- include "aws-node-termination-handler.selectorLabelsDaemonset" . | nindent 6 }} {{- end -}} diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/psp.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/psp.yaml index ea953f8..e0034c1 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/psp.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/psp.yaml @@ -38,8 +38,8 @@ spec: volumes: - '*' --- -kind: Role apiVersion: rbac.authorization.k8s.io/v1 +kind: Role metadata: name: {{ template "aws-node-termination-handler.fullname" . }}-psp namespace: {{ .Release.Namespace }} diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/service.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/service.yaml index 5534b0b..869e260 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/service.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/service.yaml @@ -8,7 +8,7 @@ metadata: spec: type: ClusterIP selector: - {{- include "aws-node-termination-handler.selectorLabels" . | nindent 4 }} + {{- include "aws-node-termination-handler.selectorLabelsDeployment" . | nindent 4 }} ports: - name: http-metrics port: {{ .Values.prometheusServerPort }} diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/serviceaccount.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/serviceaccount.yaml index 40daa39..a83276d 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/serviceaccount.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/serviceaccount.yaml @@ -3,11 +3,10 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ template "aws-node-termination-handler.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} -{{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} -{{- end }} labels: {{- include "aws-node-termination-handler.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} {{- end -}} diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/servicemonitor.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/servicemonitor.yaml index 52ff799..caee505 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/templates/servicemonitor.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/templates/servicemonitor.yaml @@ -5,8 +5,6 @@ metadata: name: {{ include "aws-node-termination-handler.fullname" . }} {{- if .Values.serviceMonitor.namespace }} namespace: {{ .Values.serviceMonitor.namespace }} - {{- else }} - namespace: {{ .Release.Namespace }} {{- end }} labels: {{- include "aws-node-termination-handler.labels" . | nindent 4 }} @@ -14,7 +12,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - jobLabel: {{ include "aws-node-termination-handler.name" . }} + jobLabel: app.kubernetes.io/name namespaceSelector: matchNames: - {{ .Release.Namespace }} @@ -29,5 +27,5 @@ spec: {{- end }} selector: matchLabels: - {{- include "aws-node-termination-handler.selectorLabels" . | nindent 6 }} + {{- include "aws-node-termination-handler.selectorLabelsDeployment" . | nindent 6 }} {{- end -}} diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/test.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/test.yaml deleted file mode 100644 index a24efaa..0000000 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/test.yaml +++ /dev/null @@ -1,175 +0,0 @@ -# Test values for aws-node-termination-handler. -# This is a YAML-formatted file. -# Declare variables to test template rendering functionality. - -image: - repository: amazon/aws-node-termination-handler - tag: v1.6.1 - pullPolicy: IfNotPresent - pullSecrets: ["test"] - -securityContext: - runAsUserID: 1000 - runAsGroupID: 1000 - -nameOverride: "test-nth" -fullnameOverride: "test-aws-node-termination-handler" - -priorityClassName: system-node-critical - -podAnnotations: { - test: test -} -linuxPodAnnotations: { - test: test -} -windowsPodAnnotations: { - test: test -} - -podLabels: { - test: test -} -linuxPodLabels: { - test: test -} -windowsPodLabels: { - test: test -} - -resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "128Mi" - cpu: "100m" - -## enableSpotInterruptionDraining If false, do not drain nodes when the spot interruption termination notice is received -enableSpotInterruptionDraining: true - -## enableScheduledEventDraining [EXPERIMENTAL] If true, drain nodes before the maintenance window starts for an EC2 instance scheduled event -enableScheduledEventDraining: true - -# Total number of times to try making the metadata request before failing. -metadataTries: 3 - -# Cordon but do not drain nodes upon spot interruption termination notice. -cordonOnly: false - -# Taint node upon spot interruption termination notice. -taintNode: false - -# Log messages in JSON format. -jsonLogging: false - -## dryRun tells node-termination-handler to only log calls to kubernetes control plane -dryRun: false - -# deleteLocalData tells kubectl to continue even if there are pods using -# emptyDir (local data that will be deleted when the node is drained). -deleteLocalData: true - -# ignoreDaemonSets causes kubectl to skip Daemon Set managed pods. -ignoreDaemonSets: true - -# gracePeriod (DEPRECATED - use podTerminationGracePeriod instead) is time in seconds given to each pod to terminate gracefully. -# If negative, the default value specified in the pod will be used. -gracePeriod: 1 -podTerminationGracePeriod: 1 - -# nodeTerminationGracePeriod specifies the period of time in seconds given to each NODE to terminate gracefully. Node draining will be scheduled based on this value to optimize the amount of compute time, but still safely drain the node before an event. -nodeTerminationGracePeriod: 1 - -# webhookURL if specified, posts event data to URL upon instance interruption action. -webhookURL: https://localhost:1338 - -# Webhook URL will be fetched from the secret store using the given name. -webhookURLSecretName: test - -# webhookProxy if specified, uses this HTTP(S) proxy configuration. -webhookProxy: tcp://localhost:1338 - -# webhookHeaders if specified, replaces the default webhook headers. -webhookHeaders: "Content-Type: json" - -# webhookTemplate if specified, replaces the default webhook message template. -webhookTemplate: "{\"Content\":\"[NTH][Instance Interruption] InstanceId\"}" - -# instanceMetadataURL is used to override the default metadata URL (default: http://169.254.169.254:80) -instanceMetadataURL: "https://localhost:1338" - -# (TESTING USE): Mount path for uptime file -procUptimeFile: "/proc/uptime" - -# Create node OS specific daemonset(s). (e.g. "linux", "windows", "linux windows") -targetNodeOs: "linux" - -# nodeSelector tells both linux and windows daemonsets where to place the node-termination-handler -# pods. By default, this value is empty and every node will receive a pod. -nodeSelector: { - test: test -} -# linuxNodeSelector tells the linux daemonset where to place the node-termination-handler -# pods. By default, this value is empty and every linux node will receive a pod. -linuxNodeSelector: { - test: test -} -# windowsNodeSelector tells the windows daemonset where to place the node-termination-handler -# pods. By default, this value is empty and every windows node will receive a pod. -windowsNodeSelector: { - test: test -} - -enablePrometheusServer: true -prometheusServerPort: 9092 - -tolerations: -- operator: "Exists" - -affinity: { - test: test -} -linuxAffinity: { - test: test -} -windowsAffinity: { - test: test -} - -serviceAccount: - # Specifies whether a service account should be created - create: true - # The name of the service account to use. If name is not set and create is true, - # a name is generated using fullname template - name: test - annotations: { - test: test - } - # eks.amazonaws.com/role-arn: arn:aws:iam::AWS_ACCOUNT_ID:role/IAM_ROLE_NAME - -rbac: - # rbac.pspEnabled: `true` if PodSecurityPolicy resources should be created - pspEnabled: true - -dnsPolicy: "ClusterFirstWithHostNet" - -podMonitor: - # Specifies whether PodMonitor should be created - create: true - # The Prometheus scrape interval - interval: 30s - # The number of scraped samples that will be accepted - sampleLimit: 5000 - # Additional labels to add to the metadata - labels: { - test: test - } - -# K8s DaemonSet update strategy. -updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - linuxUpdateStrategy: "RollingUpdate" - windowsUpdateStrategy: "RollingUpdate" diff --git a/charts/kubezero-addons/charts/aws-node-termination-handler/values.yaml b/charts/kubezero-addons/charts/aws-node-termination-handler/values.yaml index 22355c2..a49c7d6 100644 --- a/charts/kubezero-addons/charts/aws-node-termination-handler/values.yaml +++ b/charts/kubezero-addons/charts/aws-node-termination-handler/values.yaml @@ -4,30 +4,58 @@ image: repository: public.ecr.aws/aws-ec2/aws-node-termination-handler - tag: v1.14.0 + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: "" pullPolicy: IfNotPresent pullSecrets: [] -securityContext: - runAsUserID: 1000 - runAsGroupID: 1000 - nameOverride: "" fullnameOverride: "" -extraEnv: {} +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. If namenot set and create is true, a name is generated using fullname template + name: + annotations: {} + # eks.amazonaws.com/role-arn: arn:aws:iam::AWS_ACCOUNT_ID:role/IAM_ROLE_NAME -priorityClassName: system-node-critical +rbac: + # Specifies whether RBAC resources should be created + create: true + # Specifies if PodSecurityPolicy resources should be created + pspEnabled: true -podAnnotations: {} -linuxPodAnnotations: {} -windowsPodAnnotations: {} +customLabels: {} podLabels: {} -linuxPodLabels: {} -windowsPodLabels: {} -# liveness probe settings. +podAnnotations: {} + +podSecurityContext: + fsGroup: 1000 + +securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + allowPrivilegeEscalation: false + runAsUser: 1000 + runAsGroup: 1000 + +terminationGracePeriodSeconds: + +resources: {} + +nodeSelector: {} + +affinity: {} + +tolerations: [] + +# Extra environment variables +extraEnv: [] + +# Liveness probe settings probes: httpGet: path: /healthz @@ -35,50 +63,17 @@ probes: initialDelaySeconds: 5 periodSeconds: 5 -resources: - requests: - memory: "64Mi" - cpu: "50m" - limits: - memory: "128Mi" - cpu: "100m" +# Set the log level +logLevel: info -# enableSqsTerminationDraining If true, this turns on queue-processor mode which drains nodes when an SQS termination event is received -enableSqsTerminationDraining: false +# Log messages in JSON format +jsonLogging: false -# enableRebalanceMonitoring If true, cordon nodes when the rebalance recommendation notice is received -enableRebalanceMonitoring: false +enablePrometheusServer: false +prometheusServerPort: 9092 -# enableRebalanceDraining If true, drain nodes when the rebalance recommendation notice is received -enableRebalanceDraining: false - -# queueURL Listens for messages on the specified SQS queue URL -queueURL: "" - -# checkASGTagBeforeDraining If true, check that the instance is tagged with "aws-node-termination-handler/managed" as the key before draining the node -checkASGTagBeforeDraining: true - -# managedAsgTag The tag to ensure is on a node if checkASGTagBeforeDraining is true -managedAsgTag: "aws-node-termination-handler/managed" - -# awsRegion If specified, use the AWS region for AWS API calls -awsRegion: "" - -# awsEndpoint If specified, use the AWS endpoint to make API calls. -awsEndpoint: "" - -# These should only be used for testing w/ localstack! -awsSecretAccessKey: -awsAccessKeyID: - -# enableSpotInterruptionDraining If false, do not drain nodes when the spot interruption termination notice is received -enableSpotInterruptionDraining: "" - -# enableScheduledEventDraining [EXPERIMENTAL] If true, drain nodes before the maintenance window starts for an EC2 instance scheduled event -enableScheduledEventDraining: "" - -# Total number of times to try making the metadata request before failing. -metadataTries: 3 +# dryRun tells node-termination-handler to only log calls to kubernetes control plane +dryRun: false # Cordon but do not drain nodes upon spot interruption termination notice. cordonOnly: false @@ -86,80 +81,18 @@ cordonOnly: false # Taint node upon spot interruption termination notice. taintNode: false -# Log messages in JSON format. -jsonLogging: false - -# Sets the log level -logLevel: "info" - -# dryRun tells node-termination-handler to only log calls to kubernetes control plane -dryRun: false - # deleteLocalData tells kubectl to continue even if there are pods using # emptyDir (local data that will be deleted when the node is drained). -deleteLocalData: "" +deleteLocalData: true # ignoreDaemonSets causes kubectl to skip Daemon Set managed pods. -ignoreDaemonSets: "" +ignoreDaemonSets: true -# gracePeriod (DEPRECATED - use podTerminationGracePeriod instead) is time in seconds given to each pod to terminate gracefully. -# If negative, the default value specified in the pod will be used. -gracePeriod: "" -podTerminationGracePeriod: "" +# podTerminationGracePeriod is time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used. +podTerminationGracePeriod: -1 # nodeTerminationGracePeriod specifies the period of time in seconds given to each NODE to terminate gracefully. Node draining will be scheduled based on this value to optimize the amount of compute time, but still safely drain the node before an event. -nodeTerminationGracePeriod: "" - -# webhookURL if specified, posts event data to URL upon instance interruption action. -webhookURL: "" - -# Webhook URL will be fetched from the secret store using the given name. -webhookURLSecretName: "" - -# webhookProxy if specified, uses this HTTP(S) proxy configuration. -webhookProxy: "" - -# webhookHeaders if specified, replaces the default webhook headers. -webhookHeaders: "" - -# webhook template file will be fetched from given config map name -# if specified, replaces the default webhook message with the content of the template file -webhookTemplateConfigMapName: "" - -# template file name stored in configmap -webhookTemplateConfigMapKey: "" - -# webhookTemplate if specified, replaces the default webhook message template. -webhookTemplate: "" - -# instanceMetadataURL is used to override the default metadata URL (default: http://169.254.169.254:80) -instanceMetadataURL: "" - -# (TESTING USE): Mount path for uptime file -procUptimeFile: "" - -# Create node OS specific daemonset(s). (e.g. "linux", "windows", "linux windows") -targetNodeOs: "linux" - -# nodeSelector tells both linux and windows daemonsets where to place the node-termination-handler -# pods. By default, this value is empty and every node will receive a pod. -nodeSelector: {} -# linuxNodeSelector tells the linux daemonset where to place the node-termination-handler -# pods. By default, this value is empty and every linux node will receive a pod. -linuxNodeSelector: {} -# windowsNodeSelector tells the windows daemonset where to place the node-termination-handler -# pods. By default, this value is empty and every windows node will receive a pod. -windowsNodeSelector: {} - -nodeSelectorTermsOs: "" -nodeSelectorTermsArch: "" - -enablePrometheusServer: false -prometheusServerPort: 9092 - -enableProbesServer: false -probesServerPort: 8080 -probesServerEndpoint: "/healthz" +nodeTerminationGracePeriod: 120 # emitKubernetesEvents If true, Kubernetes events will be emitted when interruption events are received and when actions are taken on Kubernetes nodes. In IMDS Processor mode a default set of annotations with all the node metadata gathered from IMDS will be attached to each event emitKubernetesEvents: false @@ -168,27 +101,101 @@ emitKubernetesEvents: false # Example: "first=annotation,sample.annotation/number=two" kubernetesEventsExtraAnnotations: "" -tolerations: - - operator: "Exists" +# webhookURL if specified, posts event data to URL upon instance interruption action. +webhookURL: "" -affinity: {} -linuxAffinity: {} -windowsAffinity: {} +# Webhook URL will be fetched from the secret store using the given name. +webhookURLSecretName: "" -serviceAccount: - # Specifies whether a service account should be created - create: true - # The name of the service account to use. If namenot set and create is true, - # a name is generated using fullname template - name: - annotations: {} - # eks.amazonaws.com/role-arn: arn:aws:iam::AWS_ACCOUNT_ID:role/IAM_ROLE_NAME +# webhookHeaders if specified, replaces the default webhook headers. +webhookHeaders: "" -rbac: - # rbac.pspEnabled: `true` if PodSecurityPolicy resources should be created - pspEnabled: true +# webhookProxy if specified, uses this HTTP(S) proxy configuration. +webhookProxy: "" -dnsPolicy: "" +# webhookTemplate if specified, replaces the default webhook message template. +webhookTemplate: "" + +# webhook template file will be fetched from given config map name +# if specified, replaces the default webhook message with the content of the template file +webhookTemplateConfigMapName: "" + +# template file name stored in configmap +webhookTemplateConfigMapKey: "" + +# enableSqsTerminationDraining If true, this turns on queue-processor mode which drains nodes when an SQS termination event is received +enableSqsTerminationDraining: false + +# --------------------------------------------------------------------------------------------------------------------- +# Queue Processor Mode +# --------------------------------------------------------------------------------------------------------------------- + +# The number of replicas in the NTH deployment when using queue-processor mode (NOTE: increasing this may cause duplicate webhooks since NTH pods are stateless) +replicas: 1 + +# Specify the update strategy for the deployment +strategy: {} + +# podDisruptionBudget specifies the disruption budget for the controller pods. +# Disruption budget will be configured only when the replicaCount is greater than 1 +podDisruptionBudget: {} +# maxUnavailable: 1 + +serviceMonitor: + # Specifies whether ServiceMonitor should be created + # this needs enableSqsTerminationDraining: true + # and enablePrometheusServer: true + create: false + # Specifies whether the ServiceMonitor should be created in a different namespace than + # the Helm release + namespace: + # Additional labels to add to the metadata + labels: {} + # The Prometheus scrape interval + interval: 30s + # The number of scraped samples that will be accepted + sampleLimit: 5000 + +priorityClassName: system-cluster-critical + +# If specified, use the AWS region for AWS API calls +awsRegion: "" + +# Listens for messages on the specified SQS queue URL +queueURL: "" + +# The maximum amount of parallel event processors to handle concurrent events +workers: 10 + +# If true, check that the instance is tagged with "aws-node-termination-handler/managed" as the key before draining the node +checkASGTagBeforeDraining: true + +# The tag to ensure is on a node if checkASGTagBeforeDraining is true +managedAsgTag: "aws-node-termination-handler/managed" + +# If true, assume that ASG tags will be appear on the ASG's instances +assumeAsgTagPropagation: false + +# --------------------------------------------------------------------------------------------------------------------- +# IMDS Mode +# --------------------------------------------------------------------------------------------------------------------- + +# Create node OS specific daemonset(s). (e.g. "linux", "windows", "linux windows") +targetNodeOs: linux + +linuxPodLabels: {} +windowsPodLabels: {} + +linuxPodAnnotations: {} +windowsPodAnnotations: {} + +# K8s DaemonSet update strategy. +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + +daemonsetPriorityClassName: system-node-critical podMonitor: # Specifies whether PodMonitor should be created @@ -205,41 +212,68 @@ podMonitor: # The number of scraped samples that will be accepted sampleLimit: 5000 -serviceMonitor: - # Specifies whether ServiceMonitor should be created - # this needs enableSqsTerminationDraining: rue - # and enablePrometheusServer: true - create: false - # Specifies whether the ServiceMonitor should be created in a different namespace than - # the Helm release - namespace: - # Additional labels to add to the metadata - labels: {} - # The Prometheus scrape interval - interval: 30s - # The number of scraped samples that will be accepted - sampleLimit: 5000 - -# K8s DaemonSet update strategy. -updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 -linuxUpdateStrategy: "" -windowsUpdateStrategy: "" - # Determines if NTH uses host networking for Linux when running the DaemonSet (only IMDS mode; queue-processor never runs with host networking) # If you have disabled IMDSv1 and are relying on IMDSv2, you'll need to increase the IP hop count to 2 before switching this to false # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html useHostNetwork: true -# The maximal amount of parallel event processors to handle concurrent events -workers: 10 +# Daemonset DNS policy +dnsPolicy: "" +linuxDnsPolicy: ClusterFirstWithHostNet +windowsDnsPolicy: ClusterFirst -# The number of replicas in the NTH deployment when using queue-processor mode (NOTE: increasing this may cause duplicate webhooks since NTH pods are stateless) -replicas: 1 +daemonsetNodeSelector: {} +linuxNodeSelector: {} +windowsNodeSelector: {} -# podDisruptionBudget specifies the disruption budget for the controller pods. -# Disruption budget will be configured only when the replicaCount is greater than 1 -podDisruptionBudget: {} -# maxUnavailable: 1 +daemonsetAffinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "eks.amazonaws.com/compute-type" + operator: NotIn + values: + - fargate +linuxAffinity: {} +windowsAffinity: {} + +daemonsetTolerations: + - operator: Exists +linuxTolerations: [] +windowsTolerations: [] + +# If the probes server is running for the Daemonset +enableProbesServer: false + +# Total number of times to try making the metadata request before failing. +metadataTries: 3 + +# enableSpotInterruptionDraining If false, do not drain nodes when the spot interruption termination notice is received +enableSpotInterruptionDraining: true + +# enableScheduledEventDraining [EXPERIMENTAL] If true, drain nodes before the maintenance window starts for an EC2 instance scheduled event +enableScheduledEventDraining: false + +# enableRebalanceMonitoring If true, cordon nodes when the rebalance recommendation notice is received +enableRebalanceMonitoring: false + +# enableRebalanceDraining If true, drain nodes when the rebalance recommendation notice is received +enableRebalanceDraining: false + +# --------------------------------------------------------------------------------------------------------------------- +# Testing +# --------------------------------------------------------------------------------------------------------------------- + +# (TESTING USE): If specified, use the provided AWS endpoint to make API calls. +awsEndpoint: "" + +# (TESTING USE): These should only be used for testing w/ localstack! +awsAccessKeyID: +awsSecretAccessKey: + +# (TESTING USE): Override the default metadata URL (default: http://169.254.169.254:80) +instanceMetadataURL: "" + +# (TESTING USE): Mount path for uptime file +procUptimeFile: /proc/uptime diff --git a/charts/kubezero-addons/nth.patch b/charts/kubezero-addons/nth.patch index d18ff15..65a4809 100644 --- a/charts/kubezero-addons/nth.patch +++ b/charts/kubezero-addons/nth.patch @@ -1,10 +1,34 @@ diff -tuNr charts/aws-node-termination-handler.orig/templates/deployment.yaml charts/aws-node-termination-handler/templates/deployment.yaml ---- charts/aws-node-termination-handler.orig/templates/deployment.yaml 2021-12-01 16:41:46.713472250 +0100 -+++ charts/aws-node-termination-handler/templates/deployment.yaml 2021-12-01 16:41:54.276883046 +0100 -@@ -32,6 +32,13 @@ - configMap: - name: {{ .Values.webhookTemplateConfigMapName }} - {{- end }} +--- charts/aws-node-termination-handler.orig/templates/deployment.yaml 2022-01-26 18:01:36.123482217 +0100 ++++ charts/aws-node-termination-handler/templates/deployment.yaml 2022-01-26 18:08:21.464304621 +0100 +@@ -161,9 +161,9 @@ + {{- toYaml . | nindent 12 }} + {{- end }} + ports: +- - name: liveness-probe +- protocol: TCP +- containerPort: {{ .Values.probes.httpGet.port }} ++ - name: liveness-probe ++ protocol: TCP ++ containerPort: {{ .Values.probes.httpGet.port }} + {{- if .Values.enablePrometheusServer }} + - name: http-metrics + protocol: TCP +@@ -175,13 +175,23 @@ + resources: + {{- toYaml . | nindent 12 }} + {{- end }} +- {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + volumeMounts: ++ - name: aws-token ++ mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" ++ readOnly: true ++ {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + - name: webhook-template + mountPath: /config/ + {{- end }} +- {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + volumes: + - name: aws-token + projected: + sources: @@ -12,28 +36,7 @@ diff -tuNr charts/aws-node-termination-handler.orig/templates/deployment.yaml ch + path: token + expirationSeconds: 86400 + audience: "sts.amazonaws.com" - priorityClassName: {{ .Values.priorityClassName | quote }} - affinity: - nodeAffinity: -@@ -71,6 +78,9 @@ - - name: "webhook-template" - mountPath: "/config/" - {{- end }} -+ - name: aws-token -+ mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" -+ readOnly: true - env: - - name: NODE_NAME - valueFrom: -diff -tuNr charts/aws-node-termination-handler.orig/values.yaml charts/aws-node-termination-handler/values.yaml ---- charts/aws-node-termination-handler.orig/values.yaml 2021-12-01 16:41:46.713472250 +0100 -+++ charts/aws-node-termination-handler/values.yaml 2021-12-01 16:42:02.350299065 +0100 -@@ -15,7 +15,7 @@ - nameOverride: "" - fullnameOverride: "" - --extraEnv: [] -+extraEnv: {} - - priorityClassName: system-node-critical - ++ {{- if and .Values.webhookTemplateConfigMapName .Values.webhookTemplateConfigMapKey }} + - name: webhook-template + configMap: + name: {{ .Values.webhookTemplateConfigMapName }} diff --git a/charts/kubezero-addons/templates/awsController/deployment.yaml b/charts/kubezero-addons/templates/awsController/deployment.yaml deleted file mode 100644 index 5047fc3..0000000 --- a/charts/kubezero-addons/templates/awsController/deployment.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{- if .Values.awsController.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kubezero-aws-controller - namespace: kube-system - labels: - app: kubezero-aws-controller -spec: - replicas: 1 - selector: - matchLabels: - app: kubezero-aws-controller - template: - metadata: - labels: - app: kubezero-aws-controller - spec: - containers: - - name: kubezero-aws-controller - image: "{{ .Values.awsController.image.name }}:{{ .Values.awsController.image.tag }}" - imagePullPolicy: Always - serviceAccountName: kubezero-aws-controller - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/control-plane: "" - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule -{{- end }} diff --git a/charts/kubezero-addons/templates/awsController/rbac.yaml b/charts/kubezero-addons/templates/awsController/rbac.yaml deleted file mode 100644 index b2ac261..0000000 --- a/charts/kubezero-addons/templates/awsController/rbac.yaml +++ /dev/null @@ -1,31 +0,0 @@ -{{- if .Values.awsController.enabled }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kubezero-aws-controller - namespace: kube-system - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: kubezero-aws-controller -rules: -- apiGroups: [""] - resources: ["pods", "nodes"] - verbs: ["*"] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubezero-aws-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kubezero-aws-controller -subjects: - - kind: ServiceAccount - name: kubezero-aws-controller - namespace: kube-system -{{- end }} diff --git a/charts/kubezero-addons/templates/forseti/deployment.yaml b/charts/kubezero-addons/templates/forseti/deployment.yaml new file mode 100644 index 0000000..2c5f26d --- /dev/null +++ b/charts/kubezero-addons/templates/forseti/deployment.yaml @@ -0,0 +1,83 @@ +{{- if .Values.forseti.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubezero-forseti + namespace: kube-system + labels: + app: kubezero-forseti +spec: + replicas: 1 + selector: + matchLabels: + app: kubezero-forseti + template: + metadata: + labels: + app: kubezero-forseti + spec: + containers: + - name: kubezero-forseti + image: "{{ .Values.forseti.image.name }}:{{ .Values.forseti.image.tag }}" + imagePullPolicy: Always + args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=:8080 + - --zap-log-level=2 + #- --dry-run + # - --leader-elect + command: + - /forseti + env: + - name: AWS_REGION + value: "{{ .Values.forseti.aws.region }}" + - name: AWS_ROLE_ARN + value: "{{ .Values.forseti.aws.iamRoleArn }}" + - name: AWS_STS_REGIONAL_ENDPOINTS + value: regional + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/sts.amazonaws.com/serviceaccount/token + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /var/run/secrets/sts.amazonaws.com/serviceaccount/ + name: aws-token + readOnly: true + securityContext: + runAsNonRoot: true + serviceAccountName: kubezero-forseti + terminationGracePeriodSeconds: 10 + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + volumes: + - name: aws-token + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: sts.amazonaws.com + expirationSeconds: 86400 + path: token +{{- end }} diff --git a/charts/kubezero-addons/templates/forseti/rbac.yaml b/charts/kubezero-addons/templates/forseti/rbac.yaml new file mode 100644 index 0000000..7eb3f24 --- /dev/null +++ b/charts/kubezero-addons/templates/forseti/rbac.yaml @@ -0,0 +1,104 @@ +{{- if .Values.forseti.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubezero-forseti + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubezero-forseti-manager +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - nodes/finalizers + verbs: + - update +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: forseti-leader-election + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubezero-forseti-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubezero-forseti-manager +subjects: + - kind: ServiceAccount + name: kubezero-forseti + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: forseti-leader-election + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: forseti-leader-election +subjects: +- kind: ServiceAccount + name: kubezero-forseti + namespace: kube-system +{{- end }} diff --git a/charts/kubezero-addons/templates/forseti/service.yaml b/charts/kubezero-addons/templates/forseti/service.yaml new file mode 100644 index 0000000..511204a --- /dev/null +++ b/charts/kubezero-addons/templates/forseti/service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.forseti.enabled }} +apiVersion: v1 +kind: Service +metadata: + labels: + app: kubezero-forseti + name: forseti-metrics-service + namespace: kube-system +spec: + ports: + - name: http + port: 8080 + protocol: TCP + selector: + app: kubezero-forseti +{{- end }} diff --git a/charts/kubezero-addons/update.sh b/charts/kubezero-addons/update.sh index 9b1de65..0f394ca 100755 --- a/charts/kubezero-addons/update.sh +++ b/charts/kubezero-addons/update.sh @@ -3,8 +3,9 @@ set -ex NTH_VERSION=$(yq eval '.dependencies[] | select(.name=="aws-node-termination-handler") | .version' Chart.yaml) -rm -rf charts/aws-node-termination-handler -helm pull eks/aws-node-termination-handler --untar --untardir charts --version $NTH_VERSION +# Disabled until these AWS "pros" bump the chart number +#rm -rf charts/aws-node-termination-handler +#helm pull eks/aws-node-termination-handler --untar --untardir charts --version $NTH_VERSION # diff -tuNr charts/aws-node-termination-handler.orig charts/aws-node-termination-handler > nth.patch patch -p0 -i nth.patch --no-backup-if-mismatch diff --git a/charts/kubezero-addons/values.yaml b/charts/kubezero-addons/values.yaml index a65731d..0390f48 100644 --- a/charts/kubezero-addons/values.yaml +++ b/charts/kubezero-addons/values.yaml @@ -3,18 +3,23 @@ clusterBackup: image: name: public.ecr.aws/zero-downtime/kubezero-admin - tag: v1.21.8 + tag: v1.21.9 repository: "" password: "" extraEnv: [] -tenjin: +forseti: enabled: false image: - name: public.ecr.aws/zero-downtime/kubezero-tenjin - tag: v0.1.0 + name: public.ecr.aws/zero-downtime/forseti + tag: v0.1.2 + + aws: + region: "" + # -- "arn:aws:iam::${AWS::AccountId}:role/${AWS::Region}.${ClusterName}.kubezeroForseti" + iamRoleArn: "" aws-node-termination-handler: enabled: false @@ -36,10 +41,13 @@ aws-node-termination-handler: metadataTries: 0 extraEnv: - # -- "arn:aws:iam::${AWS::AccountId}:role/${AWS::Region}.${ClusterName}.awsNth" - AWS_ROLE_ARN: "" - AWS_WEB_IDENTITY_TOKEN_FILE: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" - AWS_STS_REGIONAL_ENDPOINTS: "regional" + # -- "arn:aws:iam::${AWS::AccountId}:role/${AWS::Region}.${ClusterName}.awsNth" + - name: AWS_ROLE_ARN + value: "" + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + - name: AWS_STS_REGIONAL_ENDPOINTS + value: "regional" enablePrometheusServer: false podMonitor: