# We built on top of tiny-cloud . /lib/tiny-cloud/common . /usr/lib/cloudbender/cloud/"$CLOUD".sh # boolean flags is_enabled() { local flag=$(echo "$1" | tr '[:upper:]' '[:lower:]') [ "$flag" == 1 -o "$flag" == "true" ] && return 0 [ "$flag" == 0 -o "$flag" == "false" -o "$flag" == "none" -o -z "$flag" ] && return 1 log -t user-data warn "Unknown value for boolean option: $flag - assuming False" return 1 } # setup_instance, various OS tweaks impossible to do via AMI baking setup_instance() { # create machine-id to emulate systemd [ -f /etc/machine-id ] || uuidgen > /etc/machine-id # add and mount bpf file system add_once /etc/fstab "bpffs /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0" mount -a add_once /etc/hosts "${IP_ADDRESS} ${_META_HOSTNAME} ${HOSTNAME}" # workaround for dhcpcd / openresolv to omit search domain if equal to domain breaking DNS resolution of shortnames for eg. etcd and kube-apiserver add_once /etc/resolv.conf "search $DOMAIN_NAME" case "$CLOUD" in aws) # Set system wide default region for boto3 echo "export AWS_DEFAULT_REGION=$REGION" > /etc/profile.d/aws.sh setup_sns_alarms ;; *) ewarn "Unsupported Cloud: $CLOUD" # return 1 ;; esac } ################ # IAM SSH KEYS # ################ configure_sshd() { # Change Listen port local port=${SSHPORT:-"22"} [ -w /etc/ssh/sshd_config ] && sed -i -e 's/^[\s#]*Port\s.*/Port '$port'/' /etc/ssh/sshd_config case "$CLOUD" in aws) # on AWS call IAM for allowed groups and actual keys local group=${SSHKEYIAMGROUP:-""} local role=${SSHKEYIAMROLE:-"arn:aws:iam::000000000000:role/Undefined"} [ $role == "arn:aws:iam::000000000000:role/Undefined" ] && role="" if [ -n "$group" ]; then # Configure SSHD sed -i -e 's,^[\s#]*AuthorizedKeysCommand\s.*,AuthorizedKeysCommand /usr/sbin/get_iam_sshkeys.py --user %u --group '$group' --iamRole "'$role'",' /etc/ssh/sshd_config sed -i -e 's,^[\s#]*AuthorizedKeysCommandUser\s.*,AuthorizedKeysCommandUser nobody,' /etc/ssh/sshd_config einfo "added $group to SSH admin keys" fi ;; nocloud) return 0 ;; *) ewarn "Unsupported Cloud: $CLOUD" # return 1 ;; esac } # Persist host keys # has to run before sshd starts up first time ! persistent_sshd_hostkeys() { # Top level is artifact to be able to limit the SSM IAM permissions local ssm_path=$1 local key_types="ecdsa ed25519 rsa" # try to get none existing host keys from SSM RET=0 for key in $key_types; do if [ ! -f /etc/ssh/ssh_host_${key}_key.pub -a ! -f /etc/ssh/ssh_host_${key}_key ]; then (aws ssm get-parameters --names "${ssm_path}/host_${key}.tgz" --with-decryption --query 'Parameters[0].Value' | base64 -d | tar xzf - --directory=/ 1>/dev/null 2>&1) \ && log -t user-data info "Restored ssh_host_${key}_key from SSM" || RET=1 fi done # Update keys if any key couldn't be restored from SSM if [ $RET -eq 1 ]; then # generate any missing keys ssh-keygen -A for key in $key_types; do if [ -r /etc/ssh/ssh_host_${key}_key -a -r /etc/ssh/ssh_host_${key}_key.pub ]; then (aws ssm put-parameter --name "${ssm_path}/host_${key}.tgz" --type SecureString --value \ "$(tar czf - /etc/ssh/ssh_host_${key}_key /etc/ssh/ssh_host_${key}_key.pub | base64)" --overwrite) \ && log -t user-data info "Uploaded ssh_host_${key}_key to SSM" fi done fi } # either plain custom hostname or # - `unique:` eg. `uniq:kube-worker-{:02}` -> kube-worker-01 # - `myownip: ` eg. `myip: nodegroup-` -> nodegroup-1.2.3.4 set_hostname() { local custom_hostname=$(echo $1 | awk -F. '{ print $1 }') if [ -n "$custom_hostname" ]; then if [[ "$custom_hostname" == unique:* ]]; then new_hostname=$(uniq_hostname.py $AWS_EC2LAUNCHTEMPLATE_ID $INSTANCE_ID ${custom_hostname##unique:}) elif [[ "$custom_hostname" == myownip:* ]]; then local _ip=$(echo $IP_ADDRESS | sed -e 's/\./-/g') new_hostname=$(echo "${custom_hostname##myownip:}$_ip") else new_hostname=$custom_hostname fi FQDN="${new_hostname}.${DOMAIN_NAME}" echo ${new_hostname} > /etc/hostname hostname $new_hostname export HOSTNAME=$new_hostname # add new hostname to hosts add_once /etc/hosts "${IP_ADDRESS} ${FQDN} ${new_hostname}" log -t user-data info "Hostname updated to ${new_hostname}." # hup syslog to update loghost macro /etc/init.d/syslog-ng reload # update Route53 entry for VPC internal FQDN route53.py --fqdn $FQDN --record $IP_ADDRESS # update our Name Tag to FQDN or PrivateDNSName to allow easy indentification in the AWS UI aws ec2 create-tags --resources $INSTANCE_ID --tags Key=Name,Value=$FQDN else aws ec2 create-tags --resources $INSTANCE_ID --tags Key=Name,Value=${HOSTNAME}.${REGION}.compute.internal fi } _parse_volume() { # Todo: proper checks once all is yaml # For now just replace ':' echo $1 | sed -e 's/:/ /g' } # mount optional remote volumes mount_volumes() { local volumes="$1" for vol in $volumes; do # Todo: check volume type and call matching func read volType volId volDevice volPath < <(_parse_volume $vol) [ "$volType" != "ebs" ] && { echo "Unknown volume type $volType"; break; } attach_ebs $volId $volDevice rc=$? [ $rc -ne 0 ] && { ewarn "error trying to attach $volId"; break; } # wait for the block device to become available while true; do mdev -s test -b $volDevice && break sleep 1 done # check volume for existing filesystem type=$(file -Lbs $volDevice) if [[ "$type" =~ "XFS filesystem" ]]; then xfs_repair $volDevice >/dev/null 2>&1 else mkfs.xfs -qf $volDevice >/dev/null fi # mount mkdir -p $volPath mount -t xfs -o noatime $volDevice $volPath einfo "mounting $volDevice at $volPath" done } unmount_volumes() { local volumes="$1" for vol in $volumes; do read volType volId volDevice volPath < <(_parse_volume $vol) umount $volPath && aws ec2 detach-volume --volume-id "$volId" --instance-id $INSTANCE_ID --region $REGION > /dev/null done } # msg used for sns event, last one wins msg() { MSG="$@"; log -t user-data info "$@"; } # Generic retry command wrapper, incl. timeout of 30s # $1 = number of tries; 0 = forever # $2 = number of seconds to sleep between tries # $@ actual command retry() { local tries=$1 local waitfor=$2 shift 2 while true; do # Only use timeout of $1 is an executable, call directly if function type -tf $1 >/dev/null && { timeout 30 $@ && return; } || { $@ && return; } ((tries=tries-1)) || true [ $tries -eq 0 ] && return 1 sleep $waitfor done } add_swap() { [ -f /.swapfile ] || { dd if=/dev/zero of=/.swapfile bs=1M count=$1 && chmod 600 /.swapfile && mkswap /.swapfile && swapon /.swapfile; } grep -q "/.swapfile" /etc/fstab || echo "/.swapfile none swap sw 0 0" >> /etc/fstab sysctl -w vm.swappiness=10 } # Get SSM secure string base64 decoded # $0 SSM_PATH, value to stdout get_secret() { aws ssm get-parameter --name ${1,,} --with-decryption --query 'Parameter.Value' | base64 -d } # Store values as base64 on SSM # $0 SSM_PATH VALUE put_secret() { aws ssm put-parameter --name ${1,,} --type SecureString --value "$(echo "$2" | base64 -w0)" --overwrite } # Gets existing passphrase or creates new passphrase and stores it init_passphrase() { local _URL=$1 local _PPFILE=$2 # If secret already exists noop [ -f $_PPFILE ] && return 0 get_secret $_URL > $_PPFILE && chmod 600 $_PPFILE || \ { xxd -l16 -p /dev/random > $_PPFILE; chmod 600 $_PPFILE; put_secret $_URL "$(cat $_PPFILE)"; } } # upload various useful logs to s3 if configured upload_debug_logs(){ [ -z $ZDT_CLOUDBENDER_DEBUG_REMOTELOGS ] && return 0 local s3Url="$ZDT_CLOUDBENDER_DEBUG_REMOTELOGS/$INSTANCE_ID/$(date +'%Y%m%d-%H%M%Z')" local _tmp=$(mktemp -d) ps -ef > ${_tmp}/process.list cp /var/log/messages \ /var/log/rc.log \ /var/log/user-data.log \ /etc/cloudbender.conf \ /var/lib/cloud/meta-data \ /var/log/kubelet/kubelet.log \ /var/log/crio/crio.log \ $_tmp tar cfz /tmp/debuglogs.tgz -C $_tmp . aws s3 cp /tmp/debuglogs.tgz $s3Url/debuglogs.tgz return 0 } exit_trap() { set +e trap - ERR EXIT local ERR_CODE=$1 local ERR_LINE="$2" local ERR_FUNC="$3" local ERR_CMD="$4" if [ $ERR_CODE -ne 0 ]; then CFN_STATUS="FAILURE" RESULT="ABANDON" else CFN_STATUS="SUCCESS" RESULT="CONTINUE" fi # Add SNS events on demand if [ -n "$ALARMSNSARN" ]; then if [ $ERR_CODE -ne 0 ]; then LEVEL="Error" SUBJECT="Error during cloud-init." if [ $ERR_LINE -ne 1 ]; then MSG="$ERR_CMD failed in $ERR_FUNC at $ERR_LINE. Return: $ERR_CODE" ATTACHMENT="$(pr -tn $0 | tail -n+$((ERR_LINE - 3)) | head -n7)" else MSG="$ERR_CMD" fi if [ -n "$ZDT_CLOUDBENDER_DEBUG" ]; then SUBJECT="$SUBJECT Instance kept running for debug." else SUBJECT="$SUBJECT Instance terminated by ASG lifecycle hook." fi else LEVEL="Info" SUBJECT="ZDT Alpine Instance launched." fi if [ -z "${DISABLECLOUDBENDERSNSSCALINGEVENTS}" ] || [ "$LEVEL" != "Info" ]; then /var/lib/cloud/sns_alarm.sh "$SUBJECT" "$MSG" "$LEVEL" "$ATTACHMENT" fi # Disable scaling events during shutdown [ -n "${DISABLECLOUDBENDERSNSSCALINGEVENTS}" ] && echo "DISABLE_SCALING_EVENTS=1" >> /etc/cloudbender.conf fi [ -n "$LAUNCH_HOOK" ] && aws autoscaling complete-lifecycle-action --lifecycle-action-result $RESULT --instance-id $INSTANCE_ID --lifecycle-hook-name $LAUNCH_HOOK --auto-scaling-group-name ${AWS_AUTOSCALING_GROUPNAME} || true if [ -n "${AWS_CLOUDFORMATION_LOGICAL_ID}" ]; then aws cloudformation signal-resource --stack-name ${AWS_CLOUDFORMATION_STACK_NAME} --logical-resource-id ${AWS_CLOUDFORMATION_LOGICAL_ID} --unique-id ${INSTANCE_ID} --status ${CFN_STATUS} fi # timestamp being done end_uptime=$(awk '{print $1}' < /proc/uptime) log -t user-data info "Exiting user-data. $end_uptime seconds after boot. Duration: $(echo "$end_uptime-$start_uptime" | bc)" # if we ran into error, either upload debug files or poweroff if [ $ERR_CODE -ne 0 ]; then is_enabled $ZDT_CLOUDBENDER_DEBUG && upload_debug_logs || poweroff fi exit 0 } ### S3 based locking MUTEX=mutex MUTEX_OWNER=$HOSTNAME MUTEX_TIMEOUT=600 release_lock() { local S3LOCK=$1 rm -f $MUTEX aws s3 rm $S3LOCK } # Lock not timed out and we own it: 0 # Lock not timed out and someone else owns it: 1 # Lock timed out: 2 verify_lock() { local S3LOCK=$1 aws s3 cp $S3LOCK $MUTEX _host=$(grep "MUTEX_OWNER=" $MUTEX | sed -e 's/MUTEX_OWNER=//') _time=$(grep "MUTEX_TIME=" $MUTEX | sed -e 's/MUTEX_TIME=//') # Check for timestamp and timeout let timepassed=$(date -u +%s)-$_time [ $timepassed -gt $MUTEX_TIMEOUT ] && return 2 [ "$_host" == "$MUTEX_OWNER" ] && return 0 return 1 } aquire_lock() { local S3LOCK=$1 echo "MUTEX_OWNER=${MUTEX_OWNER}" > $MUTEX echo "MUTEX_TIME=$(date -u +%s)" >> $MUTEX aws s3 cp $MUTEX $S3LOCK # verify we actually got the lock sleep 2 verify_lock $S3LOCK } check_lock() { local S3LOCK=$1 aws s3 ls $S3LOCK && rc=$? || rc=$? # No LOCK ? -> get it ! if [ $rc -ne 0 ]; then aquire_lock $S3LOCK else verify_lock $S3LOCK && rc=$? || rc=$? # Lock timeout -> we get it if [ $rc -eq 2 ]; then aquire_lock $S3LOCK # we already own it elif [ $rc -eq 0 ]; then return 0 # someone else has a valid lock else return 1 fi fi } # All things networking enable_ip_forwarding() { modprobe nf_conntrack cat < /etc/sysctl.d/40-ip-forward.conf net.ipv4.ip_forward=1 net.ipv4.ip_local_port_range = 1024 65535 net.ipv4.conf.all.send_redirects=0 net.ipv4.conf.all.rp_filter = 1 net.ipv4.conf.all.accept_redirects = 0 net.ipv6.conf.default.forwarding = 1 net.ipv6.conf.all.forwarding = 1 net.netfilter.nf_conntrack_max = 524288 EOF sysctl -p /etc/sysctl.d/40-ip-forward.conf } enable_iptables() { rc-update add iptables /etc/init.d/iptables save } register_service_dns() { if [ -n "$DNSZONE" -a -n "$SERVICENAME" ]; then _IP=${PUBLIC_IP_ADDRESS:-$IP_ADDRESS} [ -n "SERVICEPRIVATE" ] && _IP=$IP_ADDRESS route53.py --fqdn "${SERVICENAME}.${DNSZONE}" --record $_IP # Register shutdown hook to remove DNS entry on terminate add_once /etc/local.d/cloudbender.stop "echo \"Deleting Route53 record for ${SERVICENAME}.${DNSZONE}\" >> /tmp/shutdown.log" add_once /etc/local.d/cloudbender.stop "route53.py --delete --fqdn \"${SERVICENAME}.${DNSZONE}\" --record ${PUBLIC_IP_ADDRESS:-$IP_ADDRESS}" # Short cut our public IP to private one to allow talking to our own service name add_once /etc/hosts "${IP_ADDRESS} ${SERVICENAME}.${DNSZONE}" log -t user-data info "Registered $_IP with ${SERVICENAME}.$DNSZONE" fi } setup_prometheus() { rc-update add node-exporter default rc-service node-exporter start log -t user-data info "Enabled and started Prometheus node-exporter" } setup_fluentbit() { local key="cloudbender" local host="${1:-fluentd}" if [[ "$host" =~ "@" ]]; then key=${host%%@*} host=${host##*@} fi # add some AWS metadata cat < /etc/fluent-bit/metadata.conf [FILTER] Name record_modifier Match * Record source.ip $IP_ADDRESS Record source.instance_id $INSTANCE_ID Record source.region $REGION Record source.account $AWS_ACCOUNT_ID Record source.conglomerate $CONGLOMERATE Record source.artifact $ARTIFACT EOF # Configure output cat < /etc/fluent-bit/output.conf [OUTPUT] Match * Name forward Host $host Port 24224 Shared_Key $key tls on Send_options true Require_ack_response true EOF LOG_FILES=$LOGGING_FILES ## TODO: # Add parameter parsing for custom logfile tailing rc-update add fluent-bit default rc-service fluent-bit start log -t user-data info "Enabled and started fluent-bit logging agent sending logs to $host" }