feat: first working KubeZero images, alpha agebox support

This commit is contained in:
Stefan Reimer 2022-04-14 15:35:10 +02:00
parent 72689bfeb1
commit 97d66abd84
25 changed files with 444 additions and 16 deletions

3
.ageboxreg.yml Normal file
View File

@ -0,0 +1,3 @@
file_ids:
- overlay/zdt/configs/access.conf
version: "1"

1
.agekeys Normal file
View File

@ -0,0 +1 @@
age1z42dmf0cluvuyp2jz9gzkf2ly9afxqmp9cy6dy22fwak32uhjszscn25k4

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
overlay/zdt/configs/access.conf

12
Makefile Normal file
View File

@ -0,0 +1,12 @@
OVERLAY := $(shell pwd)/overlay
# FILTER := --only 3.15 kubezero --skip aarch64
FILTER := --only 3.15 --skip aarch64
STEP := publish
all: build
build:
cd alpine-cloud-images && ./build $(STEP) --clean --revise $(FILTER) --custom $(OVERLAY)/zdt --vars $(OVERLAY)/zdt/zdt.hcl
clean:
rm -rf alpine-cloud-images/work

View File

@ -152,14 +152,23 @@ class AWSCloudAdapter(CloudAdapterInterface):
# import snapshot from S3
log.info('Importing EC2 snapshot from %s', s3_url)
ss_import = ec2c.import_snapshot(
DiskContainer={
_import_opts = {
'DiskContainer': {
'Description': description, # https://github.com/boto/boto3/issues/2286
'Format': 'VHD',
'Url': s3_url
}
# NOTE: TagSpecifications -- doesn't work with ResourceType: snapshot?
)
}
# NOTE: TagSpecifications -- doesn't work with ResourceType: snapshot?
# For some reason the import_snapshot boto function cannot handle setting KmsKeyId to default / empty
# so we need to set it conditionally
if ic.encryption_key_id:
_import_opts['Encrypted'] = True
_import_opts['KmsKeyId'] = ic.encryption_key_id
ss_import = ec2c.import_snapshot(**_import_opts)
ss_task_id = ss_import['ImportTaskId']
while True:
ss_task = ec2c.describe_import_snapshot_tasks(
@ -315,6 +324,8 @@ class AWSCloudAdapter(CloudAdapterInterface):
Name=source.name,
SourceImageId=source_id,
SourceRegion=source_region,
Encrypted=True if ic.encryption_key_id else False,
KmsKeyId=ic.encryption_key_id
)
except Exception:
log.warning('Skipping %s, unable to copy image:', r, exc_info=True)
@ -343,6 +354,7 @@ class AWSCloudAdapter(CloudAdapterInterface):
if fresh:
tags.published = datetime.utcnow().isoformat()
tags.Name = tags.name # because AWS is special
image.create_tags(Tags=tags.as_list())
# tag image's snapshot, too
@ -358,14 +370,15 @@ class AWSCloudAdapter(CloudAdapterInterface):
)
# apply launch perms
log.info('%s: Applying launch perms to %s', r, image.id)
image.reset_attribute(Attribute='launchPermission')
image.modify_attribute(
Attribute='launchPermission',
OperationType='add',
UserGroups=perms['groups'],
UserIds=perms['users'],
)
if perms['groups'] or perms['users']:
log.info('%s: Applying launch perms to %s', r, image.id)
image.reset_attribute(Attribute='launchPermission')
image.modify_attribute(
Attribute='launchPermission',
OperationType='add',
UserGroups=perms['groups'],
UserIds=perms['users'],
)
# set up AMI deprecation
ec2c = image.meta.client

View File

@ -1,5 +1,5 @@
# vim: ts=2 et:
name = [cloudinit]
# name = [cloudinit]
# start cloudinit images with 3.15
EXCLUDE = ["3.12", "3.13", "3.14"]
@ -11,4 +11,4 @@ packages {
}
services.default.cloud-init-hotplugd = true
scripts = [ setup-cloudinit ]
scripts = [ setup-cloudinit ]

View File

@ -1,6 +1,6 @@
# vim: ts=2 et:
name = [bios]
#name = [bios]
bootloader = extlinux
packages.syslinux = --no-scripts
qemu.firmware = null
qemu.firmware = null

13
audit_grants.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
#set -x
for r in $(aws ec2 describe-regions --query "Regions[].{Name:RegionName}" --output text); do
keyAlias="arn:aws:kms:${r}:533404190593:alias/zdt/amis"
keyArn=$(aws kms describe-key --region $r --key-id $keyAlias --output json 2>/dev/null | jq -r '.KeyMetadata.Arn')
if [ -n "$keyArn" ]; then
aws kms list-grants --region $r --key-id $keyArn --output json | jq '.Grants[]'
fi
done

22
cleanup_amis.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
#set -x
TAG_FILTER="Name=tag:project,Values=zdt-alpine"
#for r in $(aws ec2 describe-regions --query "Regions[].{Name:RegionName}" --output text); do
for r in eu-central-1 us-west-2; do
amis=$(aws ec2 describe-images --region $r --owners self --output json --filters $TAG_FILTER | jq -r '.Images[].ImageId')
for a in $amis; do
aws ec2 deregister-image --region $r --image-id $a && echo "Deleted AMI $a in $r"
done
amis=$(aws ec2 describe-images --region $r --owners self --output json --filters Name=state,Values=failed | jq -r '.Images[].ImageId')
for a in $amis; do
aws ec2 deregister-image --region $r --image-id $a && echo "Deleted AMI $a in $r"
done
snapshots=$(aws ec2 describe-snapshots --region $r --owner-ids self --output json --filters $TAG_FILTER | jq -r '.Snapshots[].SnapshotId')
for s in $snapshots; do
aws ec2 delete-snapshot --snapshot-id $s --region $r && echo "Deleted snapshot $s in $r"
done
done

View File

@ -0,0 +1,5 @@
age-encryption.org/v1
-> X25519 ZT6m1CYk0KfJbxayb1X65OgPL6U4lnVgr90fSOiHNTA
aAo+pQyd8gS9Y2cYufu9rAsSCDr+hmjfRa2h5HtkEZw
--- JlxAy916xCRYxSIeTbFzmU9U6+TYOFSVwDMx30m8i/w
<EFBFBD>„ѳÕáËËuPŒ#®¯@h9Ëšû·åCŠÏ<C5A0>Ò mm>–áîè'Ç ™k¡°d6ºŒ¢™ö¬q™ŸÆ<C5B8>žSÁÅ¥

View File

@ -0,0 +1,16 @@
bash = true
jq = true
yq = true
logrotate = true
iptables = true
syslog-ng-json = true
podman = true
wireguard-tools = true
lvm2 = true
socat = true
ethtool = true
nvme-cli = true
xfsprogs = true
dhclient = true
monit = true
#prometheus-node-exporter = true

View File

@ -0,0 +1,14 @@
sysinit {
cgroups = true
}
boot {
syslog = null
syslog-ng = true
}
default {
local = true
monit = true
crond = true
}

View File

@ -0,0 +1 @@
zdt.conf

View File

@ -0,0 +1,24 @@
# vim: ts=2 et:
description = [ "- https://kubezero.com" ]
name = [ kubezero-1.22.8 ]
size = 2G
scripts = [ setup-common ]
packages { include required("common-packages.conf") }
services { include required("common-services.conf") }
WHEN {
kubezero {
scripts = [ setup-kubernetes ]
}
}
WHEN {
aws {
packages {
aws-cli = true
py3-boto3 = true
}
}
}

View File

@ -0,0 +1,17 @@
# vim: ts=2 et:
description = [ "- https://zero-downtime.net/cloud" ]
name = [ minimal ]
scripts = [ setup-common ]
packages { include required("common-packages.conf") }
services { include required("common-services.conf") }
WHEN {
aws {
packages {
aws-cli = true
py3-boto3 = true
}
}
}

View File

@ -0,0 +1,88 @@
# vim: ts=2 et:
project = zdt-alpine
kubeversion = 1.21
# all build configs start with these
Default {
project = ${project}
# image name/description components
encryption_key_id = null
name = [ zdt-alpine ]
description = [ "ZeroDownTime Alpine Images" ]
motd {
welcome = "Welcome to Alpine!"
wiki = "The Alpine Wiki contains a large amount of how-to guides and general\n"\
"information about administrating Alpine systems.\n"\
"See <https://wiki.alpinelinux.org/>."
version_notes = "Release Notes:\n"\
"* <https://alpinelinux.org/posts/alpine-{version}.0/released.html>"
release_notes = "* <https://alpinelinux.org/posts/{release}/released.html"
}
# initial provisioning script and data directory
scripts = [ setup ]
script_dirs = [ setup.d ]
size = 1G
login = alpine
local_format = qcow2
# image access
access.PUBLIC = false
# image publication
regions.ALL = null
}
# profile build matrix
Dimensions {
version {
"3.15" { include required("version/3.15.conf") }
# edge { include required("version/edge.conf") }
}
arch {
x86_64 { include required("arch/x86_64.conf") }
# aarch64 { include required("arch/aarch64.conf") }
}
firmware {
bios { include required("firmware/bios.conf") }
uefi { include required("firmware/uefi.conf") }
}
bootstrap {
cloudinit { include required("bootstrap/cloudinit.conf") }
}
cloud {
aws { include required("cloud/aws.conf") }
aws.regions {
ALL = false
eu-central-1 = true
us-west-2 = true
}
}
artifact {
minimal { include required("minimal.conf") }
kubezero { include required("kubezero.conf") }
}
}
# all build configs merge these at the very end
Mandatory {
name = [ "r{revision}" ]
encryption_key_id = "alias/zdt/amis"
# final motd message
motd.motd_change = "You may change this message by editing /etc/motd."
# final provisioning script
scripts = [ cleanup ]
access { include required("access.conf") }
}

View File

@ -0,0 +1,40 @@
#!/bin/sh -eu
# vim: ts=4 et:
[ -z "$DEBUG" ] || [ "$DEBUG" = 0 ] || set -x
SETUP=/tmp/setup.d
TARGET=/mnt
# Enable testing repo - do we really want versions to change randomly ?
# echo "@testing http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> "$TARGET/etc/apk/repositories"
# Fix dhcp to set MTU properly
install -o root -g root -Dm644 -t $TARGET/etc/dhcp $SETUP/dhclient.conf
echo 'Setup dhclient'
# Enable SSH keepalive
sed -i -e "s/^[\s#]*TCPKeepAlive\s.*/TCPKeepAlive yes/" -e "s/^[\s#]*ClientAliveInterval\s.*/ClientAliveInterval 60/" $TARGET/etc/ssh/sshd_config
echo 'Enabled SSH keep alives'
# CgroupsV2
sed -i -e "s/^[\s#]*rc_cgroup_mode=.*/rc_cgroup_mode=\"unified\"/" $TARGET/etc/rc.conf
# Setup syslog-ng json logging
cp $SETUP/syslog-ng.conf $TARGET/etc/syslog-ng/syslog-ng.conf
cp $SETUP/syslog-ng.logrotate.conf $TARGET/etc/logrotate.d/syslog-ng
# Install cloudbender shutdown hook
cp $SETUP/cloudbender.stop $TARGET/etc/local.d
mkdir -p $TARGET/etc/cloudbender/shutdown.d
# Install tools
cp $SETUP/route53.py $TARGET/usr/local/bin
# Install ps_mem
wget -q -O $TARGET/usr/local/bin/ps_mem.py https://raw.githubusercontent.com/pixelb/ps_mem/master/ps_mem.py
sed -i -e 's,#!/usr/bin/env python,#!/usr/bin/env python3,' $TARGET/usr/local/bin/ps_mem.py
chmod +x $TARGET/usr/local/bin/ps_mem.py
echo 'Installed ps_mem.py'
printf '\n# Zero Down Time config applied'

View File

@ -0,0 +1,30 @@
#!/bin/sh -eu
# vim: ts=4 et:
[ -z "$DEBUG" ] || [ "$DEBUG" = 0 ] || set -x
SETUP=/tmp/setup.d
TARGET=/mnt
KUBE_VERSION=1.22
AWS_IAM_VERSION=0.5.6
# Enable ZDT repo
echo "@kubezero https://cdn.zero-downtime.net/alpine/v${VERSION}/kubezero" >> "$TARGET/etc/apk/repositories"
install -o root -g root -Dm600 -t $TARGET/etc/apk/keys $SETUP/stefan@zero-downtime.net-61bb6bfb.rsa.pub
apk -U --root "$TARGET" --no-cache add \
cri-tools@kubezero \
cri-o@kubezero=~$KUBE_VERSION \
kubelet@kubezero=~$KUBE_VERSION \
kubectl@kubezero=~$KUBE_VERSION
# aws-iam-authenticator
wget -qO $TARGET/usr/local/bin/aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v${AWS_IAM_VERSION}/aws-iam-authenticator_${AWS_IAM_VERSION}_linux_amd64
chmod +x $TARGET/usr/local/bin/aws-iam-authenticator
echo "Installed aws-iam-authenticator binary version $AWS_IAM_VERSION"
# Pre-load container images
# echo 'Pre-loaded Kubernetes control container images'
printf '\n\n# Zero Down Time config applied'

View File

@ -0,0 +1,15 @@
# Include dynamic config setting create at boot
[ -r /etc/cloudbender/rc.conf ] && . /etc/cloudbender/rc.conf
rm -f /tmp/shutdown.log
for cmd in $(ls /etc/cloudbender/shutdown.d/* | sort); do
. $cmd 1>>/tmp/shutdown.log 2>&1
done
[ $DEBUG -eq 1 ] && SHUTDOWNLOG="$(cat /tmp/shutdown.log)"
[ -n "$RC_REBOOT" ] && ACTION="rebooting" || ACTION="terminated"
[ -z "$DISABLE_SCALING_EVENTS" ] && cloudbender_sns_alarm.sh "Instance $ACTION" "" Info "$SHUTDOWNLOG"
sleep ${SHUTDOWN_PAUSE:-0}

View File

@ -0,0 +1,12 @@
# Borrowed from Ubuntu 20.04LTS minimal EC2 AMi
option rfc3442-classless-static-routes code 121 = array of unsigned integer 8;
send host-name = gethostname();
request subnet-mask, broadcast-address, time-offset, routers,
domain-name, domain-name-servers, domain-search, host-name,
dhcp6.name-servers, dhcp6.domain-search, dhcp6.fqdn, dhcp6.sntp-servers,
netbios-name-servers, netbios-scope, interface-mtu,
rfc3442-classless-static-routes, ntp-servers;
timeout 300;

View File

@ -0,0 +1,53 @@
#!/usr/bin/env python3
import sys
import boto3
import json
import argparse
def update_dns(record_name, ips=[], ttl=180, action="UPSERT", record_type='A'):
route53 = boto3.client("route53")
zone_id = route53.list_hosted_zones_by_name(
DNSName=".".join(record_name.split(".")[1:])
)["HostedZones"][0]["Id"]
changeset = {
"Changes": [
{
"Action": action,
"ResourceRecordSet": {
"Name": record_name,
"Type": record_type,
"TTL": ttl,
"ResourceRecords": [],
},
}
]
}
for ip in ips:
changeset["Changes"][0]["ResourceRecordSet"]["ResourceRecords"].append(
{"Value": ip}
)
route53.change_resource_record_sets(HostedZoneId=zone_id, ChangeBatch=changeset)
parser = argparse.ArgumentParser(description='Update Route53 entries')
parser.add_argument('--fqdn', dest='fqdn', action='store', required=True,
help='FQDN for this record')
parser.add_argument('--record', action='append', required=True,
help='Value of a record')
parser.add_argument('--type', dest='record_type', action='store', default='A',
help='Record type')
parser.add_argument('--ttl', dest='ttl', action='store', default=180, type=int,
help='TTL of the entry')
parser.add_argument('--delete', dest='delete', action='store_true',
help='delete entry')
args = parser.parse_args()
action = "UPSERT"
if args.delete:
action = "DELETE"
print(args)
update_dns(args.fqdn, args.record, action=action, ttl=args.ttl, record_type=args.record_type)

View File

@ -0,0 +1,9 @@
-----BEGIN PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6BP/2VTRKfWmGtcJKf10
tHrjiOir0BUqxTlYRwOtRv2iSs2aNaxs89sH+ZCNGxao1n+zBijhI2UFbp/nxGO5
ftCPZicirASBmFN0XMg94nCt/vz+KCYjU+ASqlM/4uRFk0zf+loknzLgyyGD3SUT
tR9NCsOsZWN4sRTGDAAkseCqPOTsG/7c7bDWaEr1Gq2LQdV12KU1OqkSoR+aH9lk
xBdKMIgXssHiTQZevgMo515Z5kqaMBsOojpNUNjq7sPHmpKFlJJ93Id0QfH9duPk
0oWzT5XJdh6lrilYDAU4Bs4QNVGr1i27dQXRL57m5Gp1u705rwNjUmzwpZtCStwd
YwIDAQAB
-----END PUBLIC KEY-----

View File

@ -0,0 +1,16 @@
# syslog-ng, format all json into messages
# https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.23/administration-guide/63#TOPIC-1268643
@version: 3.30
@include "scl.conf"
options { chain_hostnames(off); flush_lines(0); use_dns(no); use_fqdn(no);
dns_cache(no); owner("root"); group("adm"); perm(0640);
stats_freq(0); bad_hostname("^gconfd$"); frac-digits(6);
};
source s_sys { system(); internal();};
destination d_mesg { file("/var/log/messages" template("$(format-json time=\"$UNIXTIME\" facility=\"$FACILITY\" host=\"$LOGHOST\" ident=\"$PROGRAM\" pid=\"$PID\" level=\"$PRIORITY\" message=\"$MESSAGE\")\n")); };
log { source(s_sys); destination(d_mesg); };

View File

@ -0,0 +1,13 @@
/var/log/messages
{
rotate 2
missingok
notifempty
compress
maxsize 64M
daily
sharedscripts
postrotate
invoke-rc.d syslog-ng reload > /dev/null
endscript
}

10
overlay/zdt/zdt.hcl Normal file
View File

@ -0,0 +1,10 @@
qemu = {
"boot_wait": {
"aarch64": "15s",
"x86_64": "15s"
}
cmd_wait = "5s"
ssh_timeout = "20s"
memory = 1024
}