From 63a522149d353cc99199400ff73da871e5d26eb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jake=20Buchholz=20G=C3=B6kt=C3=BCrk?= Date: Sun, 30 Jan 2022 19:18:09 +0000 Subject: [PATCH] Tiny Cloud / set default NTP server * switch to tiny-cloud instead of tiny-ec2-bootstrap * set default NTP server, if configured * add default /etc/network/interfaces * add urlopen() timeout to mitigate ipv6 issues connecting to alpinelinux.org --- LICENSE.txt | 2 +- alpine.pkr.hcl | 1 + alpine.py | 9 +- configs/bootstrap/tiny.conf | 18 ++- configs/cloud/aws.conf | 2 + scripts/setup | 11 +- scripts/setup-tiny | 60 +------ scripts/setup-tiny.d/assemble-interfaces | 42 ----- scripts/setup-tiny.d/eth-eni-hook | 187 ---------------------- scripts/setup-tiny.d/eth-eni-hotplug | 96 ----------- scripts/setup-tiny.d/eth-eni-setup | 19 --- scripts/setup-tiny.d/interfaces.d/DEFAULT | 3 - scripts/setup-tiny.d/interfaces.d/eth0 | 3 - scripts/setup-tiny.d/interfaces.d/lo | 3 - scripts/setup-tiny.d/mdev-ec2.conf | 6 - scripts/setup-tiny.d/nvme-ebs-links | 47 ------ scripts/setup.d/interfaces | 7 + 17 files changed, 41 insertions(+), 475 deletions(-) delete mode 100755 scripts/setup-tiny.d/assemble-interfaces delete mode 100755 scripts/setup-tiny.d/eth-eni-hook delete mode 100755 scripts/setup-tiny.d/eth-eni-hotplug delete mode 100755 scripts/setup-tiny.d/eth-eni-setup delete mode 100644 scripts/setup-tiny.d/interfaces.d/DEFAULT delete mode 100644 scripts/setup-tiny.d/interfaces.d/eth0 delete mode 100644 scripts/setup-tiny.d/interfaces.d/lo delete mode 100644 scripts/setup-tiny.d/mdev-ec2.conf delete mode 100755 scripts/setup-tiny.d/nvme-ebs-links create mode 100644 scripts/setup.d/interfaces diff --git a/LICENSE.txt b/LICENSE.txt index ab6f63d..817eab1 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2017-2021 Jake Buchholz Göktürk, Michael Crute +Copyright (c) 2017-2022 Jake Buchholz Göktürk, Michael Crute Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/alpine.pkr.hcl b/alpine.pkr.hcl index 4956903..5cd4c99 100644 --- a/alpine.pkr.hcl +++ b/alpine.pkr.hcl @@ -163,6 +163,7 @@ build { "KERNEL_MODULES=${B.value.kernel_modules}", "KERNEL_OPTIONS=${B.value.kernel_options}", "MOTD=${B.value.motd}", + "NTP_SERVER=${B.value.ntp_server}", "PACKAGES_ADD=${B.value.packages.add}", "PACKAGES_DEL=${B.value.packages.del}", "PACKAGES_NOSCRIPTS=${B.value.packages.noscripts}", diff --git a/alpine.py b/alpine.py index 09f393d..8fa4a6a 100644 --- a/alpine.py +++ b/alpine.py @@ -10,18 +10,20 @@ class Alpine(): DEFAULT_RELEASES_URL = 'https://alpinelinux.org/releases.json' DEFAULT_CDN_URL = 'https://dl-cdn.alpinelinux.org/alpine' + DEFAULT_WEB_TIMEOUT = 5 - def __init__(self, releases_url=None, cdn_url=None): + def __init__(self, releases_url=None, cdn_url=None, web_timeout=None): self.now = datetime.utcnow() self.release_today = self.now.strftime('%Y%m%d') self.eol_tomorrow = (self.now + timedelta(days=1)).strftime('%F') self.latest = None self.versions = {} self.releases_url = releases_url or self.DEFAULT_RELEASES_URL + self.web_timeout = web_timeout or self.DEFAULT_WEB_TIMEOUT self.cdn_url = cdn_url or self.DEFAULT_CDN_URL # get all Alpine versions, and their EOL and latest release - res = urlopen(self.releases_url) + res = urlopen(self.releases_url, timeout=self.web_timeout) r = json.load(res) branches = sorted( r['release_branches'], reverse=True, @@ -89,8 +91,7 @@ class Alpine(): ver = self._ver(ver) repo_url = self.repo_url(repo, arch, ver=ver) apks_re = re.compile(f'"{apk}-(\\d.*)\\.apk"') - print(repo_url) - res = urlopen(repo_url) + res = urlopen(repo_url, timeout=self.web_timeout) for line in map(lambda x: x.decode('utf8'), res): if not line.startswith('> "$TARGET/etc/fstab" diff --git a/scripts/setup-tiny b/scripts/setup-tiny index 2fa00ab..7081842 100755 --- a/scripts/setup-tiny +++ b/scripts/setup-tiny @@ -1,67 +1,15 @@ #!/bin/sh -eu # vim: ts=4 et: -# NOTE: This is lifted almost verbatim from alpine-ec2-ami's setup-ami script -# While refactoring that, it became apparent that these bits really belonged -# to the tiny-ec2-bootstrap package, and in order to also provide a -# "cloud-init" image flavor, we needed separate these parts out. -# -# It is our intention to eventually incorporate this in a tiny-cloud-bootstrap -# package (and per-cloud subpackages). - [ -z "$DEBUG" ] || [ "$DEBUG" = 0 ] || set -x TARGET=/mnt -SETUP=/tmp/setup-tiny.d -die() { - printf '\033[1;7;31m FATAL: %s \033[0m\n' "$@" >&2 # bold reversed red - exit 1 -} einfo() { printf '\n\033[1;7;36m> %s <\033[0m\n' "$@" >&2 # bold reversed cyan } -setup_mdev() { - echo " * Setting up mdev" - install -o root -g root -Dm755 -t "$TARGET/lib/mdev" \ - "$SETUP/nvme-ebs-links" \ - "$SETUP/eth-eni-hotplug" - - # insert nvme ebs mdev configs just above "# fallback" comment - sed -n -i \ - -e "/# fallback/r $SETUP/mdev-ec2.conf" \ - -e 1x -e '2,${x;p}' -e '${x;p}' \ - "$TARGET/etc/mdev.conf" -} - -setup_networking() { - echo " * Setting up networking" - # configure standard interfaces - IFACE_CFG="$TARGET/etc/network/interfaces" - install -o root -g root -Dm755 -d "$SETUP/interfaces.d" "$IFACE_CFG.d" - install -o root -g root -Dm644 -t "$IFACE_CFG.d" \ - "$SETUP/interfaces.d/"* - cat "$IFACE_CFG.d/lo" "$IFACE_CFG.d/eth0" > "$IFACE_CFG" - - install -o root -g root -Dm755 -t "$TARGET/etc/network" \ - "$SETUP/assemble-interfaces" - - # install ucdhcp hooks for EC2 ENI IPv6 and secondary IPv4 - install -o root -g root -Dm755 -t "$TARGET/usr/share/udhcpc" \ - "$SETUP/eth-eni-hook" - for i in post-bound post-renew; do - mkdir -p "$TARGET/etc/udhcpc/$i" - ln -sf /usr/share/udhcpc/eth-eni-hook \ - "$TARGET/etc/udhcpc/$i" - done - - # install ENI interface setup init script - install -o root -g root -Dm755 -t "$TARGET/etc/init.d" \ - "$SETUP/eth-eni-setup" -} - -einfo "Installing up tiny bootstrap components..." -setup_mdev -setup_networking -echo "EC2_USER=$IMAGE_LOGIN" > "$TARGET/etc/conf.d/tiny-ec2-bootstrap" +einfo "Configuring Tiny Cloud..." +sed -i.bak -Ee "s/^#?CLOUD_USER=.*/CLOUD_USER=$IMAGE_LOGIN/" \ + "$TARGET"/etc/conf.d/tiny-cloud +rm "$TARGET"/etc/conf.d/tiny-cloud.bak \ No newline at end of file diff --git a/scripts/setup-tiny.d/assemble-interfaces b/scripts/setup-tiny.d/assemble-interfaces deleted file mode 100755 index e723f43..0000000 --- a/scripts/setup-tiny.d/assemble-interfaces +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh -# vim: set ts=4 et: - -set -e - -IFACE_CFG=/etc/network/interfaces -IFACE_DIR="${IFACE_CFG}.d" - -cd "$IFACE_DIR" - -cat > "$IFACE_CFG.new" <> "$IFACE_CFG.new" - -# existing eths -for i in /sys/class/net/eth*; do - IFACE="$(basename "$i")" - [ ! -f "$IFACE" ] && sed -e "s/%%/$IFACE/g" DEFAULT > "$IFACE" - cat "$IFACE" >> "$IFACE_CFG.new" -done - -# all the rest -for i in "$IFACE_DIR"/*; do - IFACE="$(basename "$i")" - case $IFACE in - DEFAULT|lo|eth*) - continue - ;; - *) - cat "$IFACE" >> "$IFACE_CFG.new" - ;; - esac -done - -# install new interfaces config -cp -a "$IFACE_CFG" "$IFACE_CFG.bak" -mv "$IFACE_CFG.new" "$IFACE_CFG" diff --git a/scripts/setup-tiny.d/eth-eni-hook b/scripts/setup-tiny.d/eth-eni-hook deleted file mode 100755 index 317f844..0000000 --- a/scripts/setup-tiny.d/eth-eni-hook +++ /dev/null @@ -1,187 +0,0 @@ -#!/bin/sh -# vim: set ts=4 et: - -# This script should be installed as symlinks in -# /etc/udhcpc//eth-eni-hook -# :- -# post-bound - after udhcpc binds an IP to the interface -# post-renew - after udhcpc renews the lease for the IP -# -# udhcpc provides via ENV... -# IFACE - eth0, etc. -# mask - bits in IPv4 subnet mask - -set -e - -HOOK="$(basename "$(dirname "$0")")" - -DEBUG= - -log() { - [ -z "$DEBUG" ] && [ "$1" = "debug" ] && return - FACILITY="daemon.$1" - shift - logger -s -p "$FACILITY" -t "udhcpc/${HOOK}[$$]" -- "$@" -} - -if [ -z "$IFACE" ] || [ -z "$mask" ]; then - log err "Missing 'IFACE' or 'mask' ENV from udhcpc" - exit 1 -fi - -# route table number -RTABLE="${IFACE#eth}" -let RTABLE+=1000 - -IMDS=X-aws-ec2-metadata-token -IMDS_IP=169.254.169.254 -IMDS_MAC="http://$IMDS_IP/latest/meta-data/network/interfaces/macs/$( - cat "/sys/class/net/$IFACE/address")" - -get_imds_token() { - IMDS_TOKEN="$(echo -ne \ - "PUT /latest/api/token HTTP/1.0\r\n$IMDS-ttl-seconds: 60\r\n\r\n" | - nc "$IMDS_IP" 80 | tail -n 1)" -} - -mac_meta() { - wget -qO - --header "$IMDS: $IMDS_TOKEN" "$IMDS_MAC/$1" 2>/dev/null \ - || true # when no ipv6s attached (yet), IMDS returns 404 error -} - -ip() { - FAIL_OK= - if [ "$1" = '+F' ]; then - FAIL_OK=1 - shift - fi - v=-4 - if [ "$1" = '-4' ] || [ "$1" = '-6' ]; then - v="$1" - shift - fi - OP="$2" - [ "$OP" = show ] && LEV=debug || LEV=info - if /sbin/ip "$v" "$@" || [ -n "$FAIL_OK" ]; then - log "$LEV" "OK: ip $v $*" - else - log err "FAIL: ip $v $*" - fi -} - -iface_ip4s() { - ip -4 addr show "$IFACE" secondary | - sed -E -e '/inet /!d' -e 's/.*inet ([0-9.]+).*/\1/' -} - -iface_ip6s() { - ip -6 addr show "$IFACE" scope global | - sed -E -e '/inet6/!d' -e 's/.*inet6 ([0-9a-f:]+).*/\1/' -} - -ec2_ip4s() { - get_imds_token - # NOTE: metadata for ENI arrives later than hotplug events - TRIES=60 - while true; do - IP4="$(mac_meta local-ipv4s)" - [ -n "$IP4" ] && break - let TRIES-- - if [ "$TRIES" -eq 0 ]; then - log err "Unable to get IPv4 addresses for $IFACE after 30s" - exit 1 - fi - sleep 0.5 - done - IP4S="$(echo "$IP4" | tail +2)" # secondary IPs (udhcpc handles primary) - - # non-eth0 interfaces need custom route tables - # - if [ "$IFACE" != eth0 ] && [ -n "$IP4S" ] && - [ -z "$(ip +F -4 route show table "$RTABLE")" ]; then - IP4P="$(echo "$IP4" | head -1)" # primary IP - IP4_CIDR="$(mac_meta subnet-ipv4-cidr-block)" - IP4_GW="$(echo "$IP4_CIDR" | cut -d/ -f1 | - awk -F. '{ print $1"."$2"."$3"."$4+1 }')" - ip -4 route add default via "$IP4_GW" dev "$IFACE" table "$RTABLE" - ip -4 route add "$IP4_CIDR" dev "$IFACE" proto kernel scope link \ - src "$IP4P" table "$RTABLE" - fi - echo "$IP4S" -} - -ec2_ip6s() { - get_imds_token - # NOTE: IPv6 metadata (if any) may arrive later than IPv4 metadata - TRIES=60 - while true; do - IP6S="$(mac_meta ipv6s)" - [ -n "$IP6S" ] && break - let TRIES-- - if [ "$TRIES" -eq 0 ]; then - log warn "Unable to get IPv6 addresses for $IFACE after 30s" - break - fi - sleep 0.5 - done - - # non-eth0 interfaces need custom route tables - # - # NOTE: busybox iproute2 doesn't do 'route show table' properly for IPv6, - # so iproute2-minimal package is required! - # - if [ "$IFACE" != eth0 ] && [ -n "$IP6S" ] && - [ -z "$(ip +F -6 route show table "$RTABLE")" ]; then - TRIES=20 - while true; do - GW="$(ip -6 route show dev "$IFACE" default | awk '{ print $3 }')" - [ -n "$GW" ] && break - let TRIES-- - if [ "$TRIES" -eq 0 ]; then - log warn "Unable to get IPv6 gateway RA after 10s" - break - fi - sleep 0.5 - done - ip -6 route add default via "$GW" dev "$IFACE" table "$RTABLE" - fi - echo "$IP6S" -} - -in_list() { - echo "$2" | grep -q "^$1$" -} - -# ip_addr {4|6} {add|del} -ip_addr() { - [ "$1" -eq 6 ] && MASK=128 || MASK="$mask" # IP6s are always /128 - ip -"$1" addr "$2" "$3/$MASK" dev "$IFACE" - [ "$IFACE" = eth0 ] && return - - # non-eth0 interfaces get rules associating IPs with route tables - ip -"$1" rule "$2" from "$3" lookup "$RTABLE" -} - -# sync_ips {4|6} "" "" -sync_ips() { - # remove extra IPs - for i in $3; do - in_list "$i" "$2" || ip_addr "$1" del "$i" - done - # add missing IPs - for i in $2; do - in_list "$i" "$3" || ip_addr "$1" add "$i" - done -} - -log info "STARTING: $IFACE" - -if [ "$HOOK" != post-bound ] && [ "$HOOK" != post-renew ]; then - log err "Unhandled udhcpc hook '$HOOK'" - exit 1 -fi - -sync_ips 4 "$(ec2_ip4s)" "$(iface_ip4s)" -sync_ips 6 "$(ec2_ip6s)" "$(iface_ip6s)" - -log info "FINISHED: $IFACE" diff --git a/scripts/setup-tiny.d/eth-eni-hotplug b/scripts/setup-tiny.d/eth-eni-hotplug deleted file mode 100755 index b421942..0000000 --- a/scripts/setup-tiny.d/eth-eni-hotplug +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/sh -# vim: set ts=4 et: - -set -e - -PROC="$(basename "$0")[$$]" - -DEBUG= - -log() { - [ -z "$DEBUG" ] && [ "$1" = "debug" ] && return - FACILITY="kern.$1" - shift - logger -s -p "$FACILITY" -t "$PROC" "$@" -} - -if [ -z "$MDEV" ]; then - log err "MDEV env not defined" - exit 1 -fi - -RTABLE="${MDEV#eth}" -let RTABLE+=1000 - -IFACE_CFG=/etc/network/interfaces - -ip() { - v=-4 - if [ "$1" = '-4' ] || [ "$1" = '-6' ]; then - v="$1" - shift - fi - OP="$2" - [ "$OP" = show ] && LEV=debug || LEV=info - if /sbin/ip "$v" "$@" || [ -n "$FAIL_OK" ]; then - log "$LEV" "OK: ip $v $*" - else - log err "FAIL: ip $v $*" - fi -} - -assemble_interfaces() { - log info "Rebuilding $IFACE_CFG" - /etc/network/assemble-interfaces -} - -interface_up() { - log info "Bringing up $MDEV" - ifup "$MDEV" -} - -cleanup_interface() { - log info "Cleaning up $MDEV" - - # kill related udhcpc - kill "$(cat "/run/udhcpc.$MDEV.pid")" - - # tidy up /run/ifstate, if it exists - [ -f /run/ifstate ] && sed -i -e "/^$MDEV=/d" /run/ifstate - - # remove related rules - for V in 4 6; do - for P in $(ip -"$V" rule show table "$RTABLE" | cut -d: -f1); do - ip -"$V" rule del pref "$P" - done - done -} - -log info "STARTING: $ACTION $MDEV" - -if exec 200>>"$IFACE_CFG"; then - if flock 200; then - case $ACTION in - add|"") - assemble_interfaces - interface_up - ;; - remove) - assemble_interfaces - cleanup_interface - ;; - *) - log err "Unknown action '$ACTION'" - exit 1 - ;; - esac - else - log err "Unable to flock $IFACE_CFG" - exit 1 - fi -else - log err "Unable to assign fd 200 to flock $IFACE_CFG" - exit 1 -fi - -log info "FINISHED: $ACTION $MDEV" diff --git a/scripts/setup-tiny.d/eth-eni-setup b/scripts/setup-tiny.d/eth-eni-setup deleted file mode 100755 index aba272a..0000000 --- a/scripts/setup-tiny.d/eth-eni-setup +++ /dev/null @@ -1,19 +0,0 @@ -#!/sbin/openrc-run - -# vim: set ts=4 et: -# shellcheck shell=sh - -description="Sets up interfaces for attached Elastic Network Interfaces" - -depend() { - before net - need sysfs -} - -start() { - local iface - - ebegin "Setting up interfaces for attached ENIs" - /etc/network/assemble-interfaces - eend "$?" -} diff --git a/scripts/setup-tiny.d/interfaces.d/DEFAULT b/scripts/setup-tiny.d/interfaces.d/DEFAULT deleted file mode 100644 index b6fdf49..0000000 --- a/scripts/setup-tiny.d/interfaces.d/DEFAULT +++ /dev/null @@ -1,3 +0,0 @@ -auto %% -iface %% inet dhcp - diff --git a/scripts/setup-tiny.d/interfaces.d/eth0 b/scripts/setup-tiny.d/interfaces.d/eth0 deleted file mode 100644 index ed9d3b7..0000000 --- a/scripts/setup-tiny.d/interfaces.d/eth0 +++ /dev/null @@ -1,3 +0,0 @@ -auto eth0 -iface eth0 inet dhcp - diff --git a/scripts/setup-tiny.d/interfaces.d/lo b/scripts/setup-tiny.d/interfaces.d/lo deleted file mode 100644 index 77efa67..0000000 --- a/scripts/setup-tiny.d/interfaces.d/lo +++ /dev/null @@ -1,3 +0,0 @@ -auto lo -iface lo inet loopback - diff --git a/scripts/setup-tiny.d/mdev-ec2.conf b/scripts/setup-tiny.d/mdev-ec2.conf deleted file mode 100644 index 860a96e..0000000 --- a/scripts/setup-tiny.d/mdev-ec2.conf +++ /dev/null @@ -1,6 +0,0 @@ -# additional ENIs -eth[1-9] root:root 0644 */lib/mdev/eth-eni-hotplug - -# EBS NVMe links -nvme[0-9]+n[0-9]+.* root:root 0660 */lib/mdev/nvme-ebs-links - diff --git a/scripts/setup-tiny.d/nvme-ebs-links b/scripts/setup-tiny.d/nvme-ebs-links deleted file mode 100755 index e4ba49f..0000000 --- a/scripts/setup-tiny.d/nvme-ebs-links +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -# vim: set ts=4 et: - -[ -x /usr/sbin/nvme ] || exit - -PROC="$(basename "$0")[$$]" - -log() { - FACILITY="kern.$1" - shift - logger -s -p "$FACILITY" -t "$PROC" "$@" -} - -raw_ebs_alias() { - /usr/sbin/nvme id-ctrl "/dev/$BASE" -b 2>/dev/null | dd bs=32 skip=96 count=1 2>/dev/null -} - -case $ACTION in - add|"") - BASE=$(echo "$MDEV" | sed -re 's/^(nvme[0-9]+n[0-9]+).*/\1/') - PART=$(echo "$MDEV" | sed -re 's/nvme[0-9]+n[0-9]+p?//g') - MAXTRY=50 - TRY=0 - until [ -n "$EBS" ]; do - EBS=$(raw_ebs_alias | sed -nre '/^(\/dev\/)?(s|xv)d[a-z]{1,2} /p' | tr -d ' ') - [ -n "$EBS" ] && break - TRY=$((TRY + 1)) - if [ $TRY -eq $MAXTRY ]; then - log err "Failed to get EBS volume alias for $MDEV after $MAXTRY attempts ($(raw_ebs_alias))" - exit 1 - fi - sleep 0.1 - done - # remove any leading '/dev/', 'sd', or 'xvd', and append partition - EBS=${EBS#/dev/} - EBS=${EBS#sd} - EBS=${EBS#xvd}$PART - ln -sf "$MDEV" "sd$EBS" && log notice "Added sd$EBS symlink for $MDEV" - ln -sf "$MDEV" "xvd$EBS" && log notice "Added xvd$EBS symlink for $MDEV" - ;; - remove) - for TARGET in sd* xvd* - do - [ "$(readlink "$TARGET" 2>/dev/null)" = "$MDEV" ] && rm -f "$TARGET" && log notice "Removed $TARGET symlink for $MDEV" - done - ;; -esac diff --git a/scripts/setup.d/interfaces b/scripts/setup.d/interfaces new file mode 100644 index 0000000..864f3e3 --- /dev/null +++ b/scripts/setup.d/interfaces @@ -0,0 +1,7 @@ +# default alpine-cloud-images network configuration + +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet dhcp