ENI Hotplugging, etc.

ENI Hotplug / udhcpc script
* works with all Alpine versions back to 3.9
* udhcpc handles ENI's primary IPv4
* post-bound/post-renews eth-eni-hook handles secondary IPv4 & IPv6 addresses, route tables, and rules

setup-ami tweaks
* move scripts to be installed into setup-ami.d/
* move config snippets into setup-ami.d/etc/ (previously embedded in setup-ami)
This commit is contained in:
Jake Buchholz 2020-09-12 10:57:35 -07:00 committed by Mike Crute
parent a9ba2532df
commit c6f5325873
16 changed files with 354 additions and 154 deletions

View File

@ -17,12 +17,12 @@ alpine {
# Build definitions
BUILDS {
# merge version, arch, profile; add { revision = "r1" } if needed
v3_12-x86_64 = ${version-3_12} ${arch-x86_64} ${alpine}
v3_11-x86_64 = ${version-3_11} ${arch-x86_64} ${alpine}
v3_10-x86_64 = ${version-3_10} ${arch-x86_64} ${alpine}
v3_9-x86_64 = ${version-3_9} ${arch-x86_64} ${alpine}
edge-x86_64 = ${version-edge} ${arch-x86_64} ${alpine}
v3_12-x86_64 = ${version-3_12} ${arch-x86_64} ${alpine}
v3_11-x86_64 = ${version-3_11} ${arch-x86_64} ${alpine} { revision = "r1" }
v3_10-x86_64 = ${version-3_10} ${arch-x86_64} ${alpine} { revision = "r1" }
v3_9-x86_64 = ${version-3_9} ${arch-x86_64} ${alpine} { revision = "r1" }
v3_12-aarch64 = ${version-3_12} ${arch-aarch64} ${alpine}
edge-aarch64 = ${version-edge} ${arch-aarch64} ${alpine}
v3_12-aarch64 = ${version-3_12} ${arch-aarch64} ${alpine}
}

View File

@ -50,6 +50,7 @@ pkgs {
linux-virt = true
alpine-mirrors = true
chrony = true
iproute2 = true
nvme-cli = true
openssh = true
sudo = true

View File

@ -49,6 +49,7 @@ repos {}
pkgs {
linux-virt = true
chrony = true
iproute2-minimal = true
nvme-cli = true
openssh = true
sudo = true

View File

@ -1,112 +0,0 @@
#!/usr/bin/env sh
# vim: set ft=sh noet ts=4:
# This script should be installed as symlinks in
# /etc/udhcpc/<hook>/alpine-ec2-eni-hook
# <hook> :-
# post-bound - after udhcpc binds an IP to the interface
# post-renew - after udhcpc renews the lease for the IP
#
# udhcpc provides via ENV...
# IFACE - eth0, etc.
# mask - bits in IPv4 subnet mask
set -e
HOOK="$(basename "$(dirname "$0")")"
log() {
LEV="$1"
shift
logger -s -p "daemon.$LEV" -t "udhcpc/${HOOK}[$$]" "$@"
}
IMDS="X-aws-ec2-metadata-token"
meta_token() {
echo -ne "PUT /latest/api/token HTTP/1.0\r\n$IMDS-ttl-seconds: 5\r\n\r\n" |
nc 169.254.169.254 80 | tail -n 1
}
meta() {
wget -qO - --header "$IMDS: $(meta_token)" \
"http://169.254.169.254/latest/meta-data/$1" 2>/dev/null \
|| true # when no ipv6s attached, wget will 404 error
}
iface_mac() {
cat "/sys/class/net/$IFACE/address"
}
iface_ip6s() {
# only inet6 lines, except fe80::
ip -6 addr show "$IFACE" |
sed -E -e '/inet6/!d' \
-e '/inet6 fe80::/d' \
-e 's/.*inet6 ([0-9a-f:]+).*/\1/'
}
iface_sec_ip4s() {
# only inet secondary lines
ip -4 addr show "$IFACE" |
sed -E -e '/inet .* secondary/!d' \
-e 's/.*inet ([0-9.]+).*/\1/'
}
ec2_ip6s() {
meta "$MAC_PATH/ipv6s"
}
ec2_sec_ip4s() {
# first one listed is primary
meta "$MAC_PATH/local-ipv4s" | tail +2
}
in_list() {
echo "$2" | grep -q "^$1$"
}
ip_addr() {
[ "$1" -eq 4 ] && MASK="$mask" || MASK=128
MSG="$IFACE $2 $3"
if ip -"$1" addr "$2" "$3/$MASK" dev "$IFACE"; then
log notice "$MSG - success"
else
log err "$MSG - FAILED"
fi
}
case "$HOOK" in
#pre-deconfig) ACT=del ;; # issues with 'service networking restart'
post-bound) ACT=add ;;
post-renew) ACT=sync ;;
*) log err "Unhandled udhcpc hook: '$HOOK'"; exit 1 ;;
esac
MAC_PATH="network/interfaces/macs/$(iface_mac)"
EC2_IP4S="$(ec2_sec_ip4s)"
EC2_IP6S="$(ec2_ip6s)"
IFACE_IP4S="$(iface_sec_ip4s)"
IFACE_IP6S="$(iface_ip6s)"
# add or sync IPs
if [ "$ACT" != "del" ]; then
for ip4 in $EC2_IP4S; do
in_list "$ip4" "$IFACE_IP4S" || ip_addr 4 add "$ip4"
done
for ip6 in $EC2_IP6S; do
in_list "$ip6" "$IFACE_IP6S" || ip_addr 6 add "$ip6"
done
fi
# del or sync IPs
if [ "$ACT" != "add" ]; then
for ip4 in $IFACE_IP4S; do
[ "$ACT" = "sync" ] && in_list "$ip4" "$EC2_IP4S" && continue
ip_addr 4 del "$ip4"
done
for ip6 in $IFACE_IP6S; do
[ "$ACT" = "sync" ] && in_list "$ip6" "$EC2_IP6S" && continue
ip_addr 6 del "$ip6"
done
fi

View File

@ -576,9 +576,9 @@ class ConfigBuilder:
shutil.rmtree(build_dir, ignore_errors=True)
os.makedirs(setup_dir)
# symlink nvme and ENI scripts
self.rel_symlink("scripts/nvme-ebs-links", setup_dir, "nvme-ebs-links")
self.rel_symlink("scripts/alpine-ec2-eni-hook", setup_dir, "alpine-ec2-eni-hook")
# symlink everything in scripts/setup-ami.d
for item in os.listdir("scripts/setup-ami.d"):
self.rel_symlink(os.path.join("scripts/setup-ami.d", item), setup_dir, item)
# symlink additional setup_script
if "setup_script" in cfg.keys() and cfg["setup_script"] is not None:

View File

@ -1,10 +1,11 @@
#!/bin/sh
# vim: set ts=4 et:
set -eu
set -eux
DEVICE=/dev/xvdf
TARGET=/mnt/target
SETUP=/tmp/setup-ami.d
# what bootloader should we use?
[ -d "/sys/firmware/efi" ] && BOOTLOADER=grub-efi || BOOTLOADER=syslinux
@ -85,7 +86,7 @@ setup_partitions() {
make_filesystem() {
root_dev="$DEVICE"
if [ "$BOOTLOADER" = 'grub-efi' ]; then
if [ "$BOOTLOADER" = grub-efi ]; then
# create a small EFI partition (remainder for root), and mount it
setup_partitions '5M,U,*' ',L'
root_dev="${DEVICE}2"
@ -95,7 +96,7 @@ make_filesystem() {
mkfs.ext4 -O ^64bit -L / "$root_dev"
mount "$root_dev" "$TARGET"
if [ "$BOOTLOADER" = 'grub-efi' ]; then
if [ "$BOOTLOADER" = grub-efi ]; then
mkdir -p "$TARGET/boot/efi"
mount -t vfat "${DEVICE}1" "$TARGET/boot/efi"
fi
@ -117,7 +118,7 @@ fetch_keys() {
install_base() {
$apk add --root "$TARGET" --no-cache --initdb alpine-base
# verify release matches
if [ "$VERSION" != "edge" ]; then
if [ "$VERSION" != edge ]; then
ALPINE_RELEASE=$(cat "$TARGET/etc/alpine-release")
[ "$RELEASE" = "$ALPINE_RELEASE" ] || \
die "Newer Alpine release detected: $ALPINE_RELEASE"
@ -155,11 +156,12 @@ install_core_packages() {
setup_mdev() {
install -o root -g root -Dm755 -t "$TARGET/lib/mdev" \
/tmp/setup-ami.d/nvme-ebs-links
"$SETUP/nvme-ebs-links" \
"$SETUP/eth-eni-hotplug"
# insert nvme ebs mdev configs just above "# fallback" comment
sed -n -i \
-e '/# fallback/i \\n# ebs nvme links\nnvme[0-9]+n[0-9]+.* root:root 0660 */lib/mdev/nvme-ebs-links' \
-e "/# fallback/r $SETUP/etc/mdev-ec2.conf" \
-e 1x -e '2,${x;p}' -e '${x;p}' \
"$TARGET/etc/mdev.conf"
}
@ -217,46 +219,38 @@ install_grub_efi() {
# fallback mode
install -D "$TARGET/boot/efi/EFI/alpine/grub$fwa.efi" "$TARGET/boot/efi/EFI/boot/boot$fwa.efi"
cat > "$TARGET/etc/default/grub" <<- EOF
GRUB_TIMEOUT=0
GRUB_DISABLE_SUBMENU=y
GRUB_DISABLE_RECOVERY=true
GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
GRUB_CMDLINE_LINUX_DEFAULT="modules=$KERNEL_MODS $KERNEL_OPTS"
EOF
# install default grub config
install -o root -g root -Dm644 -t "$TARGET/etc/default" \
"$SETUP/etc/grub.default" grub
# generate/install new config
chroot "$TARGET" grub-mkconfig -o /boot/grub/grub.cfg
}
setup_fstab() {
cat > "$TARGET/etc/fstab" <<EOF
# <fs> <mountpoint> <type> <opts> <dump/pass>
LABEL=/ / ext4 defaults,noatime 1 1
EOF
install -o root -g root -Dm644 -t "$TARGET/etc" \
"$SETUP/etc/fstab"
# if we're using grub-efi bootloader, add extra line for EFI partition
if [ "$BOOTLOADER" = 'grub-efi' ]; then
echo "LABEL=EFI /boot/efi vfat defaults,noatime,uid=0,gid=0,umask=077 0 0" >> "$TARGET/etc/fstab"
if [ "$BOOTLOADER" = grub-efi ]; then
cat "$SETUP/etc/fstab.grub-efi" >> "$TARGET/etc/fstab"
fi
}
setup_networking() {
# configure standard interfaces
cat > "$TARGET/etc/network/interfaces" <<EOF
auto lo
iface lo inet loopback
auto eth0
iface eth0 inet dhcp
EOF
IFACE_CFG="$TARGET/etc/network/interfaces"
install -o root -g root -Dm755 -d "$SETUP/etc/interfaces.d" "$IFACE_CFG.d"
install -o root -g root -Dm644 -t "$IFACE_CFG.d" \
"$SETUP/etc/interfaces.d/"*
cat "$IFACE_CFG.d/lo" "$IFACE_CFG.d/eth0" > "$IFACE_CFG"
# install ucdhcp hooks for EC2 ENI IPv6 and secondary IPv4
install -o root -g root -Dm755 -t "$TARGET/usr/share/udhcpc" \
/tmp/setup-ami.d/alpine-ec2-eni-hook
"$SETUP/eth-eni-hook"
for i in post-bound post-renew; do
mkdir -p "$TARGET/etc/udhcpc/$i"
ln -sf /usr/share/udhcpc/alpine-ec2-eni-hook \
ln -sf /usr/share/udhcpc/eth-eni-hook \
"$TARGET/etc/udhcpc/$i"
done
}
@ -286,9 +280,7 @@ create_alpine_user() {
chroot "$TARGET" /usr/bin/passwd -u "$user"
# Let tiny-ec2-bootstrap know what the EC2 user of the AMI is
cat > "$TARGET/etc/conf.d/tiny-ec2-bootstrap" <<EOF
EC2_USER="$user"
EOF
echo "EC2_USER=$user" > "$TARGET/etc/conf.d/tiny-ec2-bootstrap"
}
configure_ntp() {
@ -303,10 +295,10 @@ configure_ntp() {
}
setup_script() {
if [ -f /tmp/setup-ami.d/setup_script ]; then
if [ -f "$SETUP/setup_script" ]; then
einfo "Executing additional setup script"
(
cd /tmp/setup-ami.d
cd "$SETUP"
chmod u+x ./setup_script
TARGET="$TARGET" ./setup_script
)
@ -324,7 +316,7 @@ cleanup() {
"$TARGET/root/.ash_history" \
"$TARGET/etc/"*-
[ "$BOOTLOADER" = 'grub-efi' ] && umount "$TARGET/boot/efi"
[ "$BOOTLOADER" = grub-efi ] && umount "$TARGET/boot/efi"
umount \
"$TARGET/dev" \

View File

@ -0,0 +1,2 @@
# <fs> <mountpoint> <type> <opts> <dump/pass>
LABEL=/ / ext4 defaults,noatime 1 1

View File

@ -0,0 +1 @@
LABEL=EFI /boot/efi vfat defaults,noatime,uid=0,gid=0,umask=077 0 0

View File

@ -0,0 +1,5 @@
GRUB_TIMEOUT=0
GRUB_DISABLE_SUBMENU=y
GRUB_DISABLE_RECOVERY=true
GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
GRUB_CMDLINE_LINUX_DEFAULT="modules=$KERNEL_MODS $KERNEL_OPTS"

View File

@ -0,0 +1,3 @@
auto %%
iface %% inet dhcp

View File

@ -0,0 +1,3 @@
auto eth0
iface eth0 inet dhcp

View File

@ -0,0 +1,6 @@
# NOTE: /lib/mdev/eth-eni-hotplug rewrites this file. Edit files in
# /etc/network/interfaces.d/ to persist any customizations.
auto lo
iface lo inet loopback

View File

@ -0,0 +1,6 @@
# additional ENIs
eth[1-9] root:root 0666 */lib/mdev/eth-eni-hotplug
# EBS NVMe links
nvme[0-9]+n[0-9]+.* root:root 0660 */lib/mdev/nvme-ebs-links

187
scripts/setup-ami.d/eth-eni-hook Executable file
View File

@ -0,0 +1,187 @@
#!/bin/sh
# vim: set ts=4 et:
# This script should be installed as symlinks in
# /etc/udhcpc/<hook>/eth-eni-hook
# <hook> :-
# post-bound - after udhcpc binds an IP to the interface
# post-renew - after udhcpc renews the lease for the IP
#
# udhcpc provides via ENV...
# IFACE - eth0, etc.
# mask - bits in IPv4 subnet mask
set -e
HOOK="$(basename "$(dirname "$0")")"
DEBUG=
log() {
[ -z "$DEBUG" ] && [ "$1" = "debug" ] && return
FACILITY="daemon.$1"
shift
logger -s -p "$FACILITY" -t "udhcpc/${HOOK}[$$]" -- "$@"
}
if [ -z "$IFACE" ] || [ -z "$mask" ]; then
log err "Missing 'IFACE' or 'mask' ENV from udhcpc"
exit 1
fi
# route table number
RTABLE="${IFACE#eth}"
let RTABLE+=1000
IMDS=X-aws-ec2-metadata-token
IMDS_IP=169.254.169.254
IMDS_MAC="http://$IMDS_IP/latest/meta-data/network/interfaces/macs/$(
cat "/sys/class/net/$IFACE/address")"
get_imds_token() {
IMDS_TOKEN="$(echo -ne \
"PUT /latest/api/token HTTP/1.0\r\n$IMDS-ttl-seconds: 60\r\n\r\n" |
nc "$IMDS_IP" 80 | tail -n 1)"
}
mac_meta() {
wget -qO - --header "$IMDS: $IMDS_TOKEN" "$IMDS_MAC/$1" 2>/dev/null \
|| true # when no ipv6s attached (yet), IMDS returns 404 error
}
ip() {
FAIL_OK=
if [ "$1" = '+F' ]; then
FAIL_OK=1
shift
fi
v=-4
if [ "$1" = '-4' ] || [ "$1" = '-6' ]; then
v="$1"
shift
fi
OP="$2"
[ "$OP" = show ] && LEV=debug || LEV=info
if /sbin/ip "$v" "$@" || [ -n "$FAIL_OK" ]; then
log "$LEV" "OK: ip $v $*"
else
log err "FAIL: ip $v $*"
fi
}
iface_ip4s() {
ip -4 addr show "$IFACE" secondary |
sed -E -e '/inet /!d' -e 's/.*inet ([0-9.]+).*/\1/'
}
iface_ip6s() {
ip -6 addr show "$IFACE" scope global |
sed -E -e '/inet6/!d' -e 's/.*inet6 ([0-9a-f:]+).*/\1/'
}
ec2_ip4s() {
get_imds_token
# NOTE: metadata for ENI arrives later than hotplug events
TRIES=60
while true; do
IP4="$(mac_meta local-ipv4s)"
[ -n "$IP4" ] && break
let TRIES--
if [ "$TRIES" -eq 0 ]; then
log err "Unable to get IPv4 addresses for $IFACE after 30s"
exit 1
fi
sleep 0.5
done
IP4S="$(echo "$IP4" | tail +2)" # secondary IPs (udhcpc handles primary)
# non-eth0 interfaces need custom route tables
#
if [ "$IFACE" != eth0 ] && [ -n "$IP4S" ] &&
[ -z "$(ip +F -4 route show table "$RTABLE")" ]; then
IP4P="$(echo "$IP4" | head -1)" # primary IP
IP4_CIDR="$(mac_meta subnet-ipv4-cidr-block)"
IP4_GW="$(echo "$IP4_CIDR" | cut -d/ -f1 |
awk -F. '{ print $1"."$2"."$3"."$4+1 }')"
ip -4 route add default via "$IP4_GW" dev "$IFACE" table "$RTABLE"
ip -4 route add "$IP4_CIDR" dev "$IFACE" proto kernel scope link \
src "$IP4P" table "$RTABLE"
fi
echo "$IP4S"
}
ec2_ip6s() {
get_imds_token
# NOTE: IPv6 metadata (if any) may arrive later than IPv4 metadata
TRIES=60
while true; do
IP6S="$(mac_meta ipv6s)"
[ -n "$IP6S" ] && break
let TRIES--
if [ "$TRIES" -eq 0 ]; then
log warn "Unable to get IPv6 addresses for $IFACE after 30s"
break
fi
sleep 0.5
done
# non-eth0 interfaces need custom route tables
#
# NOTE: busybox iproute2 doesn't do 'route show table' properly for IPv6,
# so iproute2-minimal package is required!
#
if [ "$IFACE" != eth0 ] && [ -n "$IP6S" ] &&
[ -z "$(ip +F -6 route show table "$RTABLE")" ]; then
TRIES=20
while true; do
GW="$(ip -6 route show dev "$IFACE" default | awk '{ print $3 }')"
[ -n "$GW" ] && break
let TRIES--
if [ "$TRIES" -eq 0 ]; then
log warn "Unable to get IPv6 gateway RA after 10s"
break
fi
sleep 0.5
done
ip -6 route add default via "$GW" dev "$IFACE" table "$RTABLE"
fi
echo "$IP6S"
}
in_list() {
echo "$2" | grep -q "^$1$"
}
# ip_addr {4|6} {add|del} <ip>
ip_addr() {
[ "$1" -eq 6 ] && MASK=128 || MASK="$mask" # IP6s are always /128
ip -"$1" addr "$2" "$3/$MASK" dev "$IFACE"
[ "$IFACE" = eth0 ] && return
# non-eth0 interfaces get rules associating IPs with route tables
ip -"$1" rule "$2" from "$3" lookup "$RTABLE"
}
# sync_ips {4|6} "<ec2-ips>" "<iface-ips>"
sync_ips() {
# remove extra IPs
for i in $3; do
in_list "$i" "$2" || ip_addr "$1" del "$i"
done
# add missing IPs
for i in $2; do
in_list "$i" "$3" || ip_addr "$1" add "$i"
done
}
log info "STARTING: $IFACE"
if [ "$HOOK" != post-bound ] && [ "$HOOK" != post-renew ]; then
log err "Unhandled udhcpc hook '$HOOK'"
exit 1
fi
sync_ips 4 "$(ec2_ip4s)" "$(iface_ip4s)"
sync_ips 6 "$(ec2_ip6s)" "$(iface_ip6s)"
log info "FINISHED: $IFACE"

View File

@ -0,0 +1,105 @@
#!/bin/sh
# vim: set ts=4 et:
set -e
PROC="$(basename "$0")[$$]"
DEBUG=
log() {
[ -z "$DEBUG" ] && [ "$1" = "debug" ] && return
FACILITY="kern.$1"
shift
logger -s -p "$FACILITY" -t "$PROC" "$@"
}
if [ -z "$MDEV" ]; then
log err "MDEV env not defined"
exit 1
fi
RTABLE="${MDEV#eth}"
let RTABLE+=1000
IFACE_CFG=/etc/network/interfaces
IFACE_DIR="${IFACE_CFG}.d"
ip() {
v=-4
if [ "$1" = '-4' ] || [ "$1" = '-6' ]; then
v="$1"
shift
fi
OP="$2"
[ "$OP" = show ] && LEV=debug || LEV=info
if /sbin/ip "$v" "$@" || [ -n "$FAIL_OK" ]; then
log "$LEV" "OK: ip $v $*"
else
log err "FAIL: ip $v $*"
fi
}
assemble_interfaces() {
log info "Rebuilding $IFACE_CFG"
cd "$IFACE_DIR"
cat lo > "$IFACE_CFG.new"
for i in /sys/class/net/eth*; do
IFACE="$(basename "$i")"
[ ! -f "$IFACE" ] && sed -e "s/%%/$IFACE/g" DEFAULT > "$IFACE"
cat "$IFACE" >> "$IFACE_CFG.new"
done
cp -a "$IFACE_CFG" "$IFACE_CFG.bak"
mv "$IFACE_CFG.new" "$IFACE_CFG"
}
interface_up() {
log info "Bringing up $MDEV"
ifup "$MDEV"
}
cleanup_interface() {
log info "Cleaning up $MDEV"
# kill related udhcpc
kill "$(cat "/run/udhcpc.$MDEV.pid")"
# tidy up /run/ifstate, if it exists
[ -f /run/ifstate ] && sed -i -e "/^$MDEV=/d" /run/ifstate
# remove related rules
for V in 4 6; do
for P in $(ip -"$V" rule show table "$RTABLE" | cut -d: -f1); do
ip -"$V" rule del pref "$P"
done
done
}
log info "STARTING: $ACTION $MDEV"
if exec 200>>"$IFACE_CFG"; then
if flock 200; then
case $ACTION in
add|"")
assemble_interfaces
interface_up
;;
remove)
assemble_interfaces
cleanup_interface
;;
*)
log err "Unknown action '$ACTION'"
exit 1
;;
esac
else
log err "Unable to flock $IFACE_CFG"
exit 1
fi
else
log err "Unable to assign fd 200 to flock $IFACE_CFG"
exit 1
fi
log info "FINISHED: $ACTION $MDEV"