From 638be8d8b674acfb56bf59d586859a4622b9ad22 Mon Sep 17 00:00:00 2001 From: Mike Crute Date: Mon, 25 Dec 2017 02:06:54 +0000 Subject: [PATCH] Initial import --- .gitignore | 3 + LICENSE.txt | 19 +++ Makefile | 22 ++++ README.md | 72 +++++++++++ alpine-ami.yaml | 57 +++++++++ make_ami.sh | 325 ++++++++++++++++++++++++++++++++++++++++++++++++ release.yaml | 21 ++++ 7 files changed, 519 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 Makefile create mode 100644 README.md create mode 100644 alpine-ami.yaml create mode 100755 make_ami.sh create mode 100644 release.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..98c621a --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/build/ +/.py3/ +/variables.json diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..736d3fe --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2017 Michael Crute + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..de19494 --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +.PHONY: ami +ami: build/convert + build/convert alpine-ami.yaml > build/alpine-ami.json + packer build -var-file=variables.json build/alpine-ami.json + +build/convert: + [ -d ".py3" ] || python3 -m venv .py3 + .py3/bin/pip install pyyaml + + [ -d "build" ] || mkdir build + + # Make stupid simple little YAML/JSON converter so we can maintain our + # packer configs in a sane format that allows comments but also use packer + # which only supports JSON + @echo "#!`pwd`/.py3/bin/python" > build/convert + @echo "import yaml, json, sys" >> build/convert + @echo "json.dump(yaml.load(open(sys.argv[1])), sys.stdout, indent=4, separators=(',', ': '))" >> build/convert + @chmod +x build/convert + +.PHONY: clean +clean: + rm -rf build .py3 diff --git a/README.md b/README.md new file mode 100644 index 0000000..369ae45 --- /dev/null +++ b/README.md @@ -0,0 +1,72 @@ +# Alpine Linux EC2 AMI Build + +**NOTE: This is not an official Amazon or AWS provided image. This is community +built and supported.** + +This repository contains a packer file and a script to create an EC2 AMI +containing Alpine Linux. The AMI is designed to work with most EC2 features +natively and thus should launch on any instance type. If anything is missing +please report a bug. + +This image can be launched on any modern instance type. Including T2, M5, C5, +I3, R4, P2, P3, X1, X1e, D2. Other instances may also work but have not been +tested. If you find an issue with instance support for any current generation +instance please file a bug against this project. + +To get started use one of the AMIs below. The default user is `alpine` and will +be configured to use whatever SSH keys you chose when you launched the image. +If user data is specified it must be a shell script that begins with `#!`. If a +script is provided it will be executed as root after the network is configured. + +**Note:** The AMI is not yet available in all regions. This file and +[releases.yaml](https://github.com/mcrute/alpine-ec2-ami/blob/master/releases.yaml) +will be updated as new regions are made available. + +| Alpine Version | Region Code | Region Name | AMI ID | +| -------------- | -------------- | ------------------------- | --------------------------------------------------------------------------------------------- | +| 3.7 | us-east-1 | US East (N. Virginia) | [ami-XXXXXXXX](https://us-east-1.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | us-east-2 | US East (Ohio) | [ami-XXXXXXXX](https://us-east-2.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | us-west-1 | US West (N. California) | [ami-XXXXXXXX](https://us-west-1.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | us-west-2 | US West (Oregon) | [ami-032b877b](https://us-west-2.console.aws.amazon.com/ec2/home#launchAmi=ami-032b877b) | +| 3.7 | ca-central-1 | Canada (Central) | [ami-XXXXXXXX](https://ca-central-1.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | eu-central-1 | EU (Frankfurt) | [ami-XXXXXXXX](https://eu-central-1.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | eu-west-1 | EU (Ireland) | [ami-XXXXXXXX](https://eu-west-1.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | eu-west-2 | EU (London) | [ami-XXXXXXXX](https://eu-west-2.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | eu-west-3 | EU (Paris) | [ami-XXXXXXXX](https://eu-west-3.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | ap-northeast-1 | Asia Pacific (Tokyo) | [ami-XXXXXXXX](https://ap-northeast-1.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | ap-northeast-2 | Asia Pacific (Seoul) | [ami-XXXXXXXX](https://ap-northeast-2.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | ap-southeast-1 | Asia Pacific (Singapore) | [ami-XXXXXXXX](https://ap-southeast-1.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | ap-southeast-2 | Asia Pacific (Sydney) | [ami-XXXXXXXX](https://ap-southeast-2.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | ap-south-1 | Asia Pacific (Mumbai) | [ami-XXXXXXXX](https://ap-south-1.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | +| 3.7 | sa-east-1 | South America (São Paulo) | [ami-XXXXXXXX](https://sa-east-1.console.aws.amazon.com/ec2/home#launchAmi=ami-XXXXXXXX) | + +## Caveats + +This image is being used in production but it's still somewhat early stage in +its development and thus there are some sharp edges. + +- Only EBS-backed HVM instances are supported. While paravirtualized instances + are still available from AWS they are not supported on any of the newer + hardware so it seems unlikely that they will be supported going forward. Thus + this project does not support them. + +- Not all packages required have been merged into the upstream aports tree. + When they are they will still only be available on edge. Until then the image + sources a few packages from a testing repo managed by the owner of this + repository. The builds in this repository should be identical to what is + eventually merged into the official tree. + +- [cloud-init](https://cloudinit.readthedocs.io/en/latest/) is not currently + supported on Alpine Linux. Instead this image uses + [tiny-ec2-bootstrap](https://github.com/mcrute/tiny-ec2-bootstrap). Hostname + setting will work as will setting the ssh keys for the Alpine user based on + what was configured during instance launch. User data is supported as long + as it's a shell script (starts with #!). See the tiny-ec2-bootstrap README + for more details. You can still install cloud-init using aports but the + version in the tree is somewhat old and may not work correctly for Alpine. + If full cloud-init support is important to you please file a bug against this + project. + +- CloudFormation support is still forthcoming. This requires patches and + packaging for the upstream cfn tools that have not yet been accepted. + Eventually full CloudFormation support will be available. diff --git a/alpine-ami.yaml b/alpine-ami.yaml new file mode 100644 index 0000000..26b75b1 --- /dev/null +++ b/alpine-ami.yaml @@ -0,0 +1,57 @@ +variables: + security_group: "" + subnet: "" + destination_regions: "" + alpine_release: "3.7" + + # Don't override this without a good reason and if you do just make sure it + # gets passed all the way through to the make_ami script + volume_name: "/dev/xvdf" + +builders: + - type: "amazon-ebssurrogate" + + # Image is built inside a custom VPC so let Packer use the existing + # resources + security_group_id: "{{user `security_group`}}" + subnet_id: "{{user `subnet`}}" + + # Input Instance Setting + instance_type: "t2.micro" + launch_block_device_mappings: + - volume_type: "gp2" + device_name: "{{user `volume_name`}}" + delete_on_termination: false + volume_size: 5 + + # Output AMI Settings + ena_support: true + ami_name: "Alpine-{{user `alpine_release`}}-Hardened-EC2" + ami_description: "Alpine Linux {{user `alpine_release`}} Release with Hardened Kernel and EC2 Optimizations" + ami_groups: + - "all" + ami_virtualization_type: "hvm" + ami_regions: "{{user `destination_regions`}}" + ami_root_device: + source_device_name: "{{user `volume_name`}}" + device_name: "/dev/xvda" + delete_on_termination: true + volume_size: 5 + volume_type: "gp2" + + # Use the most recent Amazon Linux AMI as our base + ssh_username: "ec2-user" + source_ami_filter: + filters: + virtualization-type: "hvm" + root-device-type: "ebs" + architecture: "x86_64" + name: "amzn-ami-hvm-*-x86_64-gp2" + owners: + - "137112412989" + most_recent: true + +provisioners: + - type: "shell" + script: "make_ami.sh" + execute_command: "sudo sh -c '{{ .Vars }} {{ .Path }} {{user `volume_name`}}'" diff --git a/make_ami.sh b/make_ami.sh new file mode 100755 index 0000000..6ad4b0d --- /dev/null +++ b/make_ami.sh @@ -0,0 +1,325 @@ +#!/bin/sh +# vim:set ts=4: + +set -eu + +: ${ALPINE_RELEASE:="3.7"} # not tested against edge +: ${APK_TOOLS_URI:="https://github.com/alpinelinux/apk-tools/releases/download/v2.8.0/apk-tools-2.8.0-x86_64-linux.tar.gz"} +: ${APK_TOOLS_SHA256:="da21cefd2121e3a6cd4e8742b38118b2a1132aad7f707646ee946a6b32ee6df9"} +: ${ALPINE_KEYS:="http://dl-cdn.alpinelinux.org/alpine/v3.7/main/x86_64/alpine-keys-2.1-r1.apk"} +: ${ALPINE_KEYS_SHA256:="7b2d1e9a00324c8eee49785dc22355be02534201e77473ba9762027e1a475cc7"} + +die() { + printf '\033[1;31mERROR:\033[0m %s\n' "$@" >&2 # bold red + exit 1 +} + +einfo() { + printf '\n\033[1;36m> %s\033[0m\n' "$@" >&2 # bold cyan +} + +rc_add() { + local target="$1"; shift # target directory + local runlevel="$1"; shift # runlevel name + local services="$*" # names of services + + local svc; for svc in $services; do + mkdir -p "$target"/etc/runlevels/$runlevel + ln -s /etc/init.d/$svc "$target"/etc/runlevels/$runlevel/$svc + echo " * service $svc added to runlevel $runlevel" + done +} + +wgets() ( + local url="$1" # url to fetch + local sha256="$2" # expected SHA256 sum of output + local dest="$3" # output path and filename + + wget -T 10 -q -O "$dest" "$url" + echo "$sha256 $dest" | sha256sum -c > /dev/null +) + + +validate_block_device() { + local dev="$1" # target directory + + lsblk -P --fs "$dev" >/dev/null 2>&1 || \ + die "'$dev' is not a valid block device" + + if lsblk -P --fs "$dev" | grep -vq 'FSTYPE=""'; then + die "Block device '$dev' is not blank" + fi +} + +fetch_apk_tools() { + local store="$(mktemp -d)" + local tarball="$(basename $APK_TOOLS_URI)" + + wgets "$APK_TOOLS_URI" "$APK_TOOLS_SHA256" "$store/$tarball" + tar -C "$store" -xf "$store/$tarball" + + find "$store" -name apk +} + +make_filesystem() { + local device="$1" # target device path + local target="$2" # mount target + + mkfs.ext4 "$device" + e2label "$device" / + mount "$device" "$target" +} + +setup_repositories() { + local target="$1" # target directory + + mkdir -p "$target"/etc/apk/keys + cat > "$target"/etc/apk/repositories <<-EOF + http://dl-cdn.alpinelinux.org/alpine/v$ALPINE_RELEASE/main + http://dl-cdn.alpinelinux.org/alpine/v$ALPINE_RELEASE/community + EOF +} + +# This is mostly a temporary measure because some required packages have not +# yet been accepted upstream. This can be removed when the following pull +# requests are merged: +# +# - https://github.com/alpinelinux/aports/pull/2962 +# - https://github.com/alpinelinux/aports/pull/2961 +setup_staging_repos() { + local target="$1" # target directory + + echo "https://mcrute-build-artifacts.s3.us-west-2.amazonaws.com/alpine-packages/$ALPINE_RELEASE/testing" >> "$target"/etc/apk/repositories + + cat > "$target"/etc/apk/keys/mcrute-5a3eecec.rsa.pub <<-EOF + -----BEGIN PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5fW5dyTqgs9Yf93xKn5U + cYzY9t//M3TAaiDWH7rFxqBqTGnVGkP9QAGqsbXyoo/JpIalazkOfm/1L+XaK7NI + IUD/8KxfrnBW53cc/KOkPcGAga36aTBz/HmLQQvjWcizPxWepjdfvAnRTMV69Oud + zaRPGKx8nCRqLy1YFAEXn+zpHRh+OHCzzQFlkJop+2PCXqDFaMWC7+oWwrqFs1i0 + CXc4pq5oT6vAQyt6pUwN85sLVxtxXSt5G5ALYzQtaIj7IAR3jGlwU26wOAv5YP7z + xn/Z1ebQsPbAl3rw48v2T2ohPEX2TUtUq4OuwOG+z1pi3woIGOlOFVAP3k6lm8Z9 + 9QIDAQAB + -----END PUBLIC KEY----- + EOF +} + +fetch_keys() { + local target="$1" + local tmp="$(mktemp -d)" + + wgets "$ALPINE_KEYS" "$ALPINE_KEYS_SHA256" "$tmp/alpine-keys.apk" + tar -C "$target" -xvf "$tmp"/alpine-keys.apk etc/apk/keys + rm -rf "$tmp" +} + +setup_chroot() { + local target="$1" + + mount -t proc none "$target"/proc + mount --bind /dev "$target"/dev + mount --bind /sys "$target"/sys + + # Don't want to ship this but it's needed for bootstrap. Will be removed in + # the cleanup stage. + install -Dm644 /etc/resolv.conf "$target"/etc/resolv.conf +} + +install_core_packages() { + local target="$1" + + # Most from: https://git.alpinelinux.org/cgit/alpine-iso/tree/alpine-virt.packages + # + # acct - installed by some configurations, so added here + # aws-ena-driver-hardened - required for ENA enabled instances + # e2fsprogs - required by init scripts to maintain ext4 volumes + # linux-hardened - can't use virthardened because it's missing NVME support + # mkinitfs - required to build custom initfs + # sudo - to allow alpine user to become root, disallow root SSH logins + # tiny-ec2-bootstrap - to bootstrap system from EC2 metadata + chroot "$target" apk --no-cache add \ + acct \ + alpine-mirrors \ + aws-ena-driver-hardened \ + chrony \ + e2fsprogs \ + linux-hardened \ + mkinitfs \ + openssh \ + sudo \ + tiny-ec2-bootstrap \ + tzdata + + chroot "$target" apk --no-cache add --no-scripts syslinux +} + +create_initfs() { + local target="$1" + + # Create ENA feature for mkinitfs + # Submitted upstream: https://github.com/alpinelinux/mkinitfs/pull/19 + echo "kernel/drivers/net/ethernet/amazon" > \ + "$target"/etc/mkinitfs/features.d/ena.modules + + # Enable ENA and NVME features these don't hurt for any instance and are + # hard requirements of the 5 series and i3 series of instances + sed -Ei 's/^features="([^"]+)"/features="\1 nvme ena"/' \ + "$target"/etc/mkinitfs/mkinitfs.conf + + chroot "$target" /sbin/mkinitfs $(basename $(find "$target"/lib/modules/* -maxdepth 0)) +} + +setup_extlinux() { + local target="$1" + + # Must use disk labels instead of UUID or devices paths so that this works + # across instance familes. UUID works for many instances but breaks on the + # NVME ones because EBS volumes are hidden behind NVME devices. + # + # Enable ext4 because the root device is formatted ext4 + # + # Shorten timeout because EC2 has no way to interact with instance console + sed -Ei -e "s|^[# ]*(root)=.*|\1=LABEL=/|" \ + -e "s|^[# ]*(default_kernel_opts)=.*|\1=|" \ + -e "s|^[# ]*(modules)=.*|\1=sd-mod,usb-storage,ext4|" \ + -e "s|^[# ]*(default)=.*|\1=hardened|" \ + -e "s|^[# ]*(timeout)=.*|\1=1|" \ + "$target"/etc/update-extlinux.conf +} + +install_extlinux() { + local target="$1" + + chroot "$target" /sbin/extlinux --install /boot + chroot "$target" /sbin/update-extlinux --warn-only +} + +setup_fstab() { + local target="$1" + + cat > "$target"/etc/fstab <<-EOF + # + LABEL=/ / ext4 defaults,noatime 1 1 + EOF +} + +setup_networking() { + local target="$1" + + cat > "$target"/etc/network/interfaces <<-EOF + auto lo + iface lo inet loopback + + auto eth0 + iface eth0 inet dhcp + EOF +} + +enable_services() { + local target="$1" + + rc_add "$target" default sshd chronyd networking tiny-ec2-bootstrap + rc_add "$target" sysinit devfs dmesg mdev hwdrivers + rc_add "$target" boot modules hwclock swap hostname sysctl bootmisc syslog + rc_add "$target" shutdown killprocs savecache mount-ro +} + +create_alpine_user() { + local target="$1" + + # Allow members of the wheel group to sudo without a password. By default + # this will only be the alpine user. This allows us to ship an AMI that is + # accessible via SSH using the user's configured SSH keys (thanks to + # tiny-ec2-bootstrap) but does not allow remote root access which is the + # best-practice. + sed -i '/%wheel .* NOPASSWD: .*/s/^# //' "$target"/etc/sudoers + + # There is no real standard ec2 username across AMIs, Amazon uses ec2-user + # for their Amazon Linux AMIs but Ubuntu uses ubuntu, Fedora uses fedora, + # etc... (see: https://alestic.com/2014/01/ec2-ssh-username/). So our user + # and group are alpine because this is Alpine Linux. On instance bootstrap + # the user can create whatever users they want and delete this one. + chroot "$target" /usr/sbin/addgroup alpine + chroot "$target" /usr/sbin/adduser -h /home/alpine -s /bin/sh -G alpine -D alpine + chroot "$target" /usr/sbin/addgroup alpine wheel + chroot "$target" /usr/bin/passwd -u alpine +} + +configure_ntp() { + local target="$1" + + # EC2 provides an instance-local NTP service syncronized with GPS and + # atomic clocks in-region. Prefer this over external NTP hosts when running + # in EC2. + # + # See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html + sed -i 's/^server .*/server 169.254.169.123/' "$target"/etc/chrony/chrony.conf +} + +cleanup() { + local target="$1" + + # Sweep cruft out of the image that doesn't need to ship or will be + # re-generated when the image boots + rm -f \ + "$target"/var/cache/apk/* \ + "$target"/etc/resolv.conf \ + "$target"/root/.ash_history \ + "$target"/etc/*- + + umount \ + "$target"/dev \ + "$target"/proc \ + "$target"/sys + + umount "$target" +} + +main() { + [ "$#" -ne 1 ] && { echo "usage: $0 "; exit 1; } + + device="$1" + target="/mnt/target" + + validate_block_device "$device" + + [ -d "$target" ] || mkdir "$target" + + einfo "Fetching static APK tools" + apk="$(fetch_apk_tools)" + + einfo "Creating root filesystem" + make_filesystem "$device" "$target" + + setup_repositories "$target" + + einfo "Fetching Alpine signing keys" + fetch_keys "$target" + + setup_staging_repos "$target" + + einfo "Installing base system" + $apk add --root "$target" --update-cache --initdb alpine-base + + setup_chroot "$target" + + einfo "Installing core packages" + install_core_packages "$target" + + einfo "Configuring and enabling boot loader" + create_initfs "$target" + setup_extlinux "$target" + install_extlinux "$target" + + einfo "Configuring system" + setup_fstab "$target" + setup_networking "$target" + enable_services "$target" + create_alpine_user "$target" + configure_ntp "$target" + + einfo "All done, cleaning up" + cleanup "$target" +} + +main "$@" diff --git a/release.yaml b/release.yaml new file mode 100644 index 0000000..66d80c7 --- /dev/null +++ b/release.yaml @@ -0,0 +1,21 @@ +Alpine-3.7-Hardened-EC2: + description: "Alpine Linux 3.7 Release with Hardened Kernel and EC2 Optimizations" + alpine-release: 3.7 + kernel-flavor: hardened + ami-release-date: "2017-12-25 03:02:00" + region-identifiers: + #us-east-1: ami-XXXXXXXX + #us-east-2: ami-XXXXXXXX + #us-west-1: ami-XXXXXXXX + us-west-2: ami-032b877b + #ca-central-1: ami-XXXXXXXX + #eu-central-1: ami-XXXXXXXX + #eu-west-1: ami-XXXXXXXX + #eu-west-2: ami-XXXXXXXX + #eu-west-3: ami-XXXXXXXX + #ap-northeast-1: ami-XXXXXXXX + #ap-northeast-2: ami-XXXXXXXX + #ap-southeast-1: ami-XXXXXXXX + #ap-southeast-2: ami-XXXXXXXX + #ap-south-1: ami-XXXXXXXX + #sa-east-1: ami-XXXXXXXX