Improve Configurability

* move config variables from alpine-ami.yaml to variables.json-*
  + variables.json-default - ready-for-action original default config
  + variables.json-example - original defaults with comments
* clean up tabs vs. spaces in make_ami.sh
* make_ami.sh handles custom kernel flavor, extra repos, and extra packages
* tweak README with regards to aws-ena-driver caveat

README.md

@@ -50,9 +50,10 @@ its development and thus there are some sharp edges.
   hardware so it seems unlikely that they will be supported going forward. Thus
   this project does not support them.
 
-- The aws-ena-driver-vanilla package is still in edge/testing.  When it is
+- The aws-ena-driver-vanilla package is still in edge/testing, and requires the
-  available in a release, the edge/testing repository can be removed from
+  matching linux-vanilla package from edge/main.  When ENA is available in an
-  /etc/apk/repositories.
+  alpine version release, edge/testing and edge/main should no longer be
+  necessary.
 
 - [cloud-init](https://cloudinit.readthedocs.io/en/latest/) is not currently
   supported on Alpine Linux. Instead this image uses

alpine-ami.yaml

@@ -1,84 +1,59 @@
 variables:
-  security_group: ""
-  subnet: ""
-  public_ip: "false"
 
-  # Treat this similar to a ABUILD pkgrel variable and increment with every
+  # NOTE:  Additional configuration is set via the `variables.json` file.
-  # release. Packer will notice an exiting AMI at build start and fail unless
+  # To use default values, simply `cp variables.json-default variables.json`.
-  # it is rmoved. To prevent a period of time where no Alpine AMI exists,
+  # See `variables.json-example` for full configuration variable descriptions.
-  # create a new variant. Old AMIs should be pruned at some point.
-  ami_release: "0"
 
-  # Overriding this requires validating that the installation script still
+  # NOTE:  Changing alpine_release requires modifying `make_ami.sh` -- don't
-  # works as expected. It probably does but stuff changes between major
+  # override this in `variables.json`!
-  # version.
   alpine_release: "3.8"
 
-  # Don't override this without a good reason and if you do just make sure it
-  # gets passed all the way through to the make_ami script
-  volume_name: "/dev/xvdf"
 
 builders:
   - type: "amazon-ebssurrogate"
 
-    # Image is built inside a custom VPC so let Packer use the existing
+    ### Builder Instance Details
-    # resources
-    security_group_id: "{{user `security_group`}}"
-    subnet_id: "{{user `subnet`}}"
 
-    # Input Instance Setting
+    vpc_id: "{{user `vpc`}}"
-    instance_type: "t2.nano"
+    subnet_id: "{{user `subnet`}}"
+    security_group_id: "{{user `security_group`}}"
+    instance_type: "{{user `build_instance_type`}}"
+    associate_public_ip_address: "{{user `public_ip`}}"
     launch_block_device_mappings:
       - volume_type: "gp2"
         device_name: "{{user `volume_name`}}"
-        delete_on_termination: true
+        delete_on_termination: "true"
-        volume_size: 1
+        volume_size: "{{user `volume_size`}}"
-    associate_public_ip_address: "{{user `public_ip`}}"
-
-    # Output AMI Settings
-    ena_support: true
-    ami_name: "Alpine-{{user `alpine_release`}}-r{{user `ami_release`}}-EC2"
-    ami_description: "Alpine Linux {{user `alpine_release`}}-r{{user `ami_release`}} Release with EC2 Optimizations"
-    ami_groups:
-      - "all"
-    ami_virtualization_type: "hvm"
-    ami_regions:
-      - us-east-1
-      - us-east-2
-      - us-west-1
-      - us-west-2
-      - ca-central-1
-      - eu-central-1
-      - eu-west-1
-      - eu-west-2
-      - eu-west-3
-      - ap-northeast-1
-      - ap-northeast-2
-#      - ap-northeast-3
-      - ap-southeast-1
-      - ap-southeast-2
-      - ap-south-1
-      - sa-east-1
-    ami_root_device:
-      source_device_name: "{{user `volume_name`}}"
-      device_name: "/dev/xvda"
-      delete_on_termination: true
-      volume_size: 1
-      volume_type: "gp2"
-
-    # Use the most recent Amazon Linux AMI as our base
     ssh_username: "ec2-user"
     source_ami_filter:
+      # use the latest Amazon Linux AMI
       filters:
         virtualization-type: "hvm"
         root-device-type: "ebs"
         architecture: "x86_64"
         name: "amzn-ami-hvm-*-x86_64-gp2"
-      owners: 
+      owners:
         - "137112412989"
-      most_recent: true
+      most_recent: "true"
+
+    ### Built AMI Details
+
+    ami_name: "{{user `ami_name_prefix`}}{{user `alpine_release`}}-r{{user `ami_release`}}{{user `ami_name_suffix`}}"
+    ami_description: "{{user `ami_desc_prefix`}}{{user `alpine_release`}}-r{{user `ami_release`}}{{user `ami_desc_suffix`}}"
+    ami_virtualization_type: "hvm"
+    ami_root_device:
+      source_device_name: "{{user `volume_name`}}"
+      device_name: "/dev/xvda"
+      delete_on_termination: "true"
+      volume_size: "{{user `volume_size`}}"
+      volume_type: "gp2"
+    ena_support: "{{user `ena_enable`}}"
+    sriov_support: "{{user `sriov_enable`}}"
+    ami_groups: "{{user `ami_access`}}"
+    ami_regions: "{{user `deploy_regions`}}"
+
 
 provisioners:
   - type: "shell"
     script: "make_ami.sh"
-    execute_command: "sudo sh -c '{{ .Vars }} {{ .Path }} {{user `volume_name`}}'"
+    execute_command: 'sudo sh -c "{{ .Vars }} {{ .Path }} {{user `volume_name`}} {{user `kernel_flavor`}} ''{{user `add_repos`}}'' ''{{user `add_pkgs`}}''"'

make_ami.sh

@@ -1,9 +1,9 @@
 #!/bin/sh
-# vim:set ts=4:
+# vim: set ts=4 noet:
 
 set -eu
 
-: ${ALPINE_RELEASE:="3.8"} # not tested against edge
+: ${ALPINE_RELEASE:="3.8"}  # not tested against edge
 : ${APK_TOOLS_URI:="https://github.com/alpinelinux/apk-tools/releases/download/v2.10.0/apk-tools-2.10.0-x86_64-linux.tar.gz"}
 : ${APK_TOOLS_SHA256:="77f2d256fcd5d6fdafadf43bb6a9c85c3da7bb471ee842dcd729175235cb9fed"}
 : ${ALPINE_KEYS:="http://dl-cdn.alpinelinux.org/alpine/v3.8/main/x86_64/alpine-keys-2.1-r1.apk"}
@@ -19,9 +19,9 @@ einfo() {
 }
 
 rc_add() {
-	local target="$1"; shift # target directory
+	local target="$1"; shift    # target directory
 	local runlevel="$1"; shift  # runlevel name
-	local services="$*"  # names of services
+	local services="$*"         # names of services
 
 	local svc; for svc in $services; do
 		mkdir -p "$target"/etc/runlevels/$runlevel
@@ -31,9 +31,9 @@ rc_add() {
 }
 
 wgets() (
-	local url="$1" # url to fetch
+	local url="$1"     # url to fetch
-	local sha256="$2" # expected SHA256 sum of output
+	local sha256="$2"  # expected SHA256 sum of output
-	local dest="$3" # output path and filename
+	local dest="$3"    # output path and filename
 
 	wget -T 10 -q -O "$dest" "$url"
 	echo "$sha256  $dest" | sha256sum -c > /dev/null
@@ -41,7 +41,7 @@ wgets() (
 
 
 validate_block_device() {
-	local dev="$1" # target directory
+	local dev="$1"  # target directory
 
 	lsblk -P --fs "$dev" >/dev/null 2>&1 || \
 		die "'$dev' is not a valid block device"
@@ -62,8 +62,8 @@ fetch_apk_tools() {
 }
 
 make_filesystem() {
-	local device="$1" # target device path
+	local device="$1"  # target device path
-	local target="$2" # mount target
+	local target="$2"  # mount target
 
 	mkfs.ext4 "$device"
 	e2label "$device" /
@@ -71,15 +71,15 @@ make_filesystem() {
 }
 
 setup_repositories() {
-	local target="$1" # target directory
+	local target="$1"     # target directory
+	local add_repos="$2"  # extra repo lines, comma separated
 
-  # NOTE: we only need @testing for aws-ena-driver-vanilla, this can be removed if/when released
 	mkdir -p "$target"/etc/apk/keys
 	cat > "$target"/etc/apk/repositories <<-EOF
 	http://dl-cdn.alpinelinux.org/alpine/v$ALPINE_RELEASE/main
 	http://dl-cdn.alpinelinux.org/alpine/v$ALPINE_RELEASE/community
-	@testing http://dl-cdn.alpinelinux.org/alpine/edge/testing
 	EOF
+	echo "$add_repos" | tr , "\012" >> "$target"/etc/apk/repositories
 }
 
 fetch_keys() {
@@ -99,45 +99,44 @@ setup_chroot() {
 	mount --bind /sys "$target"/sys
 
 	# Don't want to ship this but it's needed for bootstrap. Will be removed in
-	# the cleanup stage. 
+	# the cleanup stage.
 	install -Dm644 /etc/resolv.conf "$target"/etc/resolv.conf
 }
 
 install_core_packages() {
-	local target="$1"
+	local target="$1"    # target directory
+	local flavor="$2"    # kernel flavor
+	local add_pkgs="$3"  # extra packages, space separated
 
 	# Most from: https://git.alpinelinux.org/cgit/alpine-iso/tree/alpine-virt.packages
 	#
-	# acct - installed by some configurations, so added here
+	# linux-$flavor - linux kernel flavor to install
-	# aws-ena-driver-vanilla - required for ENA enabled instances (still in edge/testing)
 	# e2fsprogs - required by init scripts to maintain ext4 volumes
-	# linux-vanilla - can't use virt because it's missing NVME support
 	# mkinitfs - required to build custom initfs
 	# sudo - to allow alpine user to become root, disallow root SSH logins
 	# tiny-ec2-bootstrap - to bootstrap system from EC2 metadata
 	chroot "$target" apk --no-cache add \
-		acct \
+		linux-"$flavor" \
 		alpine-mirrors \
-		aws-ena-driver-vanilla@testing \
 		chrony \
 		e2fsprogs \
-		linux-vanilla \
 		mkinitfs \
 		openssh \
 		sudo \
 		tiny-ec2-bootstrap \
-		tzdata
+		tzdata \
+		$add_pkgs
 
 	chroot "$target" apk --no-cache add --no-scripts syslinux
 
-    # Disable starting getty for physical ttys because they're all inaccessible
+	# Disable starting getty for physical ttys because they're all inaccessible
-    # anyhow. With this configuration boot messages will still display in the
+	# anyhow. With this configuration boot messages will still display in the
-    # EC2 console.
+	# EC2 console.
-    sed -Ei '/^tty\d/s/^/#/' /etc/inittab
+	sed -Ei '/^tty\d/s/^/#/' /etc/inittab
 
-    # Make it a little more obvious who is logged in by adding username to the
+	# Make it a little more obvious who is logged in by adding username to the
-    # prompt
+	# prompt
-    sed -i "s/^export PS1='/&\\\\u@/" /etc/profile
+	sed -i "s/^export PS1='/&\\\\u@/" /etc/profile
 }
 
 create_initfs() {
@@ -165,10 +164,10 @@ setup_extlinux() {
 	# Enable ext4 because the root device is formatted ext4
 	#
 	# Shorten timeout because EC2 has no way to interact with instance console
-    #
+	#
-    # ttyS0 is the target for EC2s "Get System Log" feature whereas tty0 is the
+	# ttyS0 is the target for EC2s "Get System Log" feature whereas tty0 is the
-    # target for EC2s "Get Instance Screenshot" feature. Enabling the serial
+	# target for EC2s "Get Instance Screenshot" feature. Enabling the serial
-    # port early in extlinux gives the most complete output in the system log.
+	# port early in extlinux gives the most complete output in the system log.
 	sed -Ei -e "s|^[# ]*(root)=.*|\1=LABEL=/|" \
 		-e "s|^[# ]*(default_kernel_opts)=.*|\1=\"console=ttyS0 console=tty0\"|" \
 		-e "s|^[# ]*(serial_port)=.*|\1=ttyS0|" \
@@ -189,8 +188,8 @@ setup_fstab() {
 	local target="$1"
 
 	cat > "$target"/etc/fstab <<-EOF
-	# <fs>		<mountpoint>	<type>	<opts>				<dump/pass>
+	# <fs>      <mountpoint>   <type>   <opts>              <dump/pass>
-	LABEL=/		/				ext4	defaults,noatime	1 1
+	LABEL=/     /              ext4     defaults,noatime    1 1
 	EOF
 }
 
@@ -244,14 +243,14 @@ configure_ntp() {
 	# in EC2.
 	#
 	# See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html
-	sed -i 's/^server .*/server 169.254.169.123/' "$target"/etc/chrony/chrony.conf 
+	sed -i 's/^server .*/server 169.254.169.123/' "$target"/etc/chrony/chrony.conf
 }
 
 cleanup() {
 	local target="$1"
 
 	# Sweep cruft out of the image that doesn't need to ship or will be
-	# re-generated when the image boots 
+	# re-generated when the image boots
 	rm -f \
 		"$target"/var/cache/apk/* \
 		"$target"/etc/resolv.conf \
@@ -263,26 +262,30 @@ cleanup() {
 		"$target"/proc \
 		"$target"/sys
 
-	umount "$target" 
+	umount "$target"
 }
 
 main() {
-	[ "$#" -ne 1 ] && { echo "usage: $0 <block-device>"; exit 1; }
+	[ "$#" -ne 4 ] && { echo "usage: $0 <block-device> <kernel-flavor> '<repo>[,<repo>]' '<pkg>[ <pkg>]'"; exit 1; }
 
 	device="$1"
+	flavor="$2"
+	add_repos="$3"
+	add_pkgs="$4"
+
 	target="/mnt/target"
 
 	validate_block_device "$device"
 
-	[ -d "$target" ] || mkdir "$target" 
+	[ -d "$target" ] || mkdir "$target"
 
 	einfo "Fetching static APK tools"
 	apk="$(fetch_apk_tools)"
 
 	einfo "Creating root filesystem"
-	make_filesystem "$device" "$target" 
+	make_filesystem "$device" "$target"
 
-	setup_repositories "$target"
+	setup_repositories "$target" "$add_repos"
 
 	einfo "Fetching Alpine signing keys"
 	fetch_keys "$target"
@@ -293,7 +296,7 @@ main() {
 	setup_chroot "$target"
 
 	einfo "Installing core packages"
-	install_core_packages "$target"
+	install_core_packages "$target" "$flavor" "$add_pkgs"
 
 	einfo "Configuring and enabling boot loader"
 	create_initfs "$target"

variables.json-default

@@ -0,0 +1,22 @@
+{
+  "ami_release": "1",
+  "ami_name_prefix": "Alpine-",
+  "ami_name_suffix": "-EC2",
+  "ami_desc_prefix": "Alpine Linux ",
+  "ami_desc_suffix": " Release with EC2 Optimizations",
+  "kernel_flavor": "vanilla@edge-main",
+  "add_repos": "@edge-main http://dl-cdn.alpinelinux.org/alpine/edge/main,@edge-testing http://dl-cdn.alpinelinux.org/alpine/edge/testing",
+  "add_pkgs": "acct aws-ena-driver-vanilla@edge-testing",
+  "ena_enable": "true",
+  "sriov_enable": "false",
+  "volume_size": "1",
+  "ami_access": "all",
+  "deploy_regions": "us-east-1,us-east-2,us-west-1,us-west-2,ca-central-1,eu-central-1,eu-west-1,eu-west-2,eu-west-3,ap-northeast-1,ap-northeast-2,ap-southeast-1,ap-southeast-2,ap-south-1,sa-east-1",
+
+  "vpc": "",
+  "subnet": "",
+  "security_group": "",
+  "public_ip": "false",
+  "build_instance_type": "t2.nano",
+  "volume_name": "/dev/xvdf"
+}

variables.json-example

@@ -0,0 +1,72 @@
+# NOTE: This is file not valid JSON.
+{
+  ### Build Options ###
+
+  # Treat similar to a ABUILD pkgrel variable and increment with every release.
+  "ami_release": "1",
+
+  # AMI name prefix and suffix
+  "ami_name_prefix": "Alpine-",
+  "ami_name_suffix": "-EC2",
+
+  # AMI description prefix and suffix
+  "ami_desc_prefix": "Alpine Linux ",
+  "ami_desc_suffix": " Release with EC2 Optimizations",
+
+  # Kernel "flavor" to install.  'virt' is a slim choice, but doesn't currently
+  # include NVME support and there is no matching 'aws-ena-driver' package.
+  # 'vanilla' installs a lot of unneeded stuff (for an AMI), but does support
+  # NVME; however, there is no matching ENA driver in the main repo.  In order
+  # to support NVME and ENA, we need to use 'vanilla@edge-main', which matches
+  # the 'aws-ena-driver@edge-testing' package.
+  "kernel_flavor": "vanilla@edge-main",
+
+  # Comma separated list of lines to add to /etc/apk/repositories.  We need
+  # edge/main and edge/testing for simultaneous NVME and ENA support.
+  "add_repos": "@edge-main http://dl-cdn.alpinelinux.org/alpine/edge/main,@edge-testing http://dl-cdn.alpinelinux.org/alpine/edge/testing",
+
+  # Space separated list of additional packages to add to the AMI.
+  # acct - system accounting utilities (sa, etc.)
+  # aws-ena-driver-vanilla - Enhanced Network Adapter kernel module
+  "add_pkgs": "acct aws-ena-driver-vanilla@edge-testing",
+
+  # Enable ENA/SRIOV support on the AMI.
+  "ena_enable": "true",
+  "sriov_enable": "false",
+
+  # Size of the AMI image (in GiB).
+  "volume_size": "1",
+
+  # Comma separated list of groups that should have access to the AMI.  However,
+  # only two values are currently supported: 'all' for public, '' for private.
+  "ami_access": "all",
+
+  # Comma separated list of regions to where the AMI should be copied.
+  # NOTE: ap-northeast-3 skipped, as it is available by subscription-only.
+  "deploy_regions": "us-east-1,us-east-2,us-west-1,us-west-2,ca-central-1,eu-central-1,eu-west-1,eu-west-2,eu-west-3,ap-northeast-1,ap-northeast-2,ap-southeast-1,ap-southeast-2,ap-south-1,sa-east-1",
+
+
+  ### Builder-Instance Options ###
+
+  # VPC in which the builder instance is to be launched; you must also provide
+  # a subnet.
+  "vpc": "",
+
+  # Subnet in which the builder instance is to be launched.
+  "subnet": "",
+
+  # Security group to apply to the builder instance.
+  "security_group": "",
+
+  # Assign a public IP to the builder instance.  Set to 'true' for if you need
+  # to initiate the build from somewhere that wouldn't normally be able to
+  # access the builder instance's private network.
+  "public_ip": "false",
+
+  # Instance type to use for building.
+  "build_instance_type": "t2.nano",
+
+  # Don't override this without a good reason, and if you do just make sure it
+  # gets passed all the way through to the make_ami script.
+  "volume_name": "/dev/xvdf"
+}