Improve Configurability

* move config variables from alpine-ami.yaml to variables.json-*
  + variables.json-default - ready-for-action original default config
  + variables.json-example - original defaults with comments
* clean up tabs vs. spaces in make_ami.sh
* make_ami.sh handles custom kernel flavor, extra repos, and extra packages
* tweak README with regards to aws-ena-driver caveat

README.md

@@ -50,9 +50,10 @@ its development and thus there are some sharp edges.
   hardware so it seems unlikely that they will be supported going forward. Thus
   this project does not support them.
 
-- The aws-ena-driver-vanilla package is still in edge/testing.  When it is
+- The aws-ena-driver-vanilla package is still in edge/testing, and requires the
-  available in a release, the edge/testing repository can be removed from
+  matching linux-vanilla package from edge/main.  When ENA is available in an
-  /etc/apk/repositories.
+  alpine version release, edge/testing and edge/main should no longer be
+  necessary.
 
 - [cloud-init](https://cloudinit.readthedocs.io/en/latest/) is not currently
   supported on Alpine Linux. Instead this image uses

alpine-ami.yaml

@@ -1,74 +1,32 @@
 variables:
-  security_group: ""
-  subnet: ""
-  public_ip: "false"
 
-  # Treat this similar to a ABUILD pkgrel variable and increment with every
+  # NOTE:  Additional configuration is set via the `variables.json` file.
-  # release. Packer will notice an exiting AMI at build start and fail unless
+  # To use default values, simply `cp variables.json-default variables.json`.
-  # it is rmoved. To prevent a period of time where no Alpine AMI exists,
+  # See `variables.json-example` for full configuration variable descriptions.
-  # create a new variant. Old AMIs should be pruned at some point.
-  ami_release: "0"
 
-  # Overriding this requires validating that the installation script still
+  # NOTE:  Changing alpine_release requires modifying `make_ami.sh` -- don't
-  # works as expected. It probably does but stuff changes between major
+  # override this in `variables.json`!
-  # version.
   alpine_release: "3.8"
 
-  # Don't override this without a good reason and if you do just make sure it
-  # gets passed all the way through to the make_ami script
-  volume_name: "/dev/xvdf"
 
 builders:
   - type: "amazon-ebssurrogate"
 
-    # Image is built inside a custom VPC so let Packer use the existing
+    ### Builder Instance Details
-    # resources
-    security_group_id: "{{user `security_group`}}"
-    subnet_id: "{{user `subnet`}}"
 
-    # Input Instance Setting
+    vpc_id: "{{user `vpc`}}"
-    instance_type: "t2.nano"
+    subnet_id: "{{user `subnet`}}"
+    security_group_id: "{{user `security_group`}}"
+    instance_type: "{{user `build_instance_type`}}"
+    associate_public_ip_address: "{{user `public_ip`}}"
     launch_block_device_mappings:
       - volume_type: "gp2"
         device_name: "{{user `volume_name`}}"
-        delete_on_termination: true
+        delete_on_termination: "true"
-        volume_size: 1
+        volume_size: "{{user `volume_size`}}"
-    associate_public_ip_address: "{{user `public_ip`}}"
-
-    # Output AMI Settings
-    ena_support: true
-    ami_name: "Alpine-{{user `alpine_release`}}-r{{user `ami_release`}}-EC2"
-    ami_description: "Alpine Linux {{user `alpine_release`}}-r{{user `ami_release`}} Release with EC2 Optimizations"
-    ami_groups:
-      - "all"
-    ami_virtualization_type: "hvm"
-    ami_regions:
-      - us-east-1
-      - us-east-2
-      - us-west-1
-      - us-west-2
-      - ca-central-1
-      - eu-central-1
-      - eu-west-1
-      - eu-west-2
-      - eu-west-3
-      - ap-northeast-1
-      - ap-northeast-2
-#      - ap-northeast-3
-      - ap-southeast-1
-      - ap-southeast-2
-      - ap-south-1
-      - sa-east-1
-    ami_root_device:
-      source_device_name: "{{user `volume_name`}}"
-      device_name: "/dev/xvda"
-      delete_on_termination: true
-      volume_size: 1
-      volume_type: "gp2"
-
-    # Use the most recent Amazon Linux AMI as our base
     ssh_username: "ec2-user"
     source_ami_filter:
+      # use the latest Amazon Linux AMI
       filters:
         virtualization-type: "hvm"
         root-device-type: "ebs"
@@ -76,9 +34,26 @@ builders:
         name: "amzn-ami-hvm-*-x86_64-gp2"
       owners:
         - "137112412989"
-      most_recent: true
+      most_recent: "true"
+
+    ### Built AMI Details
+
+    ami_name: "{{user `ami_name_prefix`}}{{user `alpine_release`}}-r{{user `ami_release`}}{{user `ami_name_suffix`}}"
+    ami_description: "{{user `ami_desc_prefix`}}{{user `alpine_release`}}-r{{user `ami_release`}}{{user `ami_desc_suffix`}}"
+    ami_virtualization_type: "hvm"
+    ami_root_device:
+      source_device_name: "{{user `volume_name`}}"
+      device_name: "/dev/xvda"
+      delete_on_termination: "true"
+      volume_size: "{{user `volume_size`}}"
+      volume_type: "gp2"
+    ena_support: "{{user `ena_enable`}}"
+    sriov_support: "{{user `sriov_enable`}}"
+    ami_groups: "{{user `ami_access`}}"
+    ami_regions: "{{user `deploy_regions`}}"
+
 
 provisioners:
   - type: "shell"
     script: "make_ami.sh"
-    execute_command: "sudo sh -c '{{ .Vars }} {{ .Path }} {{user `volume_name`}}'"
+    execute_command: 'sudo sh -c "{{ .Vars }} {{ .Path }} {{user `volume_name`}} {{user `kernel_flavor`}} ''{{user `add_repos`}}'' ''{{user `add_pkgs`}}''"'

make_ami.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-# vim:set ts=4:
+# vim: set ts=4 noet:
 
 set -eu
 
@@ -72,14 +72,14 @@ make_filesystem() {
 
 setup_repositories() {
 	local target="$1"     # target directory
+	local add_repos="$2"  # extra repo lines, comma separated
 
-  # NOTE: we only need @testing for aws-ena-driver-vanilla, this can be removed if/when released
 	mkdir -p "$target"/etc/apk/keys
 	cat > "$target"/etc/apk/repositories <<-EOF
 	http://dl-cdn.alpinelinux.org/alpine/v$ALPINE_RELEASE/main
 	http://dl-cdn.alpinelinux.org/alpine/v$ALPINE_RELEASE/community
-	@testing http://dl-cdn.alpinelinux.org/alpine/edge/testing
 	EOF
+	echo "$add_repos" | tr , "\012" >> "$target"/etc/apk/repositories
 }
 
 fetch_keys() {
@@ -104,29 +104,28 @@ setup_chroot() {
 }
 
 install_core_packages() {
-	local target="$1"
+	local target="$1"    # target directory
+	local flavor="$2"    # kernel flavor
+	local add_pkgs="$3"  # extra packages, space separated
 
 	# Most from: https://git.alpinelinux.org/cgit/alpine-iso/tree/alpine-virt.packages
 	#
-	# acct - installed by some configurations, so added here
+	# linux-$flavor - linux kernel flavor to install
-	# aws-ena-driver-vanilla - required for ENA enabled instances (still in edge/testing)
 	# e2fsprogs - required by init scripts to maintain ext4 volumes
-	# linux-vanilla - can't use virt because it's missing NVME support
 	# mkinitfs - required to build custom initfs
 	# sudo - to allow alpine user to become root, disallow root SSH logins
 	# tiny-ec2-bootstrap - to bootstrap system from EC2 metadata
 	chroot "$target" apk --no-cache add \
-		acct \
+		linux-"$flavor" \
 		alpine-mirrors \
-		aws-ena-driver-vanilla@testing \
 		chrony \
 		e2fsprogs \
-		linux-vanilla \
 		mkinitfs \
 		openssh \
 		sudo \
 		tiny-ec2-bootstrap \
-		tzdata
+		tzdata \
+		$add_pkgs
 
 	chroot "$target" apk --no-cache add --no-scripts syslinux
 
@@ -267,9 +266,13 @@ cleanup() {
 }
 
 main() {
-	[ "$#" -ne 1 ] && { echo "usage: $0 <block-device>"; exit 1; }
+	[ "$#" -ne 4 ] && { echo "usage: $0 <block-device> <kernel-flavor> '<repo>[,<repo>]' '<pkg>[ <pkg>]'"; exit 1; }
 
 	device="$1"
+	flavor="$2"
+	add_repos="$3"
+	add_pkgs="$4"
+
 	target="/mnt/target"
 
 	validate_block_device "$device"
@@ -282,7 +285,7 @@ main() {
 	einfo "Creating root filesystem"
 	make_filesystem "$device" "$target"
 
-	setup_repositories "$target"
+	setup_repositories "$target" "$add_repos"
 
 	einfo "Fetching Alpine signing keys"
 	fetch_keys "$target"
@@ -293,7 +296,7 @@ main() {
 	setup_chroot "$target"
 
 	einfo "Installing core packages"
-	install_core_packages "$target"
+	install_core_packages "$target" "$flavor" "$add_pkgs"
 
 	einfo "Configuring and enabling boot loader"
 	create_initfs "$target"

variables.json-default

@@ -0,0 +1,22 @@
+{
+  "ami_release": "1",
+  "ami_name_prefix": "Alpine-",
+  "ami_name_suffix": "-EC2",
+  "ami_desc_prefix": "Alpine Linux ",
+  "ami_desc_suffix": " Release with EC2 Optimizations",
+  "kernel_flavor": "vanilla@edge-main",
+  "add_repos": "@edge-main http://dl-cdn.alpinelinux.org/alpine/edge/main,@edge-testing http://dl-cdn.alpinelinux.org/alpine/edge/testing",
+  "add_pkgs": "acct aws-ena-driver-vanilla@edge-testing",
+  "ena_enable": "true",
+  "sriov_enable": "false",
+  "volume_size": "1",
+  "ami_access": "all",
+  "deploy_regions": "us-east-1,us-east-2,us-west-1,us-west-2,ca-central-1,eu-central-1,eu-west-1,eu-west-2,eu-west-3,ap-northeast-1,ap-northeast-2,ap-southeast-1,ap-southeast-2,ap-south-1,sa-east-1",
+
+  "vpc": "",
+  "subnet": "",
+  "security_group": "",
+  "public_ip": "false",
+  "build_instance_type": "t2.nano",
+  "volume_name": "/dev/xvdf"
+}

variables.json-example

@@ -0,0 +1,72 @@
+# NOTE: This is file not valid JSON.
+{
+  ### Build Options ###
+
+  # Treat similar to a ABUILD pkgrel variable and increment with every release.
+  "ami_release": "1",
+
+  # AMI name prefix and suffix
+  "ami_name_prefix": "Alpine-",
+  "ami_name_suffix": "-EC2",
+
+  # AMI description prefix and suffix
+  "ami_desc_prefix": "Alpine Linux ",
+  "ami_desc_suffix": " Release with EC2 Optimizations",
+
+  # Kernel "flavor" to install.  'virt' is a slim choice, but doesn't currently
+  # include NVME support and there is no matching 'aws-ena-driver' package.
+  # 'vanilla' installs a lot of unneeded stuff (for an AMI), but does support
+  # NVME; however, there is no matching ENA driver in the main repo.  In order
+  # to support NVME and ENA, we need to use 'vanilla@edge-main', which matches
+  # the 'aws-ena-driver@edge-testing' package.
+  "kernel_flavor": "vanilla@edge-main",
+
+  # Comma separated list of lines to add to /etc/apk/repositories.  We need
+  # edge/main and edge/testing for simultaneous NVME and ENA support.
+  "add_repos": "@edge-main http://dl-cdn.alpinelinux.org/alpine/edge/main,@edge-testing http://dl-cdn.alpinelinux.org/alpine/edge/testing",
+
+  # Space separated list of additional packages to add to the AMI.
+  # acct - system accounting utilities (sa, etc.)
+  # aws-ena-driver-vanilla - Enhanced Network Adapter kernel module
+  "add_pkgs": "acct aws-ena-driver-vanilla@edge-testing",
+
+  # Enable ENA/SRIOV support on the AMI.
+  "ena_enable": "true",
+  "sriov_enable": "false",
+
+  # Size of the AMI image (in GiB).
+  "volume_size": "1",
+
+  # Comma separated list of groups that should have access to the AMI.  However,
+  # only two values are currently supported: 'all' for public, '' for private.
+  "ami_access": "all",
+
+  # Comma separated list of regions to where the AMI should be copied.
+  # NOTE: ap-northeast-3 skipped, as it is available by subscription-only.
+  "deploy_regions": "us-east-1,us-east-2,us-west-1,us-west-2,ca-central-1,eu-central-1,eu-west-1,eu-west-2,eu-west-3,ap-northeast-1,ap-northeast-2,ap-southeast-1,ap-southeast-2,ap-south-1,sa-east-1",
+
+
+  ### Builder-Instance Options ###
+
+  # VPC in which the builder instance is to be launched; you must also provide
+  # a subnet.
+  "vpc": "",
+
+  # Subnet in which the builder instance is to be launched.
+  "subnet": "",
+
+  # Security group to apply to the builder instance.
+  "security_group": "",
+
+  # Assign a public IP to the builder instance.  Set to 'true' for if you need
+  # to initiate the build from somewhere that wouldn't normally be able to
+  # access the builder instance's private network.
+  "public_ip": "false",
+
+  # Instance type to use for building.
+  "build_instance_type": "t2.nano",
+
+  # Don't override this without a good reason, and if you do just make sure it
+  # gets passed all the way through to the make_ami script.
+  "volume_name": "/dev/xvdf"
+}