diff --git a/.gitignore b/.gitignore index 00c9ff3..ca3d4df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ /build/ /.py3/ -/variables.json +/variables.yaml /scrub-old-amis.py /gen-readme.py diff --git a/Makefile b/Makefile index 04ec2fb..d2505b2 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ .PHONY: ami ami: build/convert + [ -f variables.yaml ] || cp variables.yaml-default variables.yaml + build/convert variables.yaml > build/variables.json build/convert alpine-ami.yaml > build/alpine-ami.json - packer build -var-file=variables.json build/alpine-ami.json + packer build -var-file=build/variables.json build/alpine-ami.json build/convert: [ -d ".py3" ] || python3 -m venv .py3 @@ -14,7 +16,11 @@ build/convert: # which only supports JSON @echo "#!`pwd`/.py3/bin/python" > build/convert @echo "import yaml, json, sys" >> build/convert - @echo "json.dump(yaml.load(open(sys.argv[1])), sys.stdout, indent=4, separators=(',', ': '))" >> build/convert + @echo "y = yaml.load(open(sys.argv[1]))" >> build/convert + @echo "for k in ['ami_access','deploy_regions','add_repos','add_pkgs']:" >> build/convert + @echo " if k in y and isinstance(y[k], list):" >> build/convert + @echo " y[k] = ','.join(str(x) for x in y[k])" >> build/convert + @echo "json.dump(y, sys.stdout, indent=4, separators=(',', ': '))" >> build/convert @chmod +x build/convert %.py: %.py.in diff --git a/README.md b/README.md index 10a1322..a27c2f3 100644 --- a/README.md +++ b/README.md @@ -50,16 +50,6 @@ its development and thus there are some sharp edges. hardware so it seems unlikely that they will be supported going forward. Thus this project does not support them. -- The linux-vanilla kernel all the linux-firmware packages it installs is much - larger than is necessary for an AMI designed to run on EC2. Unfortunately, - the linux-virt kernel is currently missing NVMe support, which is required for - the newest generation of instance families. - -- The aws-ena-driver-vanilla package is still in edge/testing, and requires the - matching linux-vanilla package from edge/main. When ENA is available in an - alpine version release (ideally with a 'virt' kernel flavor), edge/testing - and edge/main should no longer be necessary. - - [cloud-init](https://cloudinit.readthedocs.io/en/latest/) is not currently supported on Alpine Linux. Instead this image uses [tiny-ec2-bootstrap](https://github.com/mcrute/tiny-ec2-bootstrap). Hostname @@ -71,6 +61,13 @@ its development and thus there are some sharp edges. If full cloud-init support is important to you please file a bug against this project. +- Because several key packages in Alpine 3.8 are missing or lacking features, + we currently need to install some packages from edge. We expect that these + will be included in Alpine 3.9, or perhaps as a 3.8.x update. + - linux-virt @edge-main (includes necessary NVMe drivers) + - aws-ena-driver @edge-testing (installs 'virt' flavored subpackage) + - tiny-ec2-bootstrap @edge-main (updated to v1.2.0) + - CloudFormation support is still forthcoming. This requires patches and packaging for the upstream cfn tools that have not yet been accepted. Eventually full CloudFormation support will be available. diff --git a/alpine-ami.yaml b/alpine-ami.yaml index 09e0c4d..0ec2e21 100644 --- a/alpine-ami.yaml +++ b/alpine-ami.yaml @@ -1,8 +1,7 @@ variables: - # NOTE: Configuration is done with a `variables.json` file. - # To use default values, simply `cp variables.json-default variables.json`. - # See `variables.json-example` for full configuration variable descriptions. + # NOTE: Configuration is done with a `variables.yaml` file. If it doesn't + # exist, default configuration is copied from `variables.yaml-default`. # NOTE: Changing alpine_release requires modifying `make_ami.sh` -- don't # override this in `variables.json`! @@ -14,7 +13,6 @@ builders: ### Builder Instance Details - vpc_id: "{{user `vpc`}}" subnet_id: "{{user `subnet`}}" security_group_id: "{{user `security_group`}}" instance_type: "t3.nano" @@ -36,7 +34,7 @@ builders: - "137112412989" most_recent: "true" - ### Built AMI Details + ### AMI Build Details ami_name: "{{user `ami_name_prefix`}}{{user `alpine_release`}}-r{{user `ami_release`}}{{user `ami_name_suffix`}}" ami_description: "{{user `ami_desc_prefix`}}{{user `alpine_release`}}-r{{user `ami_release`}}{{user `ami_desc_suffix`}}" @@ -48,7 +46,7 @@ builders: volume_size: "{{user `volume_size`}}" volume_type: "gp2" encrypt_boot: "{{user `encrypt_ami`}}" - ena_support: "{{user `ena_enable`}}" + ena_support: "true" sriov_support: "true" ami_groups: "{{user `ami_access`}}" ami_regions: "{{user `deploy_regions`}}" @@ -57,4 +55,4 @@ builders: provisioners: - type: "shell" script: "make_ami.sh" - execute_command: 'sudo sh -c "{{ .Vars }} {{ .Path }} {{user `kernel_flavor`}} ''{{user `add_repos`}}'' ''{{user `add_pkgs`}}''"' + execute_command: 'sudo sh -c "{{ .Vars }} {{ .Path }} ''{{user `add_repos`}}'' ''{{user `add_pkgs`}}''"' diff --git a/make_ami.sh b/make_ami.sh index 6218694..0d10058 100755 --- a/make_ami.sh +++ b/make_ami.sh @@ -108,8 +108,7 @@ setup_chroot() { install_core_packages() { local target="$1" # target directory - local flavor="$2" # kernel flavor - local add_pkgs="$3" # extra packages, space separated + local add_pkgs="$2" # extra packages, space separated # Most from: https://git.alpinelinux.org/cgit/alpine-iso/tree/alpine-virt.packages # @@ -117,15 +116,15 @@ install_core_packages() { # tiny-ec2-bootstrap - to bootstrap system from EC2 metadata # chroot "$target" apk --no-cache add \ - linux-"$flavor" \ + linux-virt@edge-main \ + aws-ena-driver@edge-testing \ alpine-mirrors \ chrony \ - e2fsprogs \ openssh \ sudo \ - tiny-ec2-bootstrap \ + tiny-ec2-bootstrap@edge-main \ tzdata \ - $add_pkgs + $(echo "$add_pkgs" | tr , ' ') chroot "$target" apk --no-cache add --no-scripts syslinux @@ -244,7 +243,9 @@ configure_ntp() { # in EC2. # # See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html - sed -i 's/^pool .*/server 169.254.169.123 iburst/' "$target"/etc/chrony/chrony.conf + sed -e 's/^pool /server /' \ + -e 's/pool.ntp.org/169.254.169.123/g' \ + -i "$target"/etc/chrony/chrony.conf } cleanup() { @@ -267,11 +268,10 @@ cleanup() { } main() { - [ "$#" -ne 3 ] && { echo "usage: $0 '[,]' '[ ]'"; exit 1; } + [ "$#" -ne 2 ] && { echo "usage: $0 '[,]' '[,]'"; exit 1; } - local flavor="$1" - local add_repos="$2" - local add_pkgs="$3" + local add_repos="$1" + local add_pkgs="$2" local device="/dev/xvdf" local target="/mnt/target" @@ -297,7 +297,7 @@ main() { setup_chroot "$target" einfo "Installing core packages" - install_core_packages "$target" "$flavor" "$add_pkgs" + install_core_packages "$target" "$add_pkgs" einfo "Configuring and enabling boot loader" create_initfs "$target" diff --git a/variables.json-default b/variables.json-default deleted file mode 100644 index d45378f..0000000 --- a/variables.json-default +++ /dev/null @@ -1,20 +0,0 @@ -{ - "ami_release": "1", - "ami_name_prefix": "Alpine-", - "ami_name_suffix": "-EC2", - "ami_desc_prefix": "Alpine Linux ", - "ami_desc_suffix": " Release with EC2 Optimizations", - "kernel_flavor": "vanilla@edge-main", - "add_repos": "", - "add_pkgs": "aws-ena-driver-vanilla@edge-testing", - "ena_enable": "true", - "volume_size": "1", - "encrypt_ami": "false", - "ami_access": "all", - "deploy_regions": "us-east-1,us-east-2,us-west-1,us-west-2,ca-central-1,eu-central-1,eu-west-1,eu-west-2,eu-west-3,ap-northeast-1,ap-northeast-2,ap-southeast-1,ap-southeast-2,ap-south-1,sa-east-1", - - "vpc": "", - "subnet": "", - "security_group": "", - "public_ip": "false" -} diff --git a/variables.json-example b/variables.json-example deleted file mode 100644 index 0e10c9a..0000000 --- a/variables.json-example +++ /dev/null @@ -1,74 +0,0 @@ -# *** NOTE: This is file not valid JSON! *** - -{ - ### Build Options ### - - # Treat similar to a ABUILD pkgrel variable and increment with every release. - "ami_release": "1", - - # AMI name prefix and suffix - "ami_name_prefix": "Alpine-", - "ami_name_suffix": "-EC2", - - # AMI description prefix and suffix - "ami_desc_prefix": "Alpine Linux ", - "ami_desc_suffix": " Release with EC2 Optimizations", - - # Kernel "flavor" to install. - # - # 'virt' is the slim choice, but doesn't currently include NVMe support and - # there is no matching 'aws-ena-driver' package. When these features are - # available, this kernel flavor will be the default (if not hardcoded). - # - # 'vanilla' installs a lot of unneeded stuff (for an AMI), but does support - # NVMe; however, there is no matching ENA driver in the main repo. In order - # to support NVMe and ENA, we need to use 'vanilla@edge-main', which matches - # the 'aws-ena-driver@edge-testing' package. - # - "kernel_flavor": "vanilla@edge-main", - - # Comma separated list of custom lines to add to /etc/apk/repositories. - # @edge-main, @edge-community, and @edge-testing repos have been predefined. - "add_repos": "", - - # Space separated list of additional packages to add to the AMI. - # aws-ena-driver-vanilla - ENA driver (until we have a 'virt' flavor) - "add_pkgs": "aws-ena-driver-vanilla@edge-testing", - - # Enable ENA support on the AMI. - # When ENA is available for the 'virt' kernel, this will always be on. - "ena_enable": "true", - - # Size of the AMI image (in GiB). - "volume_size": "1", - - # Encrypt the AMI? - "encrypt_ami": "false", - - # Comma separated list of groups that should have access to the AMI. However, - # only two values are currently supported: 'all' for public, '' for private. - "ami_access": "all", - - # Comma separated list of regions to where the AMI should be copied. - # NOTE: ap-northeast-3 skipped, as it is available by subscription-only. - "deploy_regions": "us-east-1,us-east-2,us-west-1,us-west-2,ca-central-1,eu-central-1,eu-west-1,eu-west-2,eu-west-3,ap-northeast-1,ap-northeast-2,ap-southeast-1,ap-southeast-2,ap-south-1,sa-east-1", - - - ### Builder-Instance Options ### - - # VPC in which the builder instance is to be launched; you must also provide - # a subnet. - "vpc": "", - - # Subnet in which the builder instance is to be launched. - "subnet": "", - - # Security group to apply to the builder instance. - "security_group": "", - - # Assign a public IP to the builder instance. Set to 'true' for if you need - # to initiate the build from somewhere that wouldn't normally be able to - # access the builder instance's private network. - "public_ip": "false" - -} diff --git a/variables.yaml-default b/variables.yaml-default new file mode 100644 index 0000000..3ec54f4 --- /dev/null +++ b/variables.yaml-default @@ -0,0 +1,64 @@ +### Builder-Instance Options ### + +# Subnet ID in which the builder instance is to be launched. VPC will be +# automatically determined. +subnet: + +# Optional security group to apply to the builder instance. +security_group: + +# Assign a public IP to the builder instance. Set to "true" for if you need +# to initiate the build from somewhere that wouldn't normally be able to access +# the builder instance's private network and a private IP is not auto-assigned. +public_ip: "false" + + +### Build Options ### + +# Treat similar to a ABUILD pkgrel variable and increment with every release. +ami_release: "1" + +# AMI name prefix and suffix +ami_name_prefix: "Alpine-" +ami_name_suffix: "-EC2" + +# AMI description prefix and suffix +ami_desc_prefix: "Alpine Linux " +ami_desc_suffix: " Release with EC2 Optimizations" + +# List of custom lines to add to /etc/apk/repositories. Note that @edge-main, +# @edge-community, and @edge-testing repos have been predefined. +add_repos: + +# List of additional packages to add to the AMI. +add_pkgs: + +# Size of the AMI image (in GiB). +volume_size: "1" + +# Encrypt the AMI? +encrypt_ami: "false" + +# List of groups that should have access to the AMI. However, only two +# values are currently supported: 'all' for public, '' or unset for private. +ami_access: + - "all" + +# List of regions to where the AMI should be copied. +deploy_regions: + - "us-east-1" + - "us-east-2" + - "us-west-1" + - "us-west-2" + - "ca-central-1" + - "eu-central-1" + - "eu-west-1" + - "eu-west-2" + - "eu-west-3" + - "ap-northeast-1" + - "ap-northeast-2" +# - "ap-northeast-3" # skipped, available by subscription only + - "ap-southeast-1" + - "ap-southeast-2" + - "ap-south-1" + - "sa-east-1"