From 5e8967002e7082823ad9094d466ebf0bcb5a25e0 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sat, 12 Mar 2022 17:56:14 +0000 Subject: [PATCH 1/4] always apk upgrade builder VM, fix --clean of symlinked dirs Fix for softlinks to directories during --clean / always use latest packages within a release at build time to pick up latest security fixes etc. --- build | 2 +- scripts/setup | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build b/build index ba5b84b..f269bf9 100755 --- a/build +++ b/build @@ -95,7 +95,7 @@ def clean_work(): for x in (set(os.listdir('work')) - WORK_CLEAN): x = os.path.join('work', x) log.debug('removing %s', x) - if os.path.isdir(x): + if os.path.isdir(x) and not os.path.islink(x): shutil.rmtree(x) else: os.unlink(x) diff --git a/scripts/setup b/scripts/setup index 29aff42..8f00d29 100755 --- a/scripts/setup +++ b/scripts/setup @@ -22,6 +22,8 @@ setup_builder() { einfo "Setting up Builder Instance" setup-apkrepos -1 # main repo via dl-cdn # ODO? also uncomment community repo? + # Always use latest versions within the release, security patches etc. + apk upgrade --no-cache --available apk --no-cache add \ e2fsprogs \ dosfstools \ From 7ac20292678c8c6d8a6d6c87a4ef479cf72a0035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jake=20Buchholz=20G=C3=B6kt=C3=BCrk?= Date: Sat, 19 Mar 2022 14:24:26 -0700 Subject: [PATCH 2/4] Minor Fixes * switch 3.12 back to tiny-ec2-bootstrap (tiny-cloud requires ifupdown-ng) * restrict cloud-init to 3.15+ * pad UEFI firmware so QEMU works with aarch64 again * kinda fix motd release_notes (more to do yet) --- build | 5 ++++- configs/alpine.conf | 4 ++-- configs/bootstrap/cloudinit.conf | 3 +++ configs/bootstrap/tiny.conf | 28 +++++++++++++++++++++------- scripts/setup-tiny | 14 ++++++++++---- 5 files changed, 40 insertions(+), 14 deletions(-) diff --git a/build b/build index f269bf9..59290e3 100755 --- a/build +++ b/build @@ -185,7 +185,10 @@ def install_qemu_firmware(): log.error('STDERR:\n%s', err.decode('utf8')) sys.exit(1) - os.symlink(bin, os.path.join(firm_dir, f"uefi-{arch}.bin")) + firm_bin = os.path.join(firm_dir, f"uefi-{arch}.bin") + os.symlink(bin, firm_bin) + log.info('Padding "%s" to 67108864 bytes', firm_bin) + subprocess.run(['truncate', '-s', '67108864', firm_bin]) ### Command Line & Logging diff --git a/configs/alpine.conf b/configs/alpine.conf index 81606c6..6b44962 100644 --- a/configs/alpine.conf +++ b/configs/alpine.conf @@ -23,8 +23,8 @@ Default { "See ." version_notes = "Release Notes:\n"\ - "* " - release_notes = "* " + release_notes = "* " } # initial provisioning script and data directory diff --git a/configs/bootstrap/cloudinit.conf b/configs/bootstrap/cloudinit.conf index 19f2bd7..bd35331 100644 --- a/configs/bootstrap/cloudinit.conf +++ b/configs/bootstrap/cloudinit.conf @@ -1,6 +1,9 @@ # vim: ts=2 et: name = [cloudinit] +# start cloudinit images with 3.15 +EXCLUDE = ["3.12", "3.13", "3.14"] + packages { cloud-init = true openssh-server-pam = true diff --git a/configs/bootstrap/tiny.conf b/configs/bootstrap/tiny.conf index 018d424..351ccf8 100644 --- a/configs/bootstrap/tiny.conf +++ b/configs/bootstrap/tiny.conf @@ -1,17 +1,31 @@ # vim: ts=2 et: name = [tiny] -WHEN { - aws.packages.tiny-cloud-aws = true - # azure.packages.tiny-cloud-azure = true - # gcp.packages.tiny-cloud-gcp = true - # oci.packages.tiny-cloud-oci = true -} - services { sysinit.tiny-cloud-early = true default.tiny-cloud = true default.tiny-cloud-final = true } +WHEN { + aws { + packages.tiny-cloud-aws = true + WHEN { + "3.12" { + # tiny-cloud-network requires ifupdown-ng (unavailable in 3.12) + packages.tiny-cloud-aws = null + services.sysinit.tiny-cloud-early = null + services.default.tiny-cloud = null + services.default.tiny-cloud-final = null + # fall back to tiny-ec2-bootstrap instead + packages.tiny-ec2-bootstrap = true + services.default.tiny-ec2-bootstrap = true + } + } + } + # azure.packages.tiny-cloud-azure = true + # gcp.packages.tiny-cloud-gcp = true + # oci.packages.tiny-cloud-oci = true +} + scripts = [ setup-tiny ] \ No newline at end of file diff --git a/scripts/setup-tiny b/scripts/setup-tiny index 7081842..9fc91fc 100755 --- a/scripts/setup-tiny +++ b/scripts/setup-tiny @@ -9,7 +9,13 @@ einfo() { printf '\n\033[1;7;36m> %s <\033[0m\n' "$@" >&2 # bold reversed cyan } -einfo "Configuring Tiny Cloud..." -sed -i.bak -Ee "s/^#?CLOUD_USER=.*/CLOUD_USER=$IMAGE_LOGIN/" \ - "$TARGET"/etc/conf.d/tiny-cloud -rm "$TARGET"/etc/conf.d/tiny-cloud.bak \ No newline at end of file +if [ "$VERSION" = "3.12" ]; then + # tiny-cloud-network requires ifupdown-ng, not in 3.12 + einfo "Configuring Tiny EC2 Bootstrap..." + echo "EC2_USER=$IMAGE_LOGIN" > /etc/conf.d/tiny-ec2-bootstrap +else + einfo "Configuring Tiny Cloud..." + sed -i.bak -Ee "s/^#?CLOUD_USER=.*/CLOUD_USER=$IMAGE_LOGIN/" \ + "$TARGET"/etc/conf.d/tiny-cloud + rm "$TARGET"/etc/conf.d/tiny-cloud.bak +fi From dd0e7b280c81e92f8ae6bf406b46f51922148bff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jake=20Buchholz=20G=C3=B6kt=C3=BCrk?= Date: Sat, 19 Mar 2022 17:17:27 -0700 Subject: [PATCH 3/4] filter out non-tiny bootstrap images --- gen_releases.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gen_releases.py b/gen_releases.py index dd9cae2..707242f 100755 --- a/gen_releases.py +++ b/gen_releases.py @@ -93,6 +93,9 @@ yaml = YAML() releases = dictfactory() for i_key, i_cfg in configs.get().items(): + if i_cfg.bootstrap != 'tiny': + continue + release = i_cfg.version if i_cfg.version == 'edge' else i_cfg.release releases[release][i_key][i_cfg.tags.name] = dict(i_cfg.tags) | { 'creation_date': i_cfg.published, From faf93af63e5a58cffea5a4425033780747faa9aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jake=20Buchholz=20G=C3=B6kt=C3=BCrk?= Date: Mon, 23 May 2022 19:17:45 -0700 Subject: [PATCH 4/4] Alpine 3.16 Released alpine.pkr.hcl * fix setup-ssh in QEMU boot command build * add 'rollback' target, which when used with --revise will clean up any imported but unpublished images (use before 'build --clean configs') * fix download of UEFI firmware - ovmf apk split aarch64 to new aavmf apk configs * alpine.conf - remove 3.12, add 3.16 * add ..._name to dimensional configs * add cloud_(image|launch)_url image_configs.py * methods to resolve (image|launch)_url values clouds/aws.py * warn if setting image deprecation fails, but continue --- alpine.pkr.hcl | 2 +- build | 31 ++++++++++++++++++++++--------- clouds/aws.py | 11 +++++++---- configs/alpine.conf | 2 +- configs/arch/aarch64.conf | 1 + configs/arch/x86_64.conf | 1 + configs/bootstrap/cloudinit.conf | 4 +++- configs/bootstrap/tiny.conf | 4 +++- configs/cloud/aws.conf | 5 ++++- configs/firmware/bios.conf | 3 ++- configs/firmware/uefi.conf | 3 ++- configs/version/3.12.conf | 4 +++- configs/version/3.16.conf | 7 +++++++ image_configs.py | 6 ++++++ 14 files changed, 63 insertions(+), 21 deletions(-) create mode 100644 configs/version/3.16.conf diff --git a/alpine.pkr.hcl b/alpine.pkr.hcl index 5cd4c99..433c121 100644 --- a/alpine.pkr.hcl +++ b/alpine.pkr.hcl @@ -61,7 +61,7 @@ source qemu alpine { "root", "setup-interfaces", "ifup eth0", - "setup-sshd -c openssh", + "setup-sshd openssh", "echo PermitRootLogin yes >> /etc/ssh/sshd_config", "service sshd restart", "echo 'root:${local.password}' | chpasswd", diff --git a/build b/build index 59290e3..49bcbe0 100755 --- a/build +++ b/build @@ -48,13 +48,19 @@ from image_configs import ImageConfigManager ### Constants & Variables -STEPS = ['configs', 'state', 'local', 'import', 'publish'] +STEPS = ['configs', 'state', 'rollback', 'local', 'import', 'publish'] LOGFORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' WORK_CLEAN = {'bin', 'include', 'lib', 'pyvenv.cfg', '__pycache__'} WORK_OVERLAYS = ['configs', 'scripts'] -OVMF_FIRMWARE = { - 'aarch64': 'usr/share/OVMF/QEMU_EFI.fd', - 'x86_64': 'usr/share/OVMF/OVMF.fd' +UEFI_FIRMWARE = { + 'aarch64': { + 'apk': 'aavmf', + 'bin': 'usr/share/AAVMF/QEMU_EFI.fd', + }, + 'x86_64': { + 'apk': 'ovmf', + 'bin': 'usr/share/OVMF/OVMF.fd', + } } alpine = Alpine() @@ -120,6 +126,9 @@ def install_overlay(overlay): os.makedirs(dest_dir, exist_ok=True) for src in unique_list(['.'] + args.custom): src_dir = os.path.join(src, overlay) + if not os.path.exists(src_dir): + log.debug('%s does not exist, skipping', src_dir) + continue for x in glob(os.path.join(src_dir, '**'), recursive=True): x = x.removeprefix(src_dir + '/') src_x = os.path.join(src_dir, x) @@ -169,10 +178,12 @@ def install_qemu_firmware(): log.info('Installing UEFI firmware in work environment') os.makedirs(firm_dir) - for arch, bin in OVMF_FIRMWARE.items(): - v = alpine.apk_version('community', arch, 'ovmf') - ovmf_url = f"{alpine.repo_url('community', arch)}/ovmf-{v}.apk" - data = urlopen(ovmf_url).read() + for arch, a_cfg in UEFI_FIRMWARE.items(): + apk = a_cfg['apk'] + bin = a_cfg['bin'] + v = alpine.apk_version('community', arch, apk) + apk_url = f"{alpine.repo_url('community', arch)}/{apk}-{v}.apk" + data = urlopen(apk_url).read() # Python tarfile library can't extract from APKs tar_cmd = ['tar', '-zxf', '-', '-C', firm_dir, bin] @@ -241,6 +252,8 @@ console.setFormatter(logfmt) log.addHandler(console) log.debug(args) +# TODO: rollback requires --revise + # set up credential provider, if we're going to use it if args.use_broker: clouds.set_credential_provider(debug=args.debug) @@ -274,7 +287,7 @@ if not image_configs.refresh_state( log.info('No pending actions to take at this time.') sys.exit(0) -if args.step == 'state': +if args.step == 'state' or args.step == 'rollback': sys.exit(0) # install firmware if missing diff --git a/clouds/aws.py b/clouds/aws.py index 436caa7..e98239c 100644 --- a/clouds/aws.py +++ b/clouds/aws.py @@ -370,10 +370,13 @@ class AWSCloudAdapter(CloudAdapterInterface): # set up AMI deprecation ec2c = image.meta.client log.info('%s: Setting EOL deprecation time on %s', r, image.id) - ec2c.enable_image_deprecation( - ImageId=image.id, - DeprecateAt=f"{tags.end_of_life}T23:59:59Z" - ) + try: + ec2c.enable_image_deprecation( + ImageId=image.id, + DeprecateAt=f"{tags.end_of_life}T23:59:59Z" + ) + except Exception: + log.warning('Unable to set EOL Deprecation on %s image:', r, exc_info=True) artifacts[r] = image.id diff --git a/configs/alpine.conf b/configs/alpine.conf index 6b44962..37cf0bd 100644 --- a/configs/alpine.conf +++ b/configs/alpine.conf @@ -46,10 +46,10 @@ Default { # profile build matrix Dimensions { version { + "3.16" { include required("version/3.16.conf") } "3.15" { include required("version/3.15.conf") } "3.14" { include required("version/3.14.conf") } "3.13" { include required("version/3.13.conf") } - "3.12" { include required("version/3.12.conf") } edge { include required("version/edge.conf") } } arch { diff --git a/configs/arch/aarch64.conf b/configs/arch/aarch64.conf index 41f1161..dc4cfc3 100644 --- a/configs/arch/aarch64.conf +++ b/configs/arch/aarch64.conf @@ -1,5 +1,6 @@ # vim: ts=2 et: name = [aarch64] +arch_name = aarch64 # aarch64 is UEFI only EXCLUDE = [bios] diff --git a/configs/arch/x86_64.conf b/configs/arch/x86_64.conf index af68640..d90ba09 100644 --- a/configs/arch/x86_64.conf +++ b/configs/arch/x86_64.conf @@ -1,5 +1,6 @@ # vim: ts=2 et: name = [x86_64] +arch_name = x86_64 # TODO: until we have a image metadata service, let's avoid UEFI EXCLUDE = [uefi] diff --git a/configs/bootstrap/cloudinit.conf b/configs/bootstrap/cloudinit.conf index bd35331..19dd869 100644 --- a/configs/bootstrap/cloudinit.conf +++ b/configs/bootstrap/cloudinit.conf @@ -1,5 +1,7 @@ # vim: ts=2 et: -name = [cloudinit] +name = [cloudinit] +bootstrap_name = cloud-init +bootstrap_url = "https://cloud-init.io/" # start cloudinit images with 3.15 EXCLUDE = ["3.12", "3.13", "3.14"] diff --git a/configs/bootstrap/tiny.conf b/configs/bootstrap/tiny.conf index 351ccf8..1a50056 100644 --- a/configs/bootstrap/tiny.conf +++ b/configs/bootstrap/tiny.conf @@ -1,5 +1,7 @@ # vim: ts=2 et: -name = [tiny] +name = [tiny] +bootstrap_name = Tiny Cloud +bootstrap_url = "https://gitlab.alpinelinux.org/alpine/cloud/tiny-cloud" services { sysinit.tiny-cloud-early = true diff --git a/configs/cloud/aws.conf b/configs/cloud/aws.conf index f7e6329..fd23f4e 100644 --- a/configs/cloud/aws.conf +++ b/configs/cloud/aws.conf @@ -1,6 +1,9 @@ # vim: ts=2 et: -builder = qemu +cloud_name = Amazon Web Services +cloud_image_url = "https://{region}.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId={image_id}", +cloud_launch_url = "https://{region}.console.aws.amazon.com/ec2/home#launchAmi={image_id}" +builder = qemu ntp_server = 169.254.169.123 kernel_modules { diff --git a/configs/firmware/bios.conf b/configs/firmware/bios.conf index c1d9602..6bd8161 100644 --- a/configs/firmware/bios.conf +++ b/configs/firmware/bios.conf @@ -1,5 +1,6 @@ # vim: ts=2 et: -name = [bios] +name = [bios] +firmware_name = BIOS bootloader = extlinux packages.syslinux = --no-scripts diff --git a/configs/firmware/uefi.conf b/configs/firmware/uefi.conf index fe23a45..12937c7 100644 --- a/configs/firmware/uefi.conf +++ b/configs/firmware/uefi.conf @@ -1,5 +1,6 @@ # vim: ts=2 et: -name = [uefi] +name = [uefi] +firmware_name = UEFI bootloader = grub-efi packages { diff --git a/configs/version/3.12.conf b/configs/version/3.12.conf index 3fbb039..9254d91 100644 --- a/configs/version/3.12.conf +++ b/configs/version/3.12.conf @@ -1,3 +1,5 @@ # vim: ts=2 et: -include required("base/1.conf") \ No newline at end of file +include required("base/1.conf") + +# NOTE: EOL 2022-05-01 \ No newline at end of file diff --git a/configs/version/3.16.conf b/configs/version/3.16.conf new file mode 100644 index 0000000..29a6936 --- /dev/null +++ b/configs/version/3.16.conf @@ -0,0 +1,7 @@ +# vim: ts=2 et: + +include required("base/4.conf") + +motd { + sudo_removed = "NOTE: 'sudo' is no longer installed by default, please use 'doas' instead." +} diff --git a/image_configs.py b/image_configs.py index 891ed3c..1abb964 100644 --- a/image_configs.py +++ b/image_configs.py @@ -206,6 +206,12 @@ class ImageConfig(): def image_description(self): return self.description.format(**self.__dict__) + def image_url(self, region, image_id): + return self.cloud_image_url.format(region=region, image_id=image_id, **self.__dict__) + + def launch_url(self, region, image_id): + return self.cloud_launch_url.format(region=region, image_id=image_id, **self.__dict__) + @property def tags(self): # stuff that really ought to be there