From 5d0009b5f8878e33a8e9b0ab12f806179e2916d6 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 28 Apr 2023 10:12:12 +0000 Subject: [PATCH] Squashed 'alpine-cloud-images/' content from commit 59d83fd git-subtree-dir: alpine-cloud-images git-subtree-split: 59d83fdf536c111dd48525430a6ce3d4028d6f8b --- .flake8 | 2 + .gitignore | 7 + CONFIGURATION.md | 314 +++++++++++++ LICENSE.txt | 19 + README.md | 208 +++++++++ alpine.pkr.hcl | 201 ++++++++ alpine.py | 114 +++++ build | 349 ++++++++++++++ cloud_helper.py | 98 ++++ clouds/__init__.py | 51 ++ clouds/aws.py | 397 ++++++++++++++++ clouds/azure.py | 22 + clouds/gcp.py | 22 + clouds/identity_broker_client.py | 135 ++++++ clouds/interfaces/__init__.py | 0 clouds/interfaces/adapter.py | 40 ++ clouds/nocloud.py | 21 + clouds/oci.py | 22 + configs/alpine.conf | 105 +++++ configs/arch/aarch64.conf | 15 + configs/arch/x86_64.conf | 6 + configs/bootstrap/cloudinit.conf | 17 + configs/bootstrap/tiny.conf | 35 ++ configs/cloud/aws.conf | 40 ++ configs/cloud/azure.conf | 9 + configs/cloud/gcp.conf | 15 + configs/cloud/nocloud.conf | 8 + configs/cloud/oci.conf | 8 + configs/firmware/bios.conf | 7 + configs/firmware/uefi.conf | 18 + configs/images.conf | 1 + configs/version/3.12.conf | 5 + configs/version/3.13.conf | 3 + configs/version/3.14.conf | 3 + configs/version/3.15.conf | 7 + configs/version/3.16.conf | 7 + configs/version/3.17.conf | 7 + configs/version/base/1.conf | 60 +++ configs/version/base/2.conf | 8 + configs/version/base/3.conf | 8 + configs/version/base/4.conf | 8 + configs/version/base/5.conf | 8 + configs/version/edge.conf | 15 + gen_mksite_releases.py | 215 +++++++++ image_config.py | 465 +++++++++++++++++++ image_config_manager.py | 178 +++++++ image_storage.py | 183 ++++++++ image_tags.py | 32 ++ overlays/testing/configs/alpine-testing.conf | 43 ++ overlays/testing/configs/images.conf | 1 + overlays/testing/configs/machine/metal.conf | 9 + overlays/testing/configs/machine/vm.conf | 4 + overlays/testing/configs/testing/oci.conf | 4 + scripts/cleanup | 42 ++ scripts/setup | 262 +++++++++++ scripts/setup-cloudinit | 48 ++ scripts/setup-tiny | 21 + scripts/setup.d/fstab | 2 + scripts/setup.d/fstab.grub-efi | 1 + scripts/setup.d/grub.template | 5 + scripts/setup.d/interfaces | 7 + support/aws/iam_role_vmimport_policy.json | 44 ++ support/aws/iam_role_vmimport_trust.json | 17 + 63 files changed, 4028 insertions(+) create mode 100644 .flake8 create mode 100644 .gitignore create mode 100644 CONFIGURATION.md create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 alpine.pkr.hcl create mode 100644 alpine.py create mode 100755 build create mode 100755 cloud_helper.py create mode 100644 clouds/__init__.py create mode 100644 clouds/aws.py create mode 100644 clouds/azure.py create mode 100644 clouds/gcp.py create mode 100644 clouds/identity_broker_client.py create mode 100644 clouds/interfaces/__init__.py create mode 100644 clouds/interfaces/adapter.py create mode 100644 clouds/nocloud.py create mode 100644 clouds/oci.py create mode 100644 configs/alpine.conf create mode 100644 configs/arch/aarch64.conf create mode 100644 configs/arch/x86_64.conf create mode 100644 configs/bootstrap/cloudinit.conf create mode 100644 configs/bootstrap/tiny.conf create mode 100644 configs/cloud/aws.conf create mode 100644 configs/cloud/azure.conf create mode 100644 configs/cloud/gcp.conf create mode 100644 configs/cloud/nocloud.conf create mode 100644 configs/cloud/oci.conf create mode 100644 configs/firmware/bios.conf create mode 100644 configs/firmware/uefi.conf create mode 120000 configs/images.conf create mode 100644 configs/version/3.12.conf create mode 100644 configs/version/3.13.conf create mode 100644 configs/version/3.14.conf create mode 100644 configs/version/3.15.conf create mode 100644 configs/version/3.16.conf create mode 100644 configs/version/3.17.conf create mode 100644 configs/version/base/1.conf create mode 100644 configs/version/base/2.conf create mode 100644 configs/version/base/3.conf create mode 100644 configs/version/base/4.conf create mode 100644 configs/version/base/5.conf create mode 100644 configs/version/edge.conf create mode 100755 gen_mksite_releases.py create mode 100644 image_config.py create mode 100644 image_config_manager.py create mode 100644 image_storage.py create mode 100644 image_tags.py create mode 100644 overlays/testing/configs/alpine-testing.conf create mode 120000 overlays/testing/configs/images.conf create mode 100644 overlays/testing/configs/machine/metal.conf create mode 100644 overlays/testing/configs/machine/vm.conf create mode 100644 overlays/testing/configs/testing/oci.conf create mode 100644 scripts/cleanup create mode 100755 scripts/setup create mode 100755 scripts/setup-cloudinit create mode 100755 scripts/setup-tiny create mode 100644 scripts/setup.d/fstab create mode 100644 scripts/setup.d/fstab.grub-efi create mode 100644 scripts/setup.d/grub.template create mode 100644 scripts/setup.d/interfaces create mode 100644 support/aws/iam_role_vmimport_policy.json create mode 100644 support/aws/iam_role_vmimport_trust.json diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..7b8d203 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +ignore = E265,E266,E402,E501 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3214f0a --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*~ +*.bak +*.swp +.DS_Store +.vscode/ +/work/ +releases*yaml diff --git a/CONFIGURATION.md b/CONFIGURATION.md new file mode 100644 index 0000000..00d9fee --- /dev/null +++ b/CONFIGURATION.md @@ -0,0 +1,314 @@ +# Configuration + +All the configuration for building image variants is defined by multiple +config files; the base configs for official Alpine Linux cloud images are in +the [`configs/`](configs/) directory. + +We use [HOCON](https://github.com/lightbend/config/blob/main/HOCON.md) for +configuration -- this primarily facilitates importing deeper configs from +other files, but also allows the extension/concatenation of arrays and maps +(which can be a useful feature for customization), and inline comments. + +---- +## Resolving Work Environment Configs and Scripts + +If `work/configs/` and `work/scripts/` don't exist, the `build` script will +install the contents of the base [`configs/`](configs/) and [`scripts/`](scripts/) +directories, and overlay additional `configs/` and `scripts/` subdirectories +from `--custom` directories (if any). + +Files cannot be installed over existing files, with one exception -- the +[`configs/images.conf`](configs/images.conf) same-directory symlink. Because +the `build` script _always_ loads `work/configs/images.conf`, this is the hook +for "rolling your own" custom Alpine Linux cloud images. + +The base [`configs/images.conf`](configs/images.conf) symlinks to +[`alpine.conf`](configs/images.conf), but this can be overridden using a +`--custom` directory containing a new `configs/images.conf` same-directory +symlink pointing to its custom top-level config. + +For example, the configs and scripts in the [`overlays/testing/`](overlays/testing/) +directory can be resolved in a _clean_ work environment with... +``` +./build configs --custom overlays/testing +``` +This results in the `work/configs/images.conf` symlink to point to +`work/configs/alpine-testing.conf` instead of `work/configs/alpine.conf`. + +If multiple directories are specified with `--custom`, they are applied in +the order given. + +---- +## Top-Level Config File + +Examples of top-level config files are [`configs/alpine.conf`](configs/alpine.conf) +and [`overlays/testing/configs/alpine-testing.conf`](overlays/testing/configs/alpine-testing.conf). + +There are three main blocks that need to exist (or be `import`ed into) the top +level HOCON configuration, and are merged in this exact order: + +### `Default` + +All image variant configs start with this block's contents as a starting point. +Arrays and maps can be appended by configs in `Dimensions` and `Mandatory` +blocks. + +### `Dimensions` + +The sub-blocks in `Dimensions` define the "dimensions" a variant config is +comprised of, and the different config values possible for that dimension. +The default [`alpine.conf`](configs/alpine.conf) defines the following +dimensional configs: + +* `version` - Alpine Linux _x_._y_ (plus `edge`) versions +* `arch` - machine architectures, `x86_64` or `aarch64` +* `firmware` - supports launching via legacy BIOS or UEFI +* `bootstrap` - the system/scripts responsible for setting up an instance + during its initial launch +* `cloud` - for specific cloud platforms + +The specific dimensional configs for an image variant are merged in the order +that the dimensions are listed. + +### `Mandatory` + +After a variant's dimensional configs have been applied, this is the last block +that's merged to the image variant configuration. This block is the ultimate +enforcer of any non-overrideable configuration across all variants, and can +also provide the last element to array config items. + +---- +## Dimensional Config Directives + +Because a full cross-product across all dimensional configs may produce images +variants that are not viable (i.e. `aarch64` simply does not support legacy +`bios`), or may require further adjustments (i.e. the `aws` `aarch64` images +require an additional kernel module from `3.15` forward, which aren't available +in previous versions), we have two special directives which may appear in +dimensional configs. + +### `EXCLUDE` array + +This directive provides an array of dimensional config keys which are +incompatible with the current dimensional config. For example, +[`configs/arch/aarch64.conf`](configs/arch/aarch64.conf) specifies... +``` +# aarch64 is UEFI only +EXCLUDE = [bios] +``` +...which indicates that any image variant that includes both `aarch64` (the +current dimensional config) and `bios` configuration should be skipped. + +### `WHEN` block + +This directive conditionally merges additional configuration ***IF*** the +image variant also includes a specific dimensional config key (or keys). In +order to handle more complex situations, `WHEN` blocks may be nested. For +example, [`configs/cloud/aws.conf`](configs/cloud/aws.conf) has... +``` +WHEN { + aarch64 { + # new AWS aarch64 default... + kernel_modules.gpio_pl061 = true + initfs_features.gpio_pl061 = true + WHEN { + "3.14 3.13 3.12" { + # ...but not supported for older versions + kernel_modules.gpio_pl061 = false + initfs_features.gpio_pl061 = false + } + } + } +``` +This configures AWS `aarch64` images to use the `gpio_pl061` kernel module in +order to cleanly shutdown/reboot instances from the web console, CLI, or SDK. +However, this module is unavailable on older Alpine versions. + +Spaces in `WHEN` block keys serve as an "OR" operator; nested `WHEN` blocks +function as "AND" operators. + +---- +## Config Settings + +**Scalar** values can be simply overridden in later configs. + +**Array** and **map** settings in later configs are merged with the previous +values, _or entirely reset if it's first set to `null`_, for example... +``` +some_array = [ thing ] +# [...] +some_array = null +some_array = [ other_thing ] +``` + +Mostly in order of appearance, as we walk through +[`configs/alpine.conf`](configs/alpine.conf) and the deeper configs it +imports... + +### `project` string + +This is a unique identifier for the whole collection of images being built. +For the official Alpine Linux cloud images, this is set to +`https://alpinelinux.org/cloud`. + +When building custom images, you **MUST** override **AT LEAST** this setting to +avoid image import and publishing collisions. + +### `name` array + +The ultimate contents of this array contribute to the overall naming of the +resultant image. Almost all dimensional configs will add to the `name` array, +with two notable exceptions: **version** configs' contribution to this array is +determined when `work/images.yaml` is resolved, and is set to the current +Alpine Linux release (_x.y.z_ or _YYYYMMDD_ for edge); also because +**cloud** images are isolated from each other, it's redundant to include that +in the image name. + +### `description` array + +Similar to the `name` array, the elements of this array contribute to the final +image description. However, for the official Alpine configs, only the +**version** dimension adds to this array, via the same mechanism that sets the +revision for the `name` array. + +### `motd` map + +This setting controls the contents of what ultimately gets written into the +variant image's `/etc/motd` file. Later configs can add additional messages, +replace existing contents, or remove them entirely (by setting the value to +`null`). + +The `motd.release_notes` setting will be ignored if the Alpine release does +not have a release notes web page associated with it. + +### `scripts` array + +These are the scripts that will be executed by Packer, in order, to do various +setup tasks inside a variant's image. The `work/scripts/` directory contains +all scripts, including those that may have been added via `build --custom`. + +### `script_dirs` array + +Directories (under `work/scripts/`) that contain additional data that the +`scripts` will need. Packer will copy these to the VM responsible for setting +up the variant image. + +### `size` string + +The size of the image disk, by default we use `1G` (1 GiB). This disk may (or +may not) be further partitioned, based on other factors. + +### `login` string + +The image's primary login user, set to `alpine`. + +### `repos` map + +Defines the contents of the image's `/etc/apk/repositories` file. The map's +key is the URL of the repo, and the value determines how that URL will be +represented in the `repositories` file... +| value | result | +|-|-| +| `null` | make no reference to this repo | +| `false` | this repo is commented out (disabled) | +| `true` | this repo is enabled for use | +| _tag_ | enable this repo with `@`_`tag`_ | + +### `packages` map + +Defines what APK packages to add/delete. The map's key is the package +name, and the value determines whether (or not) to install/uninstall the +package... +| value | result | +|-|-| +| `null` | don't add or delete | +| `false` | explicitly delete | +| `true` | add from default repos | +| _tag_ | add from `@`_`tag`_ repo | +| `--no-scripts` | add with `--no-scripts` option | +| `--no-scripts` _tag_ | add from `@`_`tag`_ repo, with `--no-scripts` option | + +### `services` map of maps + +Defines what services are enabled/disabled at various runlevels. The first +map's key is the runlevel, the second key is the service. The service value +determines whether (or not) to enable/disable the service at that runlevel... +| value | result | +|-|-| +| `null` | don't enable or disable | +| `false` | explicitly disable | +| `true` | explicitly enable | + +### `kernel_modules` map + +Defines what kernel modules are specified in the boot loader. The key is the +kernel module, and the value determines whether or not it's in the final +list... +| value | result | +|-|-| +| `null` | skip | +| `false` | skip | +| `true` | include | + +### `kernel_options` map + +Defines what kernel options are specified on the kernel command line. The keys +are the kernel options, the value determines whether or not it's in the final +list... +| value | result | +|-|-| +| `null` | skip | +| `false` | skip | +| `true` | include | + +### `initfs_features` map + +Defines what initfs features are included when making the image's initramfs +file. The keys are the initfs features, and the values determine whether or +not they're included in the final list... +| value | result | +|-|-| +| `null` | skip | +| `false` | skip | +| `true` | include | + +### `qemu.machine_type` string + +The QEMU machine type to use when building local images. For x86_64, this is +set to `null`, for aarch64, we use `virt`. + +### `qemu.args` list of lists + +Additional QEMU arguments. For x86_64, this is set to `null`; but aarch64 +requires several additional arguments to start an operational VM. + +### `qemu.firmware` string + +The path to the QEMU firmware (installed in `work/firmware/`). This is only +used when creating UEFI images. + +### `bootloader` string + +The bootloader to use, currently `extlinux` or `grub-efi`. + +### `access` map + +When images are published, this determines who has access to those images. +The key is the cloud account (or `PUBLIC`), and the value is whether or not +access is granted, `true` or `false`/`null`. + +### `regions` map + +Determines where images should be published. The key is the region +identifier (or `ALL`), and the value is whether or not to publish to that +region, `true` or `false`/`null`. + +### `encrypted` string + +Determines whether the image will be encrypted when imported and published. +Currently, only the **aws** cloud module supports this. + +### `repo_keys` array + +List of addtional repository keys to trust during the package installation phase. +This allows pulling in custom apk packages by simple specifying the repository name in packages block. diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..817eab1 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2017-2022 Jake Buchholz Göktürk, Michael Crute + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..54755a5 --- /dev/null +++ b/README.md @@ -0,0 +1,208 @@ +# Alpine Linux Cloud Image Builder + +This repository contains the code and and configs for the build system used to +create official Alpine Linux images for various cloud providers, in various +configurations. This build system is flexible, enabling others to build their +own customized images. + +---- +## Pre-Built Offical Cloud Images + +To get started with offical pre-built Alpine Linux cloud images, visit +https://alpinelinux.org/cloud. Currently, we build official images for the +following cloud platforms... +* AWS + +...we are working on also publishing offical images to other major cloud +providers. + +Each published image's name contains the Alpine version release, architecture, +firmware, bootstrap, and image revision. These details (and more) are also +tagged on the images... + +| Tag | Description / Values | +|-----|----------------------| +| name | `alpine-`_`release`_`-`_`arch`_`-`_`firmware`_`-`_`bootstrap`_`-r`_`revision`_ | +| project | `https://alpinelinux.org/cloud` | +| image_key | _`release`_`-`_`arch`_`-`_`firmware`_`-`_`bootstrap`_`-`_`cloud`_ | +| version | Alpine version (_`x.y`_ or `edge`) | +| release | Alpine release (_`x.y.z`_ or _`YYYYMMDD`_ for edge) | +| arch | architecture (`aarch64` or `x86_64`) | +| firmware | boot mode (`bios` or `uefi`) | +| bootstrap | initial bootstrap system (`tiny` = Tiny Cloud) | +| cloud | provider short name (`aws`) | +| revision | image revision number | +| imported | image import timestamp | +| import_id | imported image id | +| import_region | imported image region | +| published | image publication timestamp | +| description | image description | + +Although AWS does not allow cross-account filtering by tags, the image name can +still be used to filter images. For example, to get a list of available Alpine +3.x aarch64 images in AWS eu-west-2... +``` +aws ec2 describe-images \ + --region eu-west-2 \ + --owners 538276064493 \ + --filters \ + Name=name,Values='alpine-3.*-aarch64-*' \ + Name=state,Values=available \ + --output text \ + --query 'reverse(sort_by(Images, &CreationDate))[].[ImageId,Name,CreationDate]' +``` +To get just the most recent matching image, use... +``` + --query 'max_by(Image, &CreationDate).[ImageId,Name,CreationDate]' +``` + +---- +## Build System + +The build system consists of a number of components: + +* the primary `build` script +* the `configs/` directory, defining the set of images to be built +* the `scripts/` directory, containing scripts and related data used to set up + image contents during provisioning +* the Packer `alpine.pkr.hcl`, which orchestrates build, import, and publishing + of images +* the `cloud_helper.py` script that Packer runs in order to do cloud-specific + import and publish operations + +### Build Requirements +* [Python](https://python.org) (3.9.7 is known to work) +* [Packer](https://packer.io) (1.7.6 is known to work) +* [QEMU](https://www.qemu.org) (6.1.0 is known to work) +* cloud provider account(s) + +### Cloud Credentials + +By default, the build system relies on the cloud providers' Python API +libraries to find and use the necessary credentials, usually via configuration +under the user's home directory (i.e. `~/.aws/`, `~/.oci/`, etc.) or or via +environment variables (i.e. `AWS_...`, `OCI_...`, etc.) + +The credentials' user/role needs sufficient permission to query, import, and +publish images -- the exact details will vary from cloud to cloud. _It is +recommended that only the minimum required permissions are granted._ + +_We manage the credentials for publishing official Alpine images with an +"identity broker" service, and retrieve those credentials via the +`--use-broker` argument of the `build` script._ + +### The `build` Script + +``` +usage: build [-h] [--debug] [--clean] [--pad-uefi-bin-arch ARCH [ARCH ...]] + [--custom DIR [DIR ...]] [--skip KEY [KEY ...]] [--only KEY [KEY ...]] + [--revise] [--use-broker] [--no-color] [--parallel N] + [--vars FILE [FILE ...]] + {configs,state,rollback,local,upload,import,publish,release} + +positional arguments: (build up to and including this step) + configs resolve image build configuration + state refresh current image build state + rollback remove existing local/uploaded/imported images if un-published/released + local build images locally + upload upload images and metadata to storage +* import import local images to cloud provider default region (*) +* publish set image permissions and publish to cloud regions (*) + release mark images as being officially relased + +(*) may not apply to or be implemented for all cloud providers + +optional arguments: + -h, --help show this help message and exit + --debug enable debug output + --clean start with a clean work environment + --pad-uefi-bin-arch ARCH [ARCH ...] + pad out UEFI firmware to 64 MiB ('aarch64') + --custom DIR [DIR ...] overlay custom directory in work environment + --skip KEY [KEY ...] skip variants with dimension key(s) + --only KEY [KEY ...] only variants with dimension key(s) + --revise remove existing local/uploaded/imported images if + un-published/released, or bump revision and rebuild + --use-broker use the identity broker to get credentials + --no-color turn off Packer color output + --parallel N build N images in parallel + --vars FILE [FILE ...] supply Packer with -vars-file(s) (default: []) +``` + +The `build` script will automatically create a `work/` directory containing a +Python virtual environment if one does not already exist. This directory also +hosts other data related to building images. The `--clean` argument will +remove everything in the `work/` directory except for things related to the +Python virtual environment. + +If `work/configs/` or `work/scripts/` directories do not yet exist, they will +be populated with the base configuration and scripts from `configs/` and/or +`scripts/` directories. If any custom overlay directories are specified with +the `--custom` argument, their `configs/` and `scripts/` subdirectories are +also added to `work/configs/` and `work/scripts/`. + +The "build step" positional argument deterimines the last step the `build` +script should execute -- all steps before this targeted step may also be +executed. That is, `build local` will first execute the `configs` step (if +necessary) and then the `state` step (always) before proceeding to the `local` +step. + +The `configs` step resolves configuration for all buildable images, and writes +it to `work/images.yaml`, if it does not already exist. + +The `state` step always checks the current state of the image builds, +determines what actions need to be taken, and updates `work/images.yaml`. A +subset of image builds can be targeted by using the `--skip` and `--only` +arguments. + +The `rollback` step, when used with `--revise` argument indicates that any +_unpublished_ and _unreleased_ local, imported, or uploaded images should be +removed and rebuilt. + +As _published_ and _released_ images can't be removed, `--revise` can be used +with `configs` or `state` to increment the _`revision`_ value to rebuild newly +revised images. + +`local`, `upload`, `import`, `publish`, and `release` steps are orchestrated by +Packer. By default, each image will be processed serially; providing the +`--parallel` argument with a value greater than 1 will parallelize operations. +The degree to which you can parallelze `local` image builds will depend on the +local build hardware -- as QEMU virtual machines are launched for each image +being built. Image `upload`, `import`, `publish`, and `release` steps are much +more lightweight, and can support higher parallelism. + +The `local` step builds local images with QEMU, for those that are not already +built locally or have already been imported. Images are converted to formats +amenable for import into the cloud provider (if necessary) and checksums are +generated. + +The `upload` step uploads the local image, checksum, and metadata to the +defined `storage_url`. The `import`, `publish`, and `release` steps will +also upload updated image metadata. + +The `import` step imports the local images into the cloud providers' default +regions, unless they've already been imported. At this point the images are +not available publicly, allowing for additional testing prior to publishing. + +The `publish` step copies the image from the default region to other regions, +if they haven't already been copied there. This step will always update +image permissions, descriptions, tags, and deprecation date (if applicable) +in all regions where the image has been published. + +***NOTE:*** The `import` and `publish` steps are skipped for those cloud +providers where this does not make sense (i.e. NoCloud) or for those which +it has not yet been coded. + +The `release` step marks the images as being fully released. + +### The `cloud_helper.py` Script + +This script is meant to be called only by Packer from its `post-processor` +block. + +---- +## Build Configuration + +For more in-depth information about how the build system configuration works, +how to create custom config overlays, and details about individual config +settings, see [CONFIGURATION.md](CONFIGURATION.md). diff --git a/alpine.pkr.hcl b/alpine.pkr.hcl new file mode 100644 index 0000000..0e8fb27 --- /dev/null +++ b/alpine.pkr.hcl @@ -0,0 +1,201 @@ +# Alpine Cloud Images Packer Configuration + +### Variables + +# include debug output from provisioning/post-processing scripts +variable "DEBUG" { + default = 0 +} +# indicates cloud_helper.py should be run with --use-broker +variable "USE_BROKER" { + default = 0 +} + +# tuneable QEMU VM parameters, based on perfomance of the local machine; +# overrideable via build script --vars parameter referencing a Packer +# ".vars.hcl" file containing alternate settings +variable "qemu" { + default = { + boot_wait = { + aarch64 = "1m" + x86_64 = "1m" + } + cmd_wait = "5s" + ssh_timeout = "1m" + memory = 1024 # MiB + } +} + +### Local Data + +locals { + # possible actions for the post-processor + actions = [ + "local", "upload", "import", "publish", "release" + ] + + debug_arg = var.DEBUG == 0 ? "" : "--debug" + broker_arg = var.USE_BROKER == 0 ? "" : "--use-broker" + + # randomly generated password + password = uuidv4() + + # resolve actionable build configs + configs = { for b, cfg in yamldecode(file("work/images.yaml")): + b => cfg if contains(keys(cfg), "actions") + } +} + +### Build Sources + +# Don't build +source null alpine { + communicator = "none" +} + +# Common to all QEMU builds +source qemu alpine { + # qemu machine + headless = true + memory = var.qemu.memory + net_device = "virtio-net" + disk_interface = "virtio" + + # build environment + boot_command = [ + "root", + "setup-interfaces", + "ifup eth0", + "setup-sshd openssh", + "echo PermitRootLogin yes >> /etc/ssh/sshd_config", + "service sshd restart", + "echo 'root:${local.password}' | chpasswd", + ] + ssh_username = "root" + ssh_password = local.password + ssh_timeout = var.qemu.ssh_timeout + shutdown_command = "poweroff" +} + +build { + name = "alpine" + + ## Builders + + # QEMU builder + dynamic "source" { + for_each = { for b, c in local.configs: + b => c if contains(c.actions, "local") + } + iterator = B + labels = ["qemu.alpine"] # links us to the base source + + content { + name = B.key + + # qemu machine + qemu_binary = "qemu-system-${B.value.arch}" + qemuargs = B.value.qemu.args + machine_type = B.value.qemu.machine_type + firmware = B.value.qemu.firmware + + # build environment + iso_url = B.value.qemu.iso_url + iso_checksum = "file:${B.value.qemu.iso_url}.sha512" + boot_wait = var.qemu.boot_wait[B.value.arch] + + # results + output_directory = "work/images/${B.value.cloud}/${B.value.image_key}" + disk_size = B.value.size + format = "qcow2" + vm_name = "image.qcow2" + } + } + + # Null builder (don't build, but we might do other actions) + dynamic "source" { + for_each = { for b, c in local.configs: + b => c if !contains(c.actions, "local") + } + iterator = B + labels = ["null.alpine"] + content { + name = B.key + } + } + + ## build provisioners + + # install setup files + dynamic "provisioner" { + for_each = { for b, c in local.configs: + b => c if contains(c.actions, "local") + } + iterator = B + labels = ["file"] + content { + only = [ "qemu.${B.key}" ] # configs specific to one build + + sources = [ for d in B.value.script_dirs: "work/scripts/${d}" ] + destination = "/tmp/" + } + } + + # run setup scripts + dynamic "provisioner" { + for_each = { for b, c in local.configs: + b => c if contains(c.actions, "local") + } + iterator = B + labels = ["shell"] + content { + only = [ "qemu.${B.key}" ] # configs specific to one build + + scripts = [ for s in B.value.scripts: "work/scripts/${s}" ] + use_env_var_file = true + environment_vars = [ + "DEBUG=${var.DEBUG}", + "ARCH=${B.value.arch}", + "BOOTLOADER=${B.value.bootloader}", + "BOOTSTRAP=${B.value.bootstrap}", + "BUILD_NAME=${B.value.name}", + "BUILD_REVISION=${B.value.revision}", + "CLOUD=${B.value.cloud}", + "END_OF_LIFE=${B.value.end_of_life}", + "FIRMWARE=${B.value.firmware}", + "IMAGE_LOGIN=${B.value.login}", + "INITFS_FEATURES=${B.value.initfs_features}", + "KERNEL_MODULES=${B.value.kernel_modules}", + "KERNEL_OPTIONS=${B.value.kernel_options}", + "MOTD=${B.value.motd}", + "NTP_SERVER=${B.value.ntp_server}", + "PACKAGES_ADD=${B.value.packages.add}", + "PACKAGES_DEL=${B.value.packages.del}", + "PACKAGES_NOSCRIPTS=${B.value.packages.noscripts}", + "RELEASE=${B.value.release}", + "REPOS=${B.value.repos}", + "REPO_KEYS=${B.value.repo_keys}", + "SERVICES_ENABLE=${B.value.services.enable}", + "SERVICES_DISABLE=${B.value.services.disable}", + "VERSION=${B.value.version}", + ] + } + } + + ## build post-processor + + # import and/or publish cloud images + dynamic "post-processor" { + for_each = { for b, c in local.configs: + b => c if length(setintersection(c.actions, local.actions)) > 0 + } + iterator = B + labels = ["shell-local"] + content { + only = [ "qemu.${B.key}", "null.${B.key}" ] + inline = [ for action in local.actions: + "./cloud_helper.py ${action} ${local.debug_arg} ${local.broker_arg} ${B.key}" if contains(B.value.actions, action) + ] + } + } +} diff --git a/alpine.py b/alpine.py new file mode 100644 index 0000000..f334d5c --- /dev/null +++ b/alpine.py @@ -0,0 +1,114 @@ +# vim: ts=4 et: + +import json +import re +from datetime import datetime, timedelta +from urllib.request import urlopen + + +class Alpine(): + + DEFAULT_RELEASES_URL = 'https://alpinelinux.org/releases.json' + DEFAULT_POSTS_URL = 'https://alpinelinux.org/posts/' + DEFAULT_CDN_URL = 'https://dl-cdn.alpinelinux.org/alpine' + DEFAULT_WEB_TIMEOUT = 5 + + def __init__(self, releases_url=None, posts_url=None, cdn_url=None, web_timeout=None): + self.now = datetime.utcnow() + self.release_today = self.now.strftime('%Y%m%d') + self.eol_tomorrow = (self.now + timedelta(days=1)).strftime('%F') + self.latest = None + self.versions = {} + self.releases_url = releases_url or self.DEFAULT_RELEASES_URL + self.posts_url = posts_url or self.DEFAULT_POSTS_URL + self.web_timeout = web_timeout or self.DEFAULT_WEB_TIMEOUT + self.cdn_url = cdn_url or self.DEFAULT_CDN_URL + + # get all Alpine versions, and their EOL and latest release + res = urlopen(self.releases_url, timeout=self.web_timeout) + r = json.load(res) + branches = sorted( + r['release_branches'], reverse=True, + key=lambda x: x.get('branch_date', '0000-00-00') + ) + for b in branches: + ver = b['rel_branch'].lstrip('v') + if not self.latest: + self.latest = ver + + rel = None + notes = None + if releases := b.get('releases', None): + r = sorted( + releases, reverse=True, key=lambda x: x['date'] + )[0] + rel = r['version'] + notes = r.get('notes', None) + if notes: + notes = self.posts_url + notes.removeprefix('posts/').replace('.md', '.html') + + elif ver == 'edge': + # edge "releases" is today's YYYYMMDD + rel = self.release_today + + self.versions[ver] = { + 'version': ver, + 'release': rel, + 'end_of_life': b.get('eol_date', self.eol_tomorrow), + 'arches': b.get('arches'), + 'notes': notes, + } + + def _ver(self, ver=None): + if not ver or ver == 'latest' or ver == 'latest-stable': + ver = self.latest + + return ver + + def repo_url(self, repo, arch, ver=None): + ver = self._ver(ver) + if ver != 'edge': + ver = 'v' + ver + + return f"{self.cdn_url}/{ver}/{repo}/{arch}" + + def virt_iso_url(self, arch, ver=None): + ver = self._ver(ver) + rel = self.versions[ver]['release'] + return f"{self.cdn_url}/v{ver}/releases/{arch}/alpine-virt-{rel}-{arch}.iso" + + def version_info(self, ver=None): + ver = self._ver(ver) + if ver not in self.versions: + # perhaps a release candidate? + apk_ver = self.apk_version('main', 'x86_64', 'alpine-base', ver=ver) + rel = apk_ver.split('-')[0] + ver = '.'.join(rel.split('.')[:2]) + self.versions[ver] = { + 'version': ver, + 'release': rel, + 'end_of_life': self.eol_tomorrow, + 'arches': self.versions['edge']['arches'], # reasonable assumption + 'notes': None, + } + + return self.versions[ver] + + # TODO? maybe implement apk_info() to read from APKINDEX, but for now + # this apk_version() seems faster and gets what we need + + def apk_version(self, repo, arch, apk, ver=None): + ver = self._ver(ver) + repo_url = self.repo_url(repo, arch, ver=ver) + apks_re = re.compile(f'"{apk}-(\\d.*)\\.apk"') + res = urlopen(repo_url, timeout=self.web_timeout) + for line in map(lambda x: x.decode('utf8'), res): + if not line.startswith('.* and its objects + def import_image(self, ic): + log = logging.getLogger('import') + description = ic.image_description + + session = self.session() + s3r = session.resource('s3') + ec2c = session.client('ec2') + ec2r = session.resource('ec2') + + bucket_name = 'alpine-cloud-images.' + hashlib.sha1(os.urandom(40)).hexdigest() + + bucket = s3r.Bucket(bucket_name) + log.info('Creating S3 bucket %s', bucket.name) + bucket.create( + CreateBucketConfiguration={'LocationConstraint': ec2c.meta.region_name} + ) + bucket.wait_until_exists() + s3_url = f"s3://{bucket.name}/{ic.image_file}" + + try: + log.info('Uploading %s to %s', ic.image_path, s3_url) + bucket.upload_file(str(ic.image_path), ic.image_file) + + # import snapshot from S3 + log.info('Importing EC2 snapshot from %s', s3_url) + ss_import_opts = { + 'DiskContainer': { + 'Description': description, # https://github.com/boto/boto3/issues/2286 + 'Format': 'VHD', + 'Url': s3_url, + }, + 'Encrypted': True if ic.encrypted else False, + # NOTE: TagSpecifications -- doesn't work with ResourceType: snapshot? + } + if type(ic.encrypted) is str: + ss_import_opts['KmsKeyId'] = ic.encrypted + + ss_import = ec2c.import_snapshot(**ss_import_opts) + ss_task_id = ss_import['ImportTaskId'] + while True: + ss_task = ec2c.describe_import_snapshot_tasks( + ImportTaskIds=[ss_task_id] + ) + task_detail = ss_task['ImportSnapshotTasks'][0]['SnapshotTaskDetail'] + if task_detail['Status'] not in ['pending', 'active', 'completed']: + msg = f"Bad EC2 snapshot import: {task_detail['Status']} - {task_detail['StatusMessage']}" + log.error(msg) + raise RuntimeError(msg) + + if task_detail['Status'] == 'completed': + snapshot_id = task_detail['SnapshotId'] + break + + time.sleep(15) + except Exception: + log.error('Unable to import snapshot from S3:', exc_info=True) + raise + finally: + # always cleanup S3, even if there was an exception raised + log.info('Cleaning up %s', s3_url) + bucket.Object(ic.image_file).delete() + bucket.delete() + + # tag snapshot + snapshot = ec2r.Snapshot(snapshot_id) + try: + log.info('Tagging EC2 snapshot %s', snapshot_id) + tags = ic.tags + tags.Name = tags.name # because AWS is special + snapshot.create_tags(Tags=tags.as_list()) + except Exception: + log.error('Unable to tag snapshot:', exc_info=True) + log.info('Removing snapshot') + snapshot.delete() + raise + + # register AMI + try: + log.info('Registering EC2 AMI from snapshot %s', snapshot_id) + img = ec2c.register_image( + Architecture=self.ARCH[ic.arch], + BlockDeviceMappings=[{ + 'DeviceName': '/dev/xvda', + 'Ebs': { + 'SnapshotId': snapshot_id, + 'VolumeType': 'gp3' + } + }], + Description=description, + EnaSupport=True, + Name=ic.image_name, + RootDeviceName='/dev/xvda', + SriovNetSupport='simple', + VirtualizationType='hvm', + BootMode=self.BOOT_MODE[ic.firmware], + ) + except Exception: + log.error('Unable to register image:', exc_info=True) + log.info('Removing snapshot') + snapshot.delete() + raise + + image_id = img['ImageId'] + image = ec2r.Image(image_id) + + try: + # tag image (adds imported tag) + log.info('Tagging EC2 AMI %s', image_id) + tags.imported = datetime.utcnow().isoformat() + tags.import_id = image_id + tags.import_region = ec2c.meta.region_name + image.create_tags(Tags=tags.as_list()) + except Exception: + log.error('Unable to tag image:', exc_info=True) + log.info('Removing image and snapshot') + image.delete() + snapshot.delete() + raise + + # update ImageConfig with imported tag values, minus special AWS 'Name' + tags.pop('Name', None) + ic.__dict__ |= tags + + # delete an (unpublished) image + def delete_image(self, image_id): + log = logging.getLogger('build') + ec2r = self.session().resource('ec2') + image = ec2r.Image(image_id) + snapshot_id = image.block_device_mappings[0]['Ebs']['SnapshotId'] + snapshot = ec2r.Snapshot(snapshot_id) + log.info('Deregistering %s', image_id) + image.deregister() + log.info('Deleting %s', snapshot_id) + snapshot.delete() + + # publish an image + def publish_image(self, ic): + log = logging.getLogger('publish') + source_image = self.get_latest_imported_tags( + ic.project, + ic.image_key, + ) + if not source_image: + log.error('No source image for %s', ic.image_key) + raise RuntimeError('Missing source imamge') + + source_id = source_image.import_id + source_region = source_image.import_region + log.info('Publishing source: %s/%s', source_region, source_id) + source = self.session().resource('ec2').Image(source_id) + + # we may be updating tags, get them from image config + tags = ic.tags + + # sort out published image access permissions + perms = {'groups': [], 'users': []} + if ic.access.get('PUBLIC', None): + perms['groups'] = ['all'] + else: + for k, v in ic.access.items(): + if v: + log.debug('users: %s', k) + perms['users'].append(str(k)) + + log.debug('perms: %s', perms) + + # resolve destination regions + regions = self.regions + if ic.regions.pop('ALL', None): + log.info('Publishing to ALL available regions') + else: + # clear ALL out of the way if it's still there + ic.regions.pop('ALL', None) + regions = {r: regions[r] for r in ic.regions} + + publishing = {} + for r in regions.keys(): + if not regions[r]: + log.warning('Skipping unsubscribed AWS region %s', r) + continue + + images = self._get_images_with_tags( + region=r, + project=ic.project, + image_key=ic.image_key, + tags={'revision': ic.revision} + ) + if images: + image = images[0] + log.info('%s: Already exists as %s', r, image.id) + else: + ec2c = self.session(r).client('ec2') + copy_image_opts = { + 'Description': source.description, + 'Name': source.name, + 'SourceImageId': source_id, + 'SourceRegion': source_region, + 'Encrypted': True if ic.encrypted else False, + } + if type(ic.encrypted) is str: + copy_image_opts['KmsKeyId'] = ic.encrypted + + try: + res = ec2c.copy_image(**copy_image_opts) + except Exception: + log.warning('Skipping %s, unable to copy image:', r, exc_info=True) + continue + + image_id = res['ImageId'] + log.info('%s: Publishing to %s', r, image_id) + image = self.session(r).resource('ec2').Image(image_id) + + publishing[r] = image + + artifacts = {} + copy_wait = 180 + while len(artifacts) < len(publishing): + for r, image in publishing.items(): + if r not in artifacts: + image.reload() + if image.state == 'available': + # tag image + log.info('%s: Adding tags to %s', r, image.id) + image_tags = ImageTags(from_list=image.tags) + fresh = False + if 'published' not in image_tags: + fresh = True + + if fresh: + tags.published = datetime.utcnow().isoformat() + + tags.Name = tags.name # because AWS is special + image.create_tags(Tags=tags.as_list()) + + # tag image's snapshot, too + snapshot = self.session(r).resource('ec2').Snapshot( + image.block_device_mappings[0]['Ebs']['SnapshotId'] + ) + snapshot.create_tags(Tags=tags.as_list()) + + # update image description to match description in tags + log.info('%s: Updating description to %s', r, tags.description) + image.modify_attribute( + Description={'Value': tags.description}, + ) + + # apply launch perms + if perms['groups'] or perms['users']: + log.info('%s: Applying launch perms to %s', r, image.id) + image.reset_attribute(Attribute='launchPermission') + image.modify_attribute( + Attribute='launchPermission', + OperationType='add', + UserGroups=perms['groups'], + UserIds=perms['users'], + ) + + # set up AMI deprecation + ec2c = image.meta.client + log.info('%s: Setting EOL deprecation time on %s', r, image.id) + try: + ec2c.enable_image_deprecation( + ImageId=image.id, + DeprecateAt=f"{tags.end_of_life}T23:59:00Z" + ) + except Exception: + log.warning('Unable to set EOL Deprecation on %s image:', r, exc_info=True) + + artifacts[r] = image.id + + if image.state == 'failed': + log.error('%s: %s - %s - %s', r, image.id, image.state, image.state_reason) + artifacts[r] = None + + remaining = len(publishing) - len(artifacts) + if remaining > 0: + log.info('Waiting %ds for %d images to complete', copy_wait, remaining) + time.sleep(copy_wait) + copy_wait = 30 + + ic.artifacts = artifacts + + +def register(cloud, cred_provider=None): + return AWSCloudAdapter(cloud, cred_provider) diff --git a/clouds/azure.py b/clouds/azure.py new file mode 100644 index 0000000..4fda4b5 --- /dev/null +++ b/clouds/azure.py @@ -0,0 +1,22 @@ +from .interfaces.adapter import CloudAdapterInterface + +# NOTE: This stub allows images to be built locally and uploaded to storage, +# but code for automated importing and publishing of images for this cloud +# publisher has not yet been written. + +class AzureCloudAdapter(CloudAdapterInterface): + + def get_latest_imported_tags(self, project, image_key): + return None + + def import_image(self, ic): + pass + + def delete_image(self, config, image_id): + pass + + def publish_image(self, ic): + pass + +def register(cloud, cred_provider=None): + return AzureCloudAdapter(cloud, cred_provider) diff --git a/clouds/gcp.py b/clouds/gcp.py new file mode 100644 index 0000000..d58a581 --- /dev/null +++ b/clouds/gcp.py @@ -0,0 +1,22 @@ +from .interfaces.adapter import CloudAdapterInterface + +# NOTE: This stub allows images to be built locally and uploaded to storage, +# but code for automated importing and publishing of images for this cloud +# publisher has not yet been written. + +class GCPCloudAdapter(CloudAdapterInterface): + + def get_latest_imported_tags(self, project, image_key): + return None + + def import_image(self, ic): + pass + + def delete_image(self, config, image_id): + pass + + def publish_image(self, ic): + pass + +def register(cloud, cred_provider=None): + return GCPCloudAdapter(cloud, cred_provider) diff --git a/clouds/identity_broker_client.py b/clouds/identity_broker_client.py new file mode 100644 index 0000000..0465d82 --- /dev/null +++ b/clouds/identity_broker_client.py @@ -0,0 +1,135 @@ +# vim: ts=4 et: + +import json +import logging +import os +import sys +import time +import urllib.error + +from datetime import datetime +from email.utils import parsedate +from urllib.request import Request, urlopen + + +class IdentityBrokerClient: + """Client for identity broker + + Export IDENTITY_BROKER_ENDPOINT to override the default broker endpoint. + Export IDENTITY_BROKER_API_KEY to specify an API key for the broker. + + See README_BROKER.md for more information and a spec. + """ + + _DEFAULT_ENDPOINT = 'https://aws-access.crute.us/api/account' + _DEFAULT_ACCOUNT = 'alpine-amis-user' + _LOGFORMAT = '%(name)s - %(levelname)s - %(message)s' + + def __init__(self, endpoint=None, key=None, account=None, debug=False): + # log to STDOUT so that it's not all red when executed by Packer + self._logger = logging.getLogger('identity-broker') + self._logger.setLevel(logging.DEBUG if debug else logging.INFO) + console = logging.StreamHandler(sys.stdout) + console.setFormatter(logging.Formatter(self._LOGFORMAT)) + self._logger.addHandler(console) + + self._endpoint = os.environ.get('IDENTITY_BROKER_ENDPOINT') or endpoint \ + or self._DEFAULT_ENDPOINT + self._key = os.environ.get('IDENTITY_BROKER_API_KEY') or key + self._account = account or self._DEFAULT_ACCOUNT + if not self._key: + raise Exception('No identity broker key found') + + self._headers = { + 'Accept': 'application/vnd.broker.v2+json', + 'X-API-Key': self._key + } + self._cache = {} + self._expires = {} + self._default_region = {} + + def _is_cache_valid(self, path): + if path not in self._cache: + return False + + # path is subject to expiry AND its time has passed + if self._expires[path] and self._expires[path] < datetime.utcnow(): + return False + + return True + + def _get(self, path): + self._logger.debug("request: %s", path) + if not self._is_cache_valid(path): + while True: # to handle rate limits + try: + res = urlopen(Request(path, headers=self._headers)) + except urllib.error.HTTPError as ex: + if ex.status == 401: + raise Exception('Expired or invalid identity broker token') + + if ex.status == 406: + raise Exception('Invalid or malformed identity broker token') + + # TODO: will this be entirely handled by the 401 above? + if ex.headers.get('Location') == '/logout': + raise Exception('Identity broker token is expired') + + if ex.status == 429: + self._logger.warning( + 'Rate-limited by identity broker, sleeping 30 seconds') + time.sleep(30) + continue + + raise ex + + if res.status not in {200, 429}: + raise Exception(res.reason) + + # never expires without valid RFC 1123 Expires header + if expires := res.getheader('Expires'): + expires = parsedate(expires) + # convert RFC 1123 to datetime, if parsed successfully + expires = datetime(*expires[:6]) + + self._expires[path] = expires + self._cache[path] = json.load(res) + break + + self._logger.debug("response: %s", self._cache[path]) + return self._cache[path] + + def get_credentials_url(self, vendor): + accounts = self._get(self._endpoint) + if vendor not in accounts: + raise Exception(f'No {vendor} credentials found') + + for account in accounts[vendor]: + if account['short_name'] == self._account: + return account['credentials_url'] + + raise Exception('No account credentials found') + + def get_regions(self, vendor): + out = {} + + for region in self._get(self.get_credentials_url(vendor)): + if region['enabled']: + out[region['name']] = region['credentials_url'] + + if region['default']: + self._default_region[vendor] = region['name'] + + return out + + def get_default_region(self, vendor): + if vendor not in self._default_region: + self.get_regions(vendor) + + return self._default_region.get(vendor) + + def get_credentials(self, vendor, region=None): + if not region: + region = self.get_default_region(vendor) + + return self._get(self.get_regions(vendor)[region]) diff --git a/clouds/interfaces/__init__.py b/clouds/interfaces/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clouds/interfaces/adapter.py b/clouds/interfaces/adapter.py new file mode 100644 index 0000000..0ef6c55 --- /dev/null +++ b/clouds/interfaces/adapter.py @@ -0,0 +1,40 @@ +# vim: ts=4 et: + +class CloudAdapterInterface: + + def __init__(self, cloud, cred_provider=None): + self._sdk = None + self._sessions = {} + self.cloud = cloud + self.cred_provider = cred_provider + self._default_region = None + + @property + def sdk(self): + raise NotImplementedError + + @property + def regions(self): + raise NotImplementedError + + @property + def default_region(self): + raise NotImplementedError + + def credentials(self, region=None): + raise NotImplementedError + + def session(self, region=None): + raise NotImplementedError + + def get_latest_imported_tags(self, project, image_key): + raise NotImplementedError + + def import_image(self, config): + raise NotImplementedError + + def delete_image(self, config, image_id): + raise NotImplementedError + + def publish_image(self, config): + raise NotImplementedError diff --git a/clouds/nocloud.py b/clouds/nocloud.py new file mode 100644 index 0000000..8079319 --- /dev/null +++ b/clouds/nocloud.py @@ -0,0 +1,21 @@ +from .interfaces.adapter import CloudAdapterInterface + +# NOTE: NoCloud images are never imported or published because there's +# no actual cloud provider associated with them. + +class NoCloudAdapter(CloudAdapterInterface): + + def get_latest_imported_tags(self, project, image_key): + return None + + def import_image(self, ic): + pass + + def delete_image(self, config, image_id): + pass + + def publish_image(self, ic): + pass + +def register(cloud, cred_provider=None): + return NoCloudAdapter(cloud, cred_provider) diff --git a/clouds/oci.py b/clouds/oci.py new file mode 100644 index 0000000..78ffe0f --- /dev/null +++ b/clouds/oci.py @@ -0,0 +1,22 @@ +from .interfaces.adapter import CloudAdapterInterface + +# NOTE: This stub allows images to be built locally and uploaded to storage, +# but code for automated importing and publishing of images for this cloud +# publisher has not yet been written. + +class OCICloudAdapter(CloudAdapterInterface): + + def get_latest_imported_tags(self, project, image_key): + return None + + def import_image(self, ic): + pass + + def delete_image(self, config, image_id): + pass + + def publish_image(self, ic): + pass + +def register(cloud, cred_provider=None): + return OCICloudAdapter(cloud, cred_provider) diff --git a/configs/alpine.conf b/configs/alpine.conf new file mode 100644 index 0000000..65f533b --- /dev/null +++ b/configs/alpine.conf @@ -0,0 +1,105 @@ +# vim: ts=2 et: + +# NOTE: If you are using alpine-cloud-images to build public cloud images +# for something/someone other than Alpine Linux, you *MUST* override +# *AT LEAST* the 'project' setting with a unique identifier string value +# via a "config overlay" to avoid image import and publishing collisions. + +project = "https://alpinelinux.org/cloud" + +# all build configs start with these +Default { + project = ${project} + + # image name/description components + name = [ alpine ] + description = [ Alpine Linux ] + + motd { + welcome = "Welcome to Alpine!" + + wiki = [ + "The Alpine Wiki contains a large amount of how-to guides and general" + "information about administrating Alpine systems." + "See ." + ] + + release_notes = [ + "Alpine release notes:" + "* <{release_notes}>" + ] + } + + # initial provisioning script and data directory + scripts = [ setup ] + script_dirs = [ setup.d ] + + size = 1G + login = alpine + + image_format = qcow2 + + # these paths are subject to change, as image downloads are developed + storage_url = "ssh://tomalok@dev.alpinelinux.org/public_html/alpine-cloud-images/{v_version}/cloud/{cloud}/{arch}" + #storage_url = "file://~jake/tmp/alpine-cloud-images/{v_version}/cloud/{cloud}/{arch}" + download_url = "https://dev.alpinelinux.org/~tomalok/alpine-cloud-images/{v_version}/cloud/{cloud}/{arch}" # development + #download_url = "https://dl-cdn.alpinelinux.org/alpine/{v_version}/cloud/{cloud}/{arch}" + + # image access + access.PUBLIC = true + + # image publication + regions.ALL = true +} + +# profile build matrix +Dimensions { + version { + "3.17" { include required("version/3.17.conf") } + "3.16" { include required("version/3.16.conf") } + "3.15" { include required("version/3.15.conf") } + "3.14" { include required("version/3.14.conf") } + edge { include required("version/edge.conf") } + } + arch { + x86_64 { include required("arch/x86_64.conf") } + aarch64 { include required("arch/aarch64.conf") } + } + firmware { + bios { include required("firmware/bios.conf") } + uefi { include required("firmware/uefi.conf") } + } + bootstrap { + tiny { include required("bootstrap/tiny.conf") } + cloudinit { include required("bootstrap/cloudinit.conf") } + } + cloud { + aws { include required("cloud/aws.conf") } + nocloud { include required("cloud/nocloud.conf") } + # these are considered "alpha" + azure { include required("cloud/azure.conf") } + gcp { include required("cloud/gcp.conf") } + oci { include required("cloud/oci.conf") } + } +} + +# all build configs merge these at the very end +Mandatory { + name = [ "r{revision}" ] + description = [ "- https://alpinelinux.org/cloud" ] + encrypted = false + + # final motd message + motd.motd_change = "You may change this message by editing /etc/motd." + + # final provisioning script + scripts = [ cleanup ] + + # TODO: remove this after testing + #access.PUBLIC = false + #regions { + # ALL = false + # us-west-2 = true + # us-east-1 = true + #} +} diff --git a/configs/arch/aarch64.conf b/configs/arch/aarch64.conf new file mode 100644 index 0000000..dc4cfc3 --- /dev/null +++ b/configs/arch/aarch64.conf @@ -0,0 +1,15 @@ +# vim: ts=2 et: +name = [aarch64] +arch_name = aarch64 + +# aarch64 is UEFI only +EXCLUDE = [bios] + +qemu.machine_type = virt +qemu.args = [ + [-cpu, cortex-a57], + [-boot, d], + [-device, virtio-gpu-pci], + [-device, usb-ehci], + [-device, usb-kbd], +] diff --git a/configs/arch/x86_64.conf b/configs/arch/x86_64.conf new file mode 100644 index 0000000..1fdce65 --- /dev/null +++ b/configs/arch/x86_64.conf @@ -0,0 +1,6 @@ +# vim: ts=2 et: +name = [x86_64] +arch_name = x86_64 + +qemu.machine_type = null +qemu.args = null diff --git a/configs/bootstrap/cloudinit.conf b/configs/bootstrap/cloudinit.conf new file mode 100644 index 0000000..07abbcf --- /dev/null +++ b/configs/bootstrap/cloudinit.conf @@ -0,0 +1,17 @@ +# vim: ts=2 et: +name = [cloudinit] +bootstrap_name = cloud-init +bootstrap_url = "https://cloud-init.io" + +# start cloudinit images with 3.15 +EXCLUDE = ["3.12", "3.13", "3.14"] + +packages { + cloud-init = true + dhclient = true # offically supported, for now + openssh-server-pam = true + e2fsprogs-extra = true # for resize2fs +} +services.default.cloud-init-hotplugd = true + +scripts = [ setup-cloudinit ] diff --git a/configs/bootstrap/tiny.conf b/configs/bootstrap/tiny.conf new file mode 100644 index 0000000..59d96d8 --- /dev/null +++ b/configs/bootstrap/tiny.conf @@ -0,0 +1,35 @@ +# vim: ts=2 et: +name = [tiny] +bootstrap_name = Tiny Cloud +bootstrap_url = "https://gitlab.alpinelinux.org/alpine/cloud/tiny-cloud" + +services { + sysinit.tiny-cloud-early = true + default.tiny-cloud = true + default.tiny-cloud-final = true +} + +WHEN { + aws { + packages.tiny-cloud-aws = true + WHEN { + "3.12" { + # tiny-cloud-network requires ifupdown-ng (unavailable in 3.12) + packages.tiny-cloud-aws = null + services.sysinit.tiny-cloud-early = null + services.default.tiny-cloud = null + services.default.tiny-cloud-final = null + # fall back to tiny-ec2-bootstrap instead + packages.tiny-ec2-bootstrap = true + services.default.tiny-ec2-bootstrap = true + } + } + } + # other per-cloud packages + nocloud.packages.tiny-cloud-nocloud = true + azure.packages.tiny-cloud-azure = true + gcp.packages.tiny-cloud-gcp = true + oci.packages.tiny-cloud-oci = true +} + +scripts = [ setup-tiny ] diff --git a/configs/cloud/aws.conf b/configs/cloud/aws.conf new file mode 100644 index 0000000..234e1a7 --- /dev/null +++ b/configs/cloud/aws.conf @@ -0,0 +1,40 @@ +# vim: ts=2 et: +cloud_name = Amazon Web Services +image_format = vhd + +kernel_modules { + ena = true + nvme = true +} +kernel_options { + "nvme_core.io_timeout=4294967295" = true +} +initfs_features { + ena = true + nvme = true +} + +# TODO: what about IPv6-only networks? +# maybe we only set it for <= 3.17, and leave it to dhcpcd? +ntp_server = 169.254.169.123 + +access.PUBLIC = true +regions.ALL = true + +cloud_region_url = "https://{region}.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId={image_id}", +cloud_launch_url = "https://{region}.console.aws.amazon.com/ec2/home#launchAmi={image_id}" + +WHEN { + aarch64 { + # new AWS aarch64 default... + kernel_modules.gpio_pl061 = true + initfs_features.gpio_pl061 = true + WHEN { + "3.14 3.13 3.12" { + # ...but not supported for older versions + kernel_modules.gpio_pl061 = false + initfs_features.gpio_pl061 = false + } + } + } +} diff --git a/configs/cloud/azure.conf b/configs/cloud/azure.conf new file mode 100644 index 0000000..c83c286 --- /dev/null +++ b/configs/cloud/azure.conf @@ -0,0 +1,9 @@ +# vim: ts=2 et: +cloud_name = Microsoft Azure (alpha) +image_format = vhd + +# start with 3.18 +EXCLUDE = ["3.12", "3.13", "3.14", "3.15", "3.16", "3.17"] + +# TODO: https://learn.microsoft.com/en-us/azure/virtual-machines/linux/time-sync +ntp_server = "" diff --git a/configs/cloud/gcp.conf b/configs/cloud/gcp.conf new file mode 100644 index 0000000..053e545 --- /dev/null +++ b/configs/cloud/gcp.conf @@ -0,0 +1,15 @@ +# vim: ts=2 et: +cloud_name = Google Cloud Platform (alpha) +# TODO: https://cloud.google.com/compute/docs/import/importing-virtual-disks +# Mentions "VHD" but also mentions "..." if that also includes QCOW2, then +# we should use that instead. The "Manual Import" section on the sidebar +# has a "Manually import boot disks" subpage which also mentions importing +# compressed raw images... We would prefer to avoid that if possible. +image_format = vhd + +# start with 3.18 +EXCLUDE = ["3.12", "3.13", "3.14", "3.15", "3.16", "3.17"] + +# TODO: https://cloud.google.com/compute/docs/instances/configure-ntp +# (metadata.google.internal) +ntp_server = "" diff --git a/configs/cloud/nocloud.conf b/configs/cloud/nocloud.conf new file mode 100644 index 0000000..0ac44f4 --- /dev/null +++ b/configs/cloud/nocloud.conf @@ -0,0 +1,8 @@ +# vim: ts=2 et: +cloud_name = NoCloud +image_format = qcow2 + +# start with 3.18 +EXCLUDE = ["3.12", "3.13", "3.14", "3.15", "3.16", "3.17"] + +ntp_server = "" diff --git a/configs/cloud/oci.conf b/configs/cloud/oci.conf new file mode 100644 index 0000000..f9a6ad4 --- /dev/null +++ b/configs/cloud/oci.conf @@ -0,0 +1,8 @@ +# vim: ts=2 et: +cloud_name = Oracle Cloud Infrastructure (alpha) +image_format = qcow2 + +# start with 3.18 +EXCLUDE = ["3.12", "3.13", "3.14", "3.15", "3.16", "3.17"] + +ntp_server = "169.254.169.254" diff --git a/configs/firmware/bios.conf b/configs/firmware/bios.conf new file mode 100644 index 0000000..93f67c0 --- /dev/null +++ b/configs/firmware/bios.conf @@ -0,0 +1,7 @@ +# vim: ts=2 et: +name = [bios] +firmware_name = BIOS + +bootloader = extlinux +packages.syslinux = --no-scripts +qemu.firmware = null diff --git a/configs/firmware/uefi.conf b/configs/firmware/uefi.conf new file mode 100644 index 0000000..a1e514c --- /dev/null +++ b/configs/firmware/uefi.conf @@ -0,0 +1,18 @@ +# vim: ts=2 et: +name = [uefi] +firmware_name = UEFI + +bootloader = grub-efi +packages { + grub-efi = --no-scripts + dosfstools = true +} + +WHEN { + aarch64 { + qemu.firmware = work/firmware/uefi-aarch64.bin + } + x86_64 { + qemu.firmware = work/firmware/uefi-x86_64.bin + } +} diff --git a/configs/images.conf b/configs/images.conf new file mode 120000 index 0000000..99f2529 --- /dev/null +++ b/configs/images.conf @@ -0,0 +1 @@ +alpine.conf \ No newline at end of file diff --git a/configs/version/3.12.conf b/configs/version/3.12.conf new file mode 100644 index 0000000..9254d91 --- /dev/null +++ b/configs/version/3.12.conf @@ -0,0 +1,5 @@ +# vim: ts=2 et: + +include required("base/1.conf") + +# NOTE: EOL 2022-05-01 \ No newline at end of file diff --git a/configs/version/3.13.conf b/configs/version/3.13.conf new file mode 100644 index 0000000..9481943 --- /dev/null +++ b/configs/version/3.13.conf @@ -0,0 +1,3 @@ +# vim: ts=2 et: + +include required("base/2.conf") \ No newline at end of file diff --git a/configs/version/3.14.conf b/configs/version/3.14.conf new file mode 100644 index 0000000..9481943 --- /dev/null +++ b/configs/version/3.14.conf @@ -0,0 +1,3 @@ +# vim: ts=2 et: + +include required("base/2.conf") \ No newline at end of file diff --git a/configs/version/3.15.conf b/configs/version/3.15.conf new file mode 100644 index 0000000..a9f6475 --- /dev/null +++ b/configs/version/3.15.conf @@ -0,0 +1,7 @@ +# vim: ts=2 et: + +include required("base/3.conf") + +motd { + sudo_deprecated = "NOTE: 'sudo' has been deprecated, please use 'doas' instead." +} \ No newline at end of file diff --git a/configs/version/3.16.conf b/configs/version/3.16.conf new file mode 100644 index 0000000..29a6936 --- /dev/null +++ b/configs/version/3.16.conf @@ -0,0 +1,7 @@ +# vim: ts=2 et: + +include required("base/4.conf") + +motd { + sudo_removed = "NOTE: 'sudo' is no longer installed by default, please use 'doas' instead." +} diff --git a/configs/version/3.17.conf b/configs/version/3.17.conf new file mode 100644 index 0000000..7fc3e7d --- /dev/null +++ b/configs/version/3.17.conf @@ -0,0 +1,7 @@ +# vim: ts=2 et: + +include required("base/4.conf") + +motd { + sudo_removed = "NOTE: 'sudo' is not installed by default, please use 'doas' instead." +} diff --git a/configs/version/base/1.conf b/configs/version/base/1.conf new file mode 100644 index 0000000..9d95504 --- /dev/null +++ b/configs/version/base/1.conf @@ -0,0 +1,60 @@ +# vim: ts=2 et: + +repos { + "https://dl-cdn.alpinelinux.org/alpine/v{version}/main" = true + "https://dl-cdn.alpinelinux.org/alpine/v{version}/community" = true + "https://dl-cdn.alpinelinux.org/alpine/v{version}/testing" = false +} + +packages { + alpine-base = true + linux-virt = true + alpine-mirrors = true + chrony = true + e2fsprogs = true + openssh = true + sudo = true + tzdata = true +} + +services { + sysinit { + devfs = true + dmesg = true + hwdrivers = true + mdev = true + } + boot { + acpid = true + bootmisc = true + hostname = true + hwclock = true + modules = true + swap = true + sysctl = true + syslog = true + } + default { + chronyd = true + networking = true + sshd = true + } + shutdown { + killprocs = true + mount-ro = true + savecache = true + } +} + +kernel_modules { + sd-mod = true + usb-storage = true + ext4 = true +} + +kernel_options { + "console=ttyS0,115200n8" = true +} + +initfs_features { +} diff --git a/configs/version/base/2.conf b/configs/version/base/2.conf new file mode 100644 index 0000000..f6909c6 --- /dev/null +++ b/configs/version/base/2.conf @@ -0,0 +1,8 @@ +# vim: ts=2 et: + +include required("1.conf") + +packages { + # drop old alpine-mirrors + alpine-mirrors = null +} diff --git a/configs/version/base/3.conf b/configs/version/base/3.conf new file mode 100644 index 0000000..4373266 --- /dev/null +++ b/configs/version/base/3.conf @@ -0,0 +1,8 @@ +# vim: ts=2 et: + +include required("2.conf") + +packages { + # doas will officially replace sudo in 3.16 + doas = true +} diff --git a/configs/version/base/4.conf b/configs/version/base/4.conf new file mode 100644 index 0000000..6f2e978 --- /dev/null +++ b/configs/version/base/4.conf @@ -0,0 +1,8 @@ +# vim: ts=2 et: + +include required("3.conf") + +packages { + # doas officially replaces sudo in 3.16 + sudo = false +} diff --git a/configs/version/base/5.conf b/configs/version/base/5.conf new file mode 100644 index 0000000..f13f817 --- /dev/null +++ b/configs/version/base/5.conf @@ -0,0 +1,8 @@ +# vim: ts=2 et: + +include required("4.conf") + +packages { + # start using dhcpcd for improved IPv6 experience + dhcpcd = true +} diff --git a/configs/version/edge.conf b/configs/version/edge.conf new file mode 100644 index 0000000..8289861 --- /dev/null +++ b/configs/version/edge.conf @@ -0,0 +1,15 @@ +# vim: ts=2 et: + +include required("base/5.conf") + +motd { + sudo_removed = "NOTE: 'sudo' is not installed by default, please use 'doas' instead." +} + +# clear out inherited repos +repos = null +repos { + "https://dl-cdn.alpinelinux.org/alpine/edge/main" = true + "https://dl-cdn.alpinelinux.org/alpine/edge/community" = true + "https://dl-cdn.alpinelinux.org/alpine/edge/testing" = true +} diff --git a/gen_mksite_releases.py b/gen_mksite_releases.py new file mode 100755 index 0000000..b53016d --- /dev/null +++ b/gen_mksite_releases.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +# vim: ts=4 et: + +# TODO: perhaps integrate into "./build release" + +# Ensure we're using the Python virtual env with our installed dependencies +import os +import sys +import textwrap + +NOTE = textwrap.dedent(""" + This script's output provides a mustache-ready datasource to alpine-mksite + (https://gitlab.alpinelinux.org/alpine/infra/alpine-mksite) and should be + run after the main 'build' script has published ALL images. + STDOUT from this script should be saved as 'cloud/releases.yaml' in the + above alpine-mksite repo. + """) + +sys.pycache_prefix = 'work/__pycache__' + +if not os.path.exists('work'): + print('FATAL: Work directory does not exist.', file=sys.stderr) + print(NOTE, file=sys.stderr) + exit(1) + +# Re-execute using the right virtual environment, if necessary. +venv_args = [os.path.join('work', 'bin', 'python3')] + sys.argv +if os.path.join(os.getcwd(), venv_args[0]) != sys.executable: + print("Re-executing with work environment's Python...\n", file=sys.stderr) + os.execv(venv_args[0], venv_args) + +# We're now in the right Python environment + +import argparse +import logging + +from collections import defaultdict +from ruamel.yaml import YAML + +import clouds +from image_config_manager import ImageConfigManager + + +### Constants & Variables + +LOGFORMAT = '%(name)s - %(levelname)s - %(message)s' + + +### Functions + +# allows us to set values deep within an object that might not be fully defined +def dictfactory(): + return defaultdict(dictfactory) + + +# undo dictfactory() objects to normal objects +def undictfactory(o): + if isinstance(o, defaultdict): + o = {k: undictfactory(v) for k, v in o.items()} + return o + + +### Command Line & Logging + +parser = argparse.ArgumentParser(description=NOTE) +parser.add_argument( + '--use-broker', action='store_true', + help='use the identity broker to get credentials') +parser.add_argument('--debug', action='store_true', help='enable debug output') +args = parser.parse_args() + +log = logging.getLogger('gen_mksite_releases') +log.setLevel(logging.DEBUG if args.debug else logging.INFO) +console = logging.StreamHandler(sys.stderr) +console.setFormatter(logging.Formatter(LOGFORMAT)) +log.addHandler(console) +log.debug(args) + +# set up credential provider, if we're going to use it +if args.use_broker: + clouds.set_credential_provider() + +# load build configs +configs = ImageConfigManager( + conf_path='work/configs/images.conf', + yaml_path='work/images.yaml', + log='gen_mksite_releases' +) +# make sure images.yaml is up-to-date with reality +configs.refresh_state('final') + +yaml = YAML() + +filters = dictfactory() +versions = dictfactory() +data = {} + +log.info('Transforming image data') +for i_key, i_cfg in configs.get().items(): + if not i_cfg.published: + continue + + version = i_cfg.version + if version == 'edge': + continue + + image_name = i_cfg.image_name + release = i_cfg.release + arch = i_cfg.arch + firmware = i_cfg.firmware + bootstrap = i_cfg.bootstrap + cloud = i_cfg.cloud + + if cloud not in filters['clouds']: + filters['clouds'][cloud] = { + 'cloud': cloud, + 'cloud_name': i_cfg.cloud_name, + } + + filters['regions'] = {} + + if arch not in filters['archs']: + filters['archs'][arch] = { + 'arch': arch, + 'arch_name': i_cfg.arch_name, + } + + if firmware not in filters['firmwares']: + filters['firmwares'][firmware] = { + 'firmware': firmware, + 'firmware_name': i_cfg.firmware_name, + } + + if bootstrap not in filters['bootstraps']: + filters['bootstraps'][bootstrap] = { + 'bootstrap': bootstrap, + 'bootstrap_name': i_cfg.bootstrap_name, + } + + if i_cfg.artifacts: + for region, image_id in {r: i_cfg.artifacts[r] for r in sorted(i_cfg.artifacts)}.items(): + if region not in filters['regions']: + filters['regions'][region] = { + 'region': region, + 'clouds': [cloud], + } + + if cloud not in filters['regions'][region]['clouds']: + filters['regions'][region]['clouds'].append(cloud) + + versions[version] |= { + 'version': version, + 'release': release, + 'end_of_life': i_cfg.end_of_life, + } + versions[version]['images'][image_name] |= { + 'image_name': image_name, + 'arch': arch, + 'firmware': firmware, + 'bootstrap': bootstrap, + 'published': i_cfg.published.split('T')[0], # just the date + } + versions[version]['images'][image_name]['downloads'][cloud] |= { + 'cloud': cloud, + 'image_format': i_cfg.image_format, + 'image_url': i_cfg.download_url + '/' + (i_cfg.image_name) + } + versions[version]['images'][image_name]['regions'][region] |= { + 'cloud': cloud, + 'region': region, + 'region_url': i_cfg.region_url(region, image_id), + 'launch_url': i_cfg.launch_url(region, image_id), + } + +log.info('Making data mustache-compatible') + +# convert filters to mustache-compatible format +data['filters'] = {} +for f in ['clouds', 'regions', 'archs', 'firmwares', 'bootstraps']: + data['filters'][f] = [ + filters[f][k] for k in filters[f] # order as they appear in work/images.yaml + ] + +for r in data['filters']['regions']: + c = r.pop('clouds') + r['clouds'] = [{'cloud': v} for v in c] + +# convert versions to mustache-compatible format +data['versions'] = [] +versions = undictfactory(versions) +for version in sorted(versions, reverse=True, key=lambda s: [int(u) for u in s.split('.')]): + images = versions[version].pop('images') + i = [] + for image_name in images: # order as they appear in work/images.yaml + downloads = images[image_name].pop('downloads') + d = [] + for download in downloads: + d.append(downloads[download]) + + images[image_name]['downloads'] = d + + regions = images[image_name].pop('regions') + r = [] + for region in sorted(regions): + r.append(regions[region]) + + images[image_name]['regions'] = r + i.append(images[image_name]) + + versions[version]['images'] = i + data['versions'].append(versions[version]) + +log.info('Dumping YAML') +yaml.dump(data, sys.stdout) +log.info('Done') diff --git a/image_config.py b/image_config.py new file mode 100644 index 0000000..59dd3bd --- /dev/null +++ b/image_config.py @@ -0,0 +1,465 @@ +# vim: ts=4 et: + +import hashlib +import mergedeep +import os +import pyhocon +import shutil + +from copy import deepcopy +from datetime import datetime +from pathlib import Path + +import clouds +from image_storage import ImageStorage, run +from image_tags import ImageTags + + +class ImageConfig(): + + CONVERT_CMD = { + 'qcow2': ['ln', '-f'], + 'vhd': ['qemu-img', 'convert', '-f', 'qcow2', '-O', 'vpc', '-o', 'force_size=on'], + } + # these tags may-or-may-not exist at various times + OPTIONAL_TAGS = [ + 'built', 'uploaded', 'imported', 'import_id', 'import_region', 'published', 'released' + ] + STEPS = [ + 'local', 'upload', 'import', 'publish', 'release' + ] + + def __init__(self, config_key, obj={}, log=None, yaml=None): + self._log = log + self._yaml = yaml + self._storage = None + self.config_key = str(config_key) + tags = obj.pop('tags', None) + self.__dict__ |= self._deep_dict(obj) + # ensure tag values are str() when loading + if tags: + self.tags = tags + + @classmethod + def to_yaml(cls, representer, node): + d = {} + for k in node.__dict__: + # don't serialize attributes starting with _ + if k.startswith('_'): + continue + + d[k] = node.__getattribute__(k) + + return representer.represent_mapping('!ImageConfig', d) + + @property + def v_version(self): + return 'edge' if self.version == 'edge' else 'v' + self.version + + @property + def local_dir(self): + return Path('work/images') / self.cloud / self.image_key + + @property + def local_image(self): + return self.local_dir / ('image.qcow2') + + @property + def image_name(self): + return self.name.format(**self.__dict__) + + @property + def image_description(self): + return self.description.format(**self.__dict__) + + @property + def image_file(self): + return '.'.join([self.image_name, self.image_format]) + + @property + def image_path(self): + return self.local_dir / self.image_file + + @property + def metadata_file(self): + return '.'.join([self.image_name, 'yaml']) + + def region_url(self, region, image_id): + return self.cloud_region_url.format(region=region, image_id=image_id, **self.__dict__) + + def launch_url(self, region, image_id): + return self.cloud_launch_url.format(region=region, image_id=image_id, **self.__dict__) + + @property + def tags(self): + # stuff that really ought to be there + t = { + 'arch': self.arch, + 'bootstrap': self.bootstrap, + 'cloud': self.cloud, + 'description': self.image_description, + 'end_of_life': self.end_of_life, + 'firmware': self.firmware, + 'image_key': self.image_key, + 'name': self.image_name, + 'project': self.project, + 'release': self.release, + 'revision': self.revision, + 'version': self.version + } + # stuff that might not be there yet + for k in self.OPTIONAL_TAGS: + if self.__dict__.get(k, None): + t[k] = self.__dict__[k] + + return ImageTags(t) + + # recursively convert a ConfigTree object to a dict object + def _deep_dict(self, layer): + obj = deepcopy(layer) + if isinstance(layer, pyhocon.ConfigTree): + obj = dict(obj) + + try: + for key, value in layer.items(): + # some HOCON keys are quoted to preserve dots + if '"' in key: + obj.pop(key) + key = key.strip('"') + + # version values were HOCON keys at one point, too + if key == 'version' and '"' in value: + value = value.strip('"') + + obj[key] = self._deep_dict(value) + except AttributeError: + pass + + return obj + + def _merge(self, obj={}): + mergedeep.merge(self.__dict__, self._deep_dict(obj), strategy=mergedeep.Strategy.ADDITIVE) + + def _get(self, attr, default=None): + return self.__dict__.get(attr, default) + + def _pop(self, attr, default=None): + return self.__dict__.pop(attr, default) + + # make data ready for Packer ingestion + def _normalize(self): + # stringify arrays + self.name = '-'.join(self.name) + self.description = ' '.join(self.description) + self.repo_keys = ' '.join(self.repo_keys) + self._resolve_motd() + self._resolve_urls() + self._stringify_repos() + self._stringify_packages() + self._stringify_services() + self._stringify_dict_keys('kernel_modules', ',') + self._stringify_dict_keys('kernel_options', ' ') + self._stringify_dict_keys('initfs_features', ' ') + + def _resolve_motd(self): + # merge release notes, as apporpriate + if 'release_notes' not in self.motd or not self.release_notes: + self.motd.pop('release_notes', None) + + motd = {} + for k, v in self.motd.items(): + if v is None: + continue + + # join list values with newlines + if type(v) is list: + v = "\n".join(v) + + motd[k] = v + + self.motd = '\n\n'.join(motd.values()).format(**self.__dict__) + + def _resolve_urls(self): + if 'storage_url' in self.__dict__: + self.storage_url = self.storage_url.format(v_version=self.v_version, **self.__dict__) + + if 'download_url' in self.__dict__: + self.download_url = self.download_url.format(v_version=self.v_version, **self.__dict__) + + def _stringify_repos(self): + # stringify repos map + # : # @ enabled + # : false # disabled (commented out) + # : true # enabled + # : null # skip entirely + # ...and interpolate {version} + self.repos = "\n".join(filter(None, ( + f"@{v} {r}" if isinstance(v, str) else + f"#{r}" if v is False else + r if v is True else None + for r, v in self.repos.items() + ))).format(version=self.version) + + def _stringify_packages(self): + # resolve/stringify packages map + # : true # add + # : # add @ + # : --no-scripts # add --no-scripts + # : --no-scripts # add --no-scripts @ + # : false # del + # : null # skip explicit add/del + pkgs = {'add': '', 'del': '', 'noscripts': ''} + for p, v in self.packages.items(): + k = 'add' + if isinstance(v, str): + if '--no-scripts' in v: + k = 'noscripts' + v = v.replace('--no-scripts', '') + v = v.strip() + if len(v): + p += f"@{v}" + elif v is False: + k = 'del' + elif v is None: + continue + + pkgs[k] = p if len(pkgs[k]) == 0 else pkgs[k] + ' ' + p + + self.packages = pkgs + + def _stringify_services(self): + # stringify services map + # : + # : true # enable at + # : false # disable at + # : null # skip explicit en/disable at + self.services = { + 'enable': ' '.join(filter(lambda x: not x.endswith('='), ( + '{}={}'.format(lvl, ','.join(filter(None, ( + s if v is True else None + for s, v in svcs.items() + )))) + for lvl, svcs in self.services.items() + ))), + 'disable': ' '.join(filter(lambda x: not x.endswith('='), ( + '{}={}'.format(lvl, ','.join(filter(None, ( + s if v is False else None + for s, v in svcs.items() + )))) + for lvl, svcs in self.services.items() + ))) + } + + def _stringify_dict_keys(self, d, sep): + self.__dict__[d] = sep.join(filter(None, ( + m if v is True else None + for m, v in self.__dict__[d].items() + ))) + + def _is_step_or_earlier(self, s, step): + log = self._log + if step == 'state': + return True + + if step not in self.STEPS: + return False + + return self.STEPS.index(s) <= self.STEPS.index(step) + + + # TODO: this needs to be sorted out for 'upload' and 'release' steps + def refresh_state(self, step, revise=False): + log = self._log + actions = {} + revision = 0 + step_state = step == 'state' + step_rollback = step == 'rollback' + undo = {} + + # enable initial set of possible actions based on specified step + for s in self.STEPS: + if self._is_step_or_earlier(s, step): + actions[s] = True + + # pick up any updated image metadata + self.load_metadata() + + # TODO: check storage and/or cloud - use this instead of remote_image + # latest_revision = self.get_latest_revision() + + if (step_rollback or revise) and self.local_image.exists(): + undo['local'] = True + + + + if step_rollback: + if self.local_image.exists(): + undo['local'] = True + + if not self.published or self.released: + if self.uploaded: + undo['upload'] = True + + if self.imported: + undo['import'] = True + + # TODO: rename to 'remote_tags'? + # if we load remote tags into state automatically, shouldn't that info already be in self? + remote_image = clouds.get_latest_imported_tags(self) + log.debug('\n%s', remote_image) + + if revise: + if self.local_image.exists(): + # remove previously built local image artifacts + log.warning('%s existing local image dir %s', + 'Would remove' if step_state else 'Removing', + self.local_dir) + if not step_state: + shutil.rmtree(self.local_dir) + + if remote_image and remote_image.get('published', None): + log.warning('%s image revision for %s', + 'Would bump' if step_state else 'Bumping', + self.image_key) + revision = int(remote_image.revision) + 1 + + elif remote_image and remote_image.get('imported', None): + # remove existing imported (but unpublished) image + log.warning('%s unpublished remote image %s', + 'Would remove' if step_state else 'Removing', + remote_image.import_id) + if not step_state: + clouds.delete_image(self, remote_image.import_id) + + remote_image = None + + elif remote_image: + if remote_image.get('imported', None): + # already imported, don't build/upload/import again + log.debug('%s - already imported', self.image_key) + actions.pop('local', None) + actions.pop('upload', None) + actions.pop('import', None) + + if remote_image.get('published', None): + # NOTE: re-publishing can update perms or push to new regions + log.debug('%s - already published', self.image_key) + + if self.local_image.exists(): + # local image's already built, don't rebuild + log.debug('%s - already locally built', self.image_key) + actions.pop('local', None) + + else: + self.built = None + + # merge remote_image data into image state + if remote_image: + self.__dict__ |= dict(remote_image) + + else: + self.__dict__ |= { + 'revision': revision, + 'uploaded': None, + 'imported': None, + 'import_id': None, + 'import_region': None, + 'published': None, + 'artifacts': None, + 'released': None, + } + + # remove remaining actions not possible based on specified step + for s in self.STEPS: + if not self._is_step_or_earlier(s, step): + actions.pop(s, None) + + self.actions = list(actions) + log.info('%s/%s = %s', self.cloud, self.image_name, self.actions) + + self.state_updated = datetime.utcnow().isoformat() + + @property + def storage(self): + if self._storage is None: + self._storage = ImageStorage(self.local_dir, self.storage_url, log=self._log) + + return self._storage + + def _save_checksum(self, file): + self._log.info("Calculating checksum for '%s'", file) + sha256_hash = hashlib.sha256() + sha512_hash = hashlib.sha512() + with open(file, 'rb') as f: + for block in iter(lambda: f.read(4096), b''): + sha256_hash.update(block) + sha512_hash.update(block) + + with open(str(file) + '.sha256', 'w') as f: + print(sha256_hash.hexdigest(), file=f) + + with open(str(file) + '.sha512', 'w') as f: + print(sha512_hash.hexdigest(), file=f) + + # convert local QCOW2 to format appropriate for a cloud + def convert_image(self): + self._log.info('Converting %s to %s', self.local_image, self.image_path) + run( + self.CONVERT_CMD[self.image_format] + [self.local_image, self.image_path], + log=self._log, errmsg='Unable to convert %s to %s', + errvals=[self.local_image, self.image_path] + ) + self._save_checksum(self.image_path) + self.built = datetime.utcnow().isoformat() + + def upload_image(self): + self.storage.store( + self.image_file, + self.image_file + '.sha256', + self.image_file + '.sha512' + ) + self.uploaded = datetime.utcnow().isoformat() + + def save_metadata(self, action): + os.makedirs(self.local_dir, exist_ok=True) + self._log.info('Saving image metadata') + # TODO: save metadata updated timestamp as metadata? + # TODO: def self.metadata to return what we consider metadata? + metadata = dict(self.tags) + self.metadata_updated = datetime.utcnow().isoformat() + metadata |= { + 'artifacts': self._get('artifacts', None), + 'metadata_updated': self.metadata_updated + } + metadata_path = self.local_dir / self.metadata_file + self._yaml.dump(metadata, metadata_path) + self._save_checksum(metadata_path) + if action != 'local' and self.storage: + self.storage.store( + self.metadata_file, + self.metadata_file + '.sha256', + self.metadata_file + '.sha512' + ) + + def load_metadata(self): + # TODO: what if we have fresh configs, but the image is already uploaded/imported? + # we'll need to get revision first somehow + if 'revision' not in self.__dict__: + return + + # TODO: revision = '*' for now - or only if unknown? + + # get a list of local matching -r*.yaml? + metadata_path = self.local_dir / self.metadata_file + if metadata_path.exists(): + self._log.info('Loading image metadata from %s', metadata_path) + self.__dict__ |= self._yaml.load(metadata_path).items() + + # get a list of storage matching -r*.yaml + #else: + # retrieve metadata (and image?) from storage_url + # else: + # retrieve metadata from imported image + + # if there's no stored metadata, we are in transition, + # get a list of imported images matching -r*.yaml diff --git a/image_config_manager.py b/image_config_manager.py new file mode 100644 index 0000000..704aaa2 --- /dev/null +++ b/image_config_manager.py @@ -0,0 +1,178 @@ +# vim: ts=4 et: + +import itertools +import logging +import pyhocon + +from copy import deepcopy +from datetime import datetime +from pathlib import Path +from ruamel.yaml import YAML + +from image_config import ImageConfig + + + +class ImageConfigManager(): + + def __init__(self, conf_path, yaml_path, log=__name__, alpine=None): + self.conf_path = Path(conf_path) + self.yaml_path = Path(yaml_path) + self.log = logging.getLogger(log) + self.alpine = alpine + + self.now = datetime.utcnow() + self._configs = {} + + self.yaml = YAML() + self.yaml.register_class(ImageConfig) + self.yaml.explicit_start = True + # hide !ImageConfig tag from Packer + self.yaml.representer.org_represent_mapping = self.yaml.representer.represent_mapping + self.yaml.representer.represent_mapping = self._strip_yaml_tag_type + + # load resolved YAML, if exists + if self.yaml_path.exists(): + self._load_yaml() + else: + self._resolve() + + def get(self, key=None): + if not key: + return self._configs + + return self._configs[key] + + # load already-resolved YAML configs, restoring ImageConfig objects + def _load_yaml(self): + self.log.info('Loading existing %s', self.yaml_path) + for key, config in self.yaml.load(self.yaml_path).items(): + self._configs[key] = ImageConfig(key, config, log=self.log, yaml=self.yaml) + + # save resolved configs to YAML + def _save_yaml(self): + self.log.info('Saving %s', self.yaml_path) + self.yaml.dump(self._configs, self.yaml_path) + + # hide !ImageConfig tag from Packer + def _strip_yaml_tag_type(self, tag, mapping, flow_style=None): + if tag == '!ImageConfig': + tag = u'tag:yaml.org,2002:map' + + return self.yaml.representer.org_represent_mapping(tag, mapping, flow_style=flow_style) + + # resolve from HOCON configs + def _resolve(self): + self.log.info('Generating configs.yaml in work environment') + cfg = pyhocon.ConfigFactory.parse_file(self.conf_path) + # set version releases + for v, vcfg in cfg.Dimensions.version.items(): + # version keys are quoted to protect dots + self._set_version_release(v.strip('"'), vcfg) + + dimensions = list(cfg.Dimensions.keys()) + self.log.debug('dimensions: %s', dimensions) + + for dim_keys in (itertools.product(*cfg['Dimensions'].values())): + config_key = '-'.join(dim_keys).replace('"', '') + + # dict of dimension -> dimension_key + dim_map = dict(zip(dimensions, dim_keys)) + + # replace version with release, and make image_key from that + release = cfg.Dimensions.version[dim_map['version']].release + (rel_map := dim_map.copy())['version'] = release + image_key = '-'.join(rel_map.values()) + + image_config = ImageConfig( + config_key, + { + 'image_key': image_key, + 'release': release + } | dim_map, + log=self.log, + yaml=self.yaml + ) + + # merge in the Default config + image_config._merge(cfg.Default) + skip = False + # merge in each dimension key's configs + for dim, dim_key in dim_map.items(): + dim_cfg = deepcopy(cfg.Dimensions[dim][dim_key]) + + image_config._merge(dim_cfg) + + # now that we're done with ConfigTree/dim_cfg, remove " from dim_keys + dim_keys = set(k.replace('"', '') for k in dim_keys) + + # WHEN blocks inside WHEN blocks are considered "and" operations + while (when := image_config._pop('WHEN', None)): + for when_keys, when_conf in when.items(): + # WHEN keys with spaces are considered "or" operations + if len(set(when_keys.split(' ')) & dim_keys) > 0: + image_config._merge(when_conf) + + exclude = image_config._pop('EXCLUDE', None) + if exclude and set(exclude) & set(dim_keys): + self.log.debug('%s SKIPPED, %s excludes %s', config_key, dim_key, exclude) + skip = True + break + + if eol := image_config._get('end_of_life', None): + if self.now > datetime.fromisoformat(eol): + self.log.warning('%s SKIPPED, %s end_of_life %s', config_key, dim_key, eol) + skip = True + break + + if skip is True: + continue + + # merge in the Mandatory configs at the end + image_config._merge(cfg.Mandatory) + + # clean stuff up + image_config._normalize() + image_config.qemu['iso_url'] = self.alpine.virt_iso_url(arch=image_config.arch) + + # we've resolved everything, add tags attribute to config + self._configs[config_key] = image_config + + self._save_yaml() + + # set current version release + def _set_version_release(self, v, c): + info = self.alpine.version_info(v) + c.put('release', info['release']) + c.put('end_of_life', info['end_of_life']) + c.put('release_notes', info['notes']) + + # release is also appended to name & description arrays + c.put('name', [c.release]) + c.put('description', [c.release]) + + # update current config status + def refresh_state(self, step, only=[], skip=[], revise=False): + self.log.info('Refreshing State') + has_actions = False + for ic in self._configs.values(): + # clear away any previous actions + if hasattr(ic, 'actions'): + delattr(ic, 'actions') + + dim_keys = set(ic.config_key.split('-')) + if only and len(set(only) & dim_keys) != len(only): + self.log.debug("%s SKIPPED, doesn't match --only", ic.config_key) + continue + + if skip and len(set(skip) & dim_keys) > 0: + self.log.debug('%s SKIPPED, matches --skip', ic.config_key) + continue + + ic.refresh_state(step, revise) + if not has_actions and len(ic.actions): + has_actions = True + + # re-save with updated actions + self._save_yaml() + return has_actions diff --git a/image_storage.py b/image_storage.py new file mode 100644 index 0000000..c4ea602 --- /dev/null +++ b/image_storage.py @@ -0,0 +1,183 @@ +# vim: ts=4 et: + +import shutil +import os + +from glob import glob +from pathlib import Path +from subprocess import Popen, PIPE +from urllib.parse import urlparse + +from image_tags import DictObj + + +def run(cmd, log, errmsg=None, errvals=[]): + # ensure command and error values are lists of strings + cmd = [str(c) for c in cmd] + errvals = [str(ev) for ev in errvals] + + log.debug('COMMAND: %s', ' '.join(cmd)) + p = Popen(cmd, stdout=PIPE, stdin=PIPE, encoding='utf8') + out, err = p.communicate() + if p.returncode: + if errmsg: + log.error(errmsg, *errvals) + + log.error('COMMAND: %s', ' '.join(cmd)) + log.error('EXIT: %d', p.returncode) + log.error('STDOUT:\n%s', out) + log.error('STDERR:\n%s', err) + raise RuntimeError + + return out, err + + +class ImageStorage(): + + def __init__(self, local, storage_url, log): + self.log = log + self.local = local + self.url = storage_url.removesuffix('/') + url = urlparse(self.url) + if url.scheme not in ['', 'file', 'ssh']: + self.log.error('Storage with "%s" scheme is unsupported', url.scheme) + raise RuntimeError + + if url.scheme in ['', 'file']: + self.scheme = 'file' + self.remote = Path(url.netloc + url.path).expanduser() + + else: + self.scheme = 'ssh' + self.host = url.hostname + self.remote = Path(url.path[1:]) # drop leading / -- use // for absolute path + self.ssh = DictObj({ + 'port': ['-p', url.port] if url.port else [], + 'user': ['-l', url.username] if url.username else [], + }) + self.scp = DictObj({ + 'port': ['-P', url.port] if url.port else [], + 'user': url.username + '@' if url.username else '', + }) + + def store(self, *files): + log = self.log + if not files: + log.debug('No files to store') + return + + src = self.local + dest = self.remote + if self.scheme == 'file': + dest.mkdir(parents=True, exist_ok=True) + for file in files: + log.info('Storing %s', dest / file) + shutil.copy2(src / file, dest / file) + + return + + url = self.url + host = self.host + ssh = self.ssh + scp = self.scp + run( + ['ssh'] + ssh.port + ssh.user + [host, 'mkdir', '-p', dest], + log=log, errmsg='Unable to ensure existence of %s', errvals=[url] + ) + src_files = [] + for file in files: + log.info('Storing %s', url + '/' + file) + src_files.append(src / file) + + run( + ['scp'] + scp.port + src_files + [scp.user + ':'.join([host, str(dest)])], + log=log, errmsg='Failed to store files' + ) + + def retrieve(self, *files): + log = self.log + if not files: + log.debug('No files to retrieve') + return + + src = self.remote + dest = self.local + dest.mkdir(parents=True, exist_ok=True) + if self.scheme == 'file': + for file in files: + log.info('Retrieving %s', src / file) + shutil.copy2(src / file, dest / file) + + return + + url = self.url + host = self.host + scp = self.scp + src_files = [] + for file in files: + log.info('Retrieving %s', url + '/' + file) + src_files.append(scp.user + ':'.join([host, str(src / file)])) + + run( + ['scp'] + scp.port + src_files + [dest], + log=log, errmsg='Failed to retrieve files' + ) + + # TODO: optional files=[]? + def list(self, match=None): + log = self.log + path = self.remote + if not match: + match = '*' + + files = [] + if self.scheme == 'file': + path.mkdir(parents=True, exist_ok=True) + log.info('Listing of %s files in %s', match, path) + files = sorted(glob(str(path / match)), key=os.path.getmtime, reverse=True) + + else: + url = self.url + host = self.host + ssh = self.ssh + log.info('Listing %s files at %s', match, url) + run( + ['ssh'] + ssh.port + ssh.user + [host, 'mkdir', '-p', path], + log=log, errmsg='Unable to create path' + ) + out, _ = run( + ['ssh'] + ssh.port + ssh.user + [host, 'ls', '-1drt', path / match], + log=log, errmsg='Failed to list files' + ) + files = out.splitlines() + + return [os.path.basename(f) for f in files] + + def remove(self, files): + log = self.log + if not files: + log.debug('No files to remove') + return + + dest = self.remote + if self.scheme == 'file': + for file in files: + path = dest / file + log.info('Removing %s', path) + if path.exists(): + path.unlink() + + return + + url = self.url + host = self.host + ssh = self.ssh + dest_files = [] + for file in files: + log.info('Removing %s', url + '/' + file) + dest_files.append(dest / file) + + run( + ['ssh'] + ssh.port + ssh.user + [host, 'rm', '-f'] + dest_files, + log=log, errmsg='Failed to remove files' + ) diff --git a/image_tags.py b/image_tags.py new file mode 100644 index 0000000..3d7219a --- /dev/null +++ b/image_tags.py @@ -0,0 +1,32 @@ +# vim: ts=4 et: + +class DictObj(dict): + + def __getattr__(self, key): + return self[key] + + def __setattr__(self, key, value): + self[key] = value + + def __delattr__(self, key): + del self[key] + + +class ImageTags(DictObj): + + def __init__(self, d={}, from_list=None, key_name='Key', value_name='Value'): + for key, value in d.items(): + self.__setattr__(key, value) + + if from_list: + self.from_list(from_list, key_name, value_name) + + def __setattr__(self, key, value): + self[key] = str(value) + + def as_list(self, key_name='Key', value_name='Value'): + return [{key_name: k, value_name: v} for k, v in self.items()] + + def from_list(self, list=[], key_name='Key', value_name='Value'): + for tag in list: + self.__setattr__(tag[key_name], tag[value_name]) diff --git a/overlays/testing/configs/alpine-testing.conf b/overlays/testing/configs/alpine-testing.conf new file mode 100644 index 0000000..810896b --- /dev/null +++ b/overlays/testing/configs/alpine-testing.conf @@ -0,0 +1,43 @@ +# vim: ts=2 et: + +# Overlay for testing alpine-cloud-images + +# start with the production alpine config +include required("alpine.conf") + +# override specific things... + +project = alpine-cloud-images__test + +Default { + # unset before resetting + name = null + name = [ test ] + description = null + description = [ Alpine Test ] +} + +Dimensions { + cloud { + # add a machine type dimension + machine { + vm { include required("machine/vm.conf") } + metal { include required("machine/metal.conf") } + } + # just test in these regions + aws.regions { + us-west-2 = true + us-east-1 = true + } + # adapters need to be written + #oci { include required("testing/oci.conf") } + #gcp { include required("testing/gcp.conf") } + #azure { include required("testing/azure.conf") } + #generic + #nocloud + } +} + +# test in private, and only in regions specified above +Mandatory.access.PUBLIC = false +Mandatory.regions.ALL = false diff --git a/overlays/testing/configs/images.conf b/overlays/testing/configs/images.conf new file mode 120000 index 0000000..cc0f93d --- /dev/null +++ b/overlays/testing/configs/images.conf @@ -0,0 +1 @@ +alpine-testing.conf \ No newline at end of file diff --git a/overlays/testing/configs/machine/metal.conf b/overlays/testing/configs/machine/metal.conf new file mode 100644 index 0000000..ec81b5e --- /dev/null +++ b/overlays/testing/configs/machine/metal.conf @@ -0,0 +1,9 @@ +# bare metal + +name = ["metal"] +machine_name = "Bare Metal" + +packages.linux-virt = null +packages.linux-lts = true + +# TODO: other kernel_modules, kernel_options, or initfs_features? diff --git a/overlays/testing/configs/machine/vm.conf b/overlays/testing/configs/machine/vm.conf new file mode 100644 index 0000000..b36bd5a --- /dev/null +++ b/overlays/testing/configs/machine/vm.conf @@ -0,0 +1,4 @@ +#name = [vm] # don't append anything to the name +machine_name = "Virtual" + +# all image defaults are for virutal machines diff --git a/overlays/testing/configs/testing/oci.conf b/overlays/testing/configs/testing/oci.conf new file mode 100644 index 0000000..0e80256 --- /dev/null +++ b/overlays/testing/configs/testing/oci.conf @@ -0,0 +1,4 @@ +# vim: ts=2 et: +builder = qemu + +# TBD \ No newline at end of file diff --git a/scripts/cleanup b/scripts/cleanup new file mode 100644 index 0000000..5fd23f1 --- /dev/null +++ b/scripts/cleanup @@ -0,0 +1,42 @@ +#!/bin/sh -eu +# vim: ts=4 et: + +[ -z "$DEBUG" ] || [ "$DEBUG" = 0 ] || set -x + +export \ + TARGET=/mnt + + +die() { + printf '\033[1;7;31m FATAL: %s \033[0m\n' "$@" >&2 # bold reversed red + exit 1 +} +einfo() { + printf '\n\033[1;7;36m> %s <\033[0m\n' "$@" >&2 # bold reversed cyan +} + +cleanup() { + # Sweep cruft out of the image that doesn't need to ship or will be + # re-generated when the image boots + rm -f \ + "$TARGET/var/cache/apk/"* \ + "$TARGET/etc/resolv.conf" \ + "$TARGET/root/.ash_history" \ + "$TARGET/etc/"*- + + # unmount extra EFI mount + if [ "$FIRMWARE" = uefi ]; then + umount "$TARGET/boot/efi" + fi + + umount \ + "$TARGET/dev" \ + "$TARGET/proc" \ + "$TARGET/sys" + + umount "$TARGET" +} + +einfo "Cleaning up and unmounting image volume..." +cleanup +einfo "Done!" diff --git a/scripts/setup b/scripts/setup new file mode 100755 index 0000000..1c826c7 --- /dev/null +++ b/scripts/setup @@ -0,0 +1,262 @@ +#!/bin/sh -eu +# vim: ts=4 et: + +[ -z "$DEBUG" ] || [ "$DEBUG" = 0 ] || set -x + +export \ + DEVICE=/dev/vda \ + TARGET=/mnt \ + SETUP=/tmp/setup.d + + +die() { + printf '\033[1;7;31m FATAL: %s \033[0m\n' "$@" >&2 # bold reversed red + exit 1 +} +einfo() { + printf '\n\033[1;7;36m> %s <\033[0m\n' "$@" >&2 # bold reversed cyan +} + +# set up the builder's environment +setup_builder() { + einfo "Setting up Builder Instance" + setup-apkrepos -1 # main repo via dl-cdn + # ODO? also uncomment community repo? + # Always use latest versions within the release, security patches etc. + apk upgrade --no-cache --available + apk --no-cache add \ + e2fsprogs \ + dosfstools \ + gettext \ + lsblk \ + parted +} + +make_filesystem() { + einfo "Making the Filesystem" + root_dev=$DEVICE + + # make sure we're using a blank block device + lsblk -P --fs "$DEVICE" >/dev/null 2>&1 || \ + die "'$DEVICE' is not a valid block device" + if lsblk -P --fs "$DEVICE" | grep -vq 'FSTYPE=""'; then + die "Block device '$DEVICE' is not blank" + fi + + if [ "$FIRMWARE" = uefi ]; then + # EFI partition isn't optimally aligned, but is rarely used after boot + parted "$DEVICE" -- \ + mklabel gpt \ + mkpart EFI fat32 512KiB 1MiB \ + mkpart / ext4 1MiB 100% \ + set 1 esp on \ + unit MiB print + + root_dev="${DEVICE}2" + mkfs.fat -n EFI "${DEVICE}1" + fi + + mkfs.ext4 -O ^64bit -L / "$root_dev" + mkdir -p "$TARGET" + mount -t ext4 "$root_dev" "$TARGET" + + if [ "$FIRMWARE" = uefi ]; then + mkdir -p "$TARGET/boot/efi" + mount -t vfat "${DEVICE}1" "$TARGET/boot/efi" + fi +} + +install_base() { + einfo "Installing Alpine Base" + mkdir -p "$TARGET/etc/apk" + echo "$REPOS" > "$TARGET/etc/apk/repositories" + cp -a /etc/apk/keys "$TARGET/etc/apk" + + # shellcheck disable=SC2086 + for key in $REPO_KEYS; do + wget -q $key -P "$TARGET/etc/apk/keys" + done + + # shellcheck disable=SC2086 + apk --root "$TARGET" --initdb --no-cache add $PACKAGES_ADD + # shellcheck disable=SC2086 + [ -z "$PACKAGES_NOSCRIPTS" ] || \ + apk --root "$TARGET" --no-cache --no-scripts add $PACKAGES_NOSCRIPTS + # shellcheck disable=SC2086 + [ -z "$PACKAGES_DEL" ] || \ + apk --root "$TARGET" --no-cache del $PACKAGES_DEL +} + +setup_chroot() { + mount -t proc none "$TARGET/proc" + mount --bind /dev "$TARGET/dev" + mount --bind /sys "$TARGET/sys" + + # Needed for bootstrap, will be removed in the cleanup stage. + install -Dm644 /etc/resolv.conf "$TARGET/etc/resolv.conf" +} + +install_bootloader() { + einfo "Installing Bootloader" + + # create initfs + + # shellcheck disable=SC2046 + kernel=$(basename $(find "$TARGET/lib/modules/"* -maxdepth 0)) + + # ensure features can be found by mkinitfs + for FEATURE in $INITFS_FEATURES; do + # already taken care of? + [ -f "$TARGET/etc/mkinitfs/features.d/$FEATURE.modules" ] || \ + [ -f "$TARGET/etc/mkinitfs/features.d/$FEATURE.files" ] && continue + # find the kernel module directory + module=$(chroot "$TARGET" /sbin/modinfo -k "$kernel" -n "$FEATURE") + [ -z "$module" ] && die "initfs_feature '$FEATURE' kernel module not found" + # replace everything after .ko with a * + echo "$module" | cut -d/ -f5- | sed -e 's/\.ko.*/.ko*/' \ + > "$TARGET/etc/mkinitfs/features.d/$FEATURE.modules" + done + + # TODO? this appends INITFS_FEATURES, we may want to allow removal someday? + sed -Ei "s/^features=\"([^\"]+)\"/features=\"\1 $INITFS_FEATURES\"/" \ + "$TARGET/etc/mkinitfs/mkinitfs.conf" + + chroot "$TARGET" /sbin/mkinitfs "$kernel" + + if [ "$FIRMWARE" = uefi ]; then + install_grub_efi + else + install_extlinux + fi +} + +install_extlinux() { + # Use disk labels instead of UUID or devices paths so that this works across + # instance familes. UUID works for many instances but breaks on the NVME + # ones because EBS volumes are hidden behind NVME devices. + # + # Shorten timeout (1/10s), eliminating delays for instance launches. + # + # ttyS0 is for EC2 Console "Get system log" and "EC2 Serial Console" + # features, whereas tty0 is for "Get Instance screenshot" feature. Enabling + # the port early in extlinux gives the most complete output in the log. + # + # TODO: review for other clouds -- this may need to be cloud-specific. + sed -Ei -e "s|^[# ]*(root)=.*|\1=LABEL=/|" \ + -e "s|^[# ]*(default_kernel_opts)=.*|\1=\"$KERNEL_OPTIONS\"|" \ + -e "s|^[# ]*(serial_port)=.*|\1=ttyS0|" \ + -e "s|^[# ]*(modules)=.*|\1=$KERNEL_MODULES|" \ + -e "s|^[# ]*(default)=.*|\1=virt|" \ + -e "s|^[# ]*(timeout)=.*|\1=1|" \ + "$TARGET/etc/update-extlinux.conf" + + chroot "$TARGET" /sbin/extlinux --install /boot + # TODO: is this really necessary? can we set all this stuff during --install? + chroot "$TARGET" /sbin/update-extlinux --warn-only +} + +install_grub_efi() { + [ -d "/sys/firmware/efi" ] || die "/sys/firmware/efi does not exist" + + case "$ARCH" in + x86_64) grub_target=x86_64-efi ; fwa=x64 ;; + aarch64) grub_target=arm64-efi ; fwa=aa64 ;; + *) die "ARCH=$ARCH is currently unsupported" ;; + esac + + # disable nvram so grub doesn't call efibootmgr + chroot "$TARGET" /usr/sbin/grub-install --target="$grub_target" --efi-directory=/boot/efi \ + --bootloader-id=alpine --boot-directory=/boot --no-nvram + + # fallback mode + install -D "$TARGET/boot/efi/EFI/alpine/grub$fwa.efi" "$TARGET/boot/efi/EFI/boot/boot$fwa.efi" + + # install default grub config + envsubst < "$SETUP/grub.template" > "$SETUP/grub" + install -o root -g root -Dm644 -t "$TARGET/etc/default" \ + "$SETUP/grub" + + # generate/install new config + chroot "$TARGET" grub-mkconfig -o /boot/grub/grub.cfg +} + +configure_system() { + einfo "Configuring System" + + # default network configuration + install -o root -g root -Dm644 -t "$TARGET/etc/network" "$SETUP/interfaces" + + # configure NTP server, if specified + [ -n "$NTP_SERVER" ] && \ + sed -e 's/^pool /server /' -e "s/pool.ntp.org/$NTP_SERVER/g" \ + -i "$TARGET/etc/chrony/chrony.conf" + + # setup fstab + install -o root -g root -Dm644 -t "$TARGET/etc" "$SETUP/fstab" + # if we're using an EFI bootloader, add extra line for EFI partition + if [ "$FIRMWARE" = uefi ]; then + cat "$SETUP/fstab.grub-efi" >> "$TARGET/etc/fstab" + fi + + # Disable getty for physical ttys, enable getty for serial ttyS0. + sed -Ei -e '/^tty[0-9]/s/^/#/' -e '/^#ttyS0:/s/^#//' "$TARGET/etc/inittab" + + # setup sudo and/or doas + if grep -q '^sudo$' "$TARGET/etc/apk/world"; then + echo '%wheel ALL=(ALL) NOPASSWD: ALL' > "$TARGET/etc/sudoers.d/wheel" + fi + if grep -q '^doas$' "$TARGET/etc/apk/world"; then + echo 'permit nopass :wheel' > "$TARGET/etc/doas.d/wheel.conf" + fi + + # explicitly lock the root account + chroot "$TARGET" /bin/sh -c "/bin/echo 'root:*' | /usr/sbin/chpasswd -e" + chroot "$TARGET" /usr/bin/passwd -l root + + # set up image user + user="${IMAGE_LOGIN:-alpine}" + chroot "$TARGET" /usr/sbin/addgroup "$user" + chroot "$TARGET" /usr/sbin/adduser -h "/home/$user" -s /bin/sh -G "$user" -D "$user" + chroot "$TARGET" /usr/sbin/addgroup "$user" wheel + chroot "$TARGET" /bin/sh -c "echo '$user:*' | /usr/sbin/chpasswd -e" + + # modify PS1s in /etc/profile to add user + sed -Ei \ + -e "s/(^PS1=')(\\$\\{HOSTNAME%)/\\1\\$\\USER@\\2/" \ + -e "s/( PS1=')(\\\\h:)/\\1\\\\u@\\2/" \ + -e "s/( PS1=')(%m:)/\\1%n@\\2/" \ + "$TARGET"/etc/profile + + # write /etc/motd + echo "$MOTD" > "$TARGET"/etc/motd + + setup_services +} + +# shellcheck disable=SC2046 +setup_services() { + for lvl_svcs in $SERVICES_ENABLE; do + rc add $(echo "$lvl_svcs" | tr '=,' ' ') + done + for lvl_svcs in $SERVICES_DISABLE; do + rc del $(echo "$lvl_svcs" | tr '=,' ' ') + done +} + +rc() { + op="$1" # add or del + runlevel="$2" # runlevel name + shift 2 + services="$*" # names of services + + for svc in $services; do + chroot "$TARGET" rc-update "$op" "$svc" "$runlevel" + done +} + +setup_builder +make_filesystem +install_base +setup_chroot +install_bootloader +configure_system diff --git a/scripts/setup-cloudinit b/scripts/setup-cloudinit new file mode 100755 index 0000000..dd953c0 --- /dev/null +++ b/scripts/setup-cloudinit @@ -0,0 +1,48 @@ +#!/bin/sh -eu +# vim: ts=4 et: + +[ -z "$DEBUG" ] || [ "$DEBUG" = 0 ] || set -x + +TARGET=/mnt + +einfo() { + printf '\n\033[1;7;36m> %s <\033[0m\n' "$@" >&2 # bold reversed cyan +} + +einfo "Installing up cloud-init bootstrap components..." + +# This adds the init scripts at the correct boot phases +chroot "$TARGET" /sbin/setup-cloud-init + +# cloud-init locks our user by default which means alpine can't login from +# SSH. This seems like a bug in cloud-init that should be fixed but we can +# hack around it for now here. +if [ -f "$TARGET"/etc/cloud/cloud.cfg ]; then + sed -i '/lock_passwd:/s/True/False/' "$TARGET"/etc/cloud/cloud.cfg +fi + +# configure the image for a particular cloud datasource +case "$CLOUD" in + aws) + DATASOURCE="Ec2" + ;; + nocloud) + DATASOURCE="NoCloud" + ;; + azure) + DATASOURCE="Azure" + ;; + gcp) + DATASOURCE="GCE" + ;; + oci) + DATASOURCE="Oracle" + ;; + *) + echo "Unsupported Cloud '$CLOUD'" >&2 + exit 1 + ;; +esac + +printf '\n\n# Cloud-Init will use default configuration for this DataSource\n' +printf 'datasource_list: ["%s"]\n' "$DATASOURCE" >> "$TARGET"/etc/cloud/cloud.cfg diff --git a/scripts/setup-tiny b/scripts/setup-tiny new file mode 100755 index 0000000..9fc91fc --- /dev/null +++ b/scripts/setup-tiny @@ -0,0 +1,21 @@ +#!/bin/sh -eu +# vim: ts=4 et: + +[ -z "$DEBUG" ] || [ "$DEBUG" = 0 ] || set -x + +TARGET=/mnt + +einfo() { + printf '\n\033[1;7;36m> %s <\033[0m\n' "$@" >&2 # bold reversed cyan +} + +if [ "$VERSION" = "3.12" ]; then + # tiny-cloud-network requires ifupdown-ng, not in 3.12 + einfo "Configuring Tiny EC2 Bootstrap..." + echo "EC2_USER=$IMAGE_LOGIN" > /etc/conf.d/tiny-ec2-bootstrap +else + einfo "Configuring Tiny Cloud..." + sed -i.bak -Ee "s/^#?CLOUD_USER=.*/CLOUD_USER=$IMAGE_LOGIN/" \ + "$TARGET"/etc/conf.d/tiny-cloud + rm "$TARGET"/etc/conf.d/tiny-cloud.bak +fi diff --git a/scripts/setup.d/fstab b/scripts/setup.d/fstab new file mode 100644 index 0000000..3fb8864 --- /dev/null +++ b/scripts/setup.d/fstab @@ -0,0 +1,2 @@ +# +LABEL=/ / ext4 defaults,noatime 1 1 diff --git a/scripts/setup.d/fstab.grub-efi b/scripts/setup.d/fstab.grub-efi new file mode 100644 index 0000000..03d6e96 --- /dev/null +++ b/scripts/setup.d/fstab.grub-efi @@ -0,0 +1 @@ +LABEL=EFI /boot/efi vfat defaults,noatime,uid=0,gid=0,umask=077 0 0 diff --git a/scripts/setup.d/grub.template b/scripts/setup.d/grub.template new file mode 100644 index 0000000..b88f991 --- /dev/null +++ b/scripts/setup.d/grub.template @@ -0,0 +1,5 @@ +GRUB_CMDLINE_LINUX_DEFAULT="modules=$KERNEL_MODULES $KERNEL_OPTIONS" +GRUB_DISABLE_RECOVERY=true +GRUB_DISABLE_SUBMENU=y +GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1" +GRUB_TIMEOUT=0 diff --git a/scripts/setup.d/interfaces b/scripts/setup.d/interfaces new file mode 100644 index 0000000..864f3e3 --- /dev/null +++ b/scripts/setup.d/interfaces @@ -0,0 +1,7 @@ +# default alpine-cloud-images network configuration + +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet dhcp diff --git a/support/aws/iam_role_vmimport_policy.json b/support/aws/iam_role_vmimport_policy.json new file mode 100644 index 0000000..61f9e4d --- /dev/null +++ b/support/aws/iam_role_vmimport_policy.json @@ -0,0 +1,44 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:CopySnapshot", + "ec2:Describe*", + "ec2:ModifySnapshotAttribute", + "ec2:RegisterImage", + "kms:CreateGrant", + "kms:Decrypt", + "kms:DescribeKey", + "kms:Encrypt", + "kms:GenerateDataKey*", + "kms:ReEncrypt*", + "license-manager:GetLicenseConfiguration", + "license-manager:ListLicenseSpecificationsForResource", + "license-manager:UpdateLicenseSpecificationsForResource" + ], + "Resource": "*" + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "s3:GetBucketAcl", + "s3:GetBucketLocation", + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::alpine-cloud-images.*" + }, + { + "Sid": "VisualEditor2", + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject" + ], + "Resource": "arn:aws:s3:::alpine-cloud-images.*/*" + } + ] +} diff --git a/support/aws/iam_role_vmimport_trust.json b/support/aws/iam_role_vmimport_trust.json new file mode 100644 index 0000000..2bfbbde --- /dev/null +++ b/support/aws/iam_role_vmimport_trust.json @@ -0,0 +1,17 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "vmie.amazonaws.com" + }, + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": { + "sts:Externalid": "vmimport" + } + } + } + ] +}