Merge commit 'da6fb3afd13643ba9b40c4aa69656fb44458802e'

alpine-cloud-images/.gitignore

@@ -1,7 +1,8 @@
-*~
+**/*~
-*.bak
+**/*.bak
-*.swp
+**/*.swp
 .DS_Store
 .vscode/
 /work/
 releases*yaml
+/*.yaml

alpine-cloud-images/README.md

@@ -32,10 +32,13 @@ tagged on the images...
 | bootstrap | initial bootstrap system (`tiny` = Tiny Cloud) |
 | cloud | provider short name (`aws`) |
 | revision | image revision number |
+| built | image build timestamp |
+| uploaded | image storage timestamp |
 | imported | image import timestamp |
 | import_id | imported image id |
 | import_region | imported image region |
 | published | image publication timestamp |
+| released | image release timestamp _(won't be set until second publish)_ |
 | description | image description |
 
 Although AWS does not allow cross-account filtering by tags, the image name can
@@ -61,12 +64,22 @@ To get just the most recent matching image, use...
 
 The build system consists of a number of components:
 
-* the primary `build` script
+* the primary `build` script, and other related libararies...
+  * `clouds/` - specific cloud provider plugins
+  * `alpine.py` - for getting the latest Alpine information
+  * `image_config_manager.py` - manages collection of image configs
+  * `image_config.py` - individual image config functionality
+  * `image_storage.py` - persistent image/metadata storage
+  * `image_tags.py` - classes for working with image tags
+
 * the `configs/` directory, defining the set of images to be built
+
 * the `scripts/` directory, containing scripts and related data used to set up
   image contents during provisioning
+
 * the Packer `alpine.pkr.hcl`, which orchestrates build, import, and publishing
   of images
+
 * the `cloud_helper.py` script that Packer runs in order to do cloud-specific
   import and publish operations
 
@@ -102,8 +115,8 @@ usage: build [-h] [--debug] [--clean] [--pad-uefi-bin-arch ARCH [ARCH ...]]
 
 positional arguments:   (build up to and including this step)
   configs   resolve image build configuration
-  state     refresh current image build state
+  state     report current build state of images
-  rollback  remove existing local/uploaded/imported images if un-published/released
+  rollback  remove local/uploaded/imported images if not published or released
   local     build images locally
   upload    upload images and metadata to storage
 * import    import local images to cloud provider default region (*)
@@ -121,8 +134,7 @@ optional arguments:
   --custom DIR [DIR ...]  overlay custom directory in work environment
   --skip KEY [KEY ...]    skip variants with dimension key(s)
   --only KEY [KEY ...]    only variants with dimension key(s)
-  --revise                remove existing local/uploaded/imported images if
+  --revise                bump revision and rebuild if published or released
-                          un-published/released, or bump revision and rebuild
   --use-broker            use the identity broker to get credentials
   --no-color              turn off Packer color output
   --parallel N            build N images in parallel
@@ -155,13 +167,11 @@ determines what actions need to be taken, and updates `work/images.yaml`.  A
 subset of image builds can be targeted by using the `--skip` and `--only`
 arguments.
 
-The `rollback` step, when used with `--revise` argument indicates that any
+The `rollback` step will remove any imported, uploaded, or local images, but
-_unpublished_ and _unreleased_ local, imported, or uploaded images should be
+only if they are _unpublished_ and _unreleased_.
-removed and rebuilt.
 
-As _published_ and _released_ images can't be removed, `--revise` can be used
+As _published_ and _released_ images can't be rolled back, `--revise` can be
-with `configs` or `state` to increment the _`revision`_ value to rebuild newly
+used to increment the _`revision`_ value to rebuild newly revised images.
-revised images.
 
 `local`, `upload`, `import`, `publish`, and `release` steps are orchestrated by
 Packer.  By default, each image will be processed serially; providing the
@@ -193,7 +203,9 @@ in all regions where the image has been published.
 providers where this does not make sense (i.e.  NoCloud) or for those which
 it has not yet been coded.
 
-The `release` step marks the images as being fully released.
+The `release` step simply marks the images as being fully released.  _(For the
+offical Alpine releases, we have a `gen_mksite_release.py` script to convert
+the image data to a format that can be used by https://alpinelinux.org/cloud.)_
 
 ### The `cloud_helper.py` Script
 

alpine-cloud-images/TODO.md

@@ -0,0 +1,4 @@
+* clean up cloud modules now that `get_latest_imported_tags` isn't needed
+
+* do we still need to set `ntp_server` for AWS images, starting with 3.18.4?
+  _(or is this now handled via `dhcpcd`?)_

alpine-cloud-images/build

@@ -228,8 +228,8 @@ parser.add_argument(
     default=[], help='only variants with dimension key(s)')
 parser.add_argument(
     '--revise', action='store_true',
-    help='remove existing local/uploaded/imported image, or bump revision and '
+    help='bump revision and rebuild if published or released')
-        ' rebuild if published or released')
+    # --revise is not needed after new revision is uploaded
 parser.add_argument(
     '--use-broker', action='store_true',
     help='use the identity broker to get credentials')
@@ -256,9 +256,9 @@ console.setFormatter(logfmt)
 log.addHandler(console)
 log.debug(args)
 
-if args.step == 'rollback':
+if args.step == 'rollback' and args.revise:
-    log.warning('"rollback" step enables --revise option')
+    log.error('"rollback" step does not support --revise option')
-    args.revise = True
+    sys.exit(1)
 
 # set up credential provider, if we're going to use it
 if args.use_broker:
@@ -292,7 +292,7 @@ if not image_configs.refresh_state(
     log.info('No pending actions to take at this time.')
     sys.exit(0)
 
-if args.step == 'state' or args.step == 'rollback':
+if args.step == 'state':
     sys.exit(0)
 
 # install firmware if missing
@@ -339,7 +339,6 @@ if p.returncode != 0:
 log.info('Packer Completed')
 
 # update final state in work/images.yaml
-# TODO: do we need to do all of this or just save all the image_configs?
 image_configs.refresh_state(
     step='final',
     only=args.only,

alpine-cloud-images/cloud_helper.py

@@ -76,23 +76,26 @@ yaml.explicit_start = True
 
 for image_key in args.image_keys:
     image_config = configs.get(image_key)
+    image_config.load_local_metadata()  # if it exists
 
     if args.action == 'local':
         image_config.convert_image()
 
     elif args.action == 'upload':
-        if image_config.storage:
+        image_config.upload_image()
-            image_config.upload_image()
+
+    elif args.action == 'import' and 'import' in clouds.actions(image_config):
+        # if we don't have the image locally, retrieve it from storage
+        if not image_config.image_path.exists():
+            image_config.retrieve_image()
 
-    elif args.action == 'import':
         clouds.import_image(image_config)
 
-    elif args.action == 'publish':
+    elif args.action == 'publish' and 'publish' in clouds.actions(image_config):
         clouds.publish_image(image_config)
 
     elif args.action == 'release':
-        pass
+        image_config.release_image()
-        # TODO: image_config.release_image() - configurable steps to take on remote host
 
     # save per-image metadata
     image_config.save_metadata(args.action)

alpine-cloud-images/clouds/__init__.py

@@ -31,7 +31,7 @@ def set_credential_provider(debug=False):
 
 ### forward to the correct adapter
 
-# TODO: latest_imported_tags(...)
+# TODO: deprexcate/remove
 def get_latest_imported_tags(config):
     return ADAPTERS[config.cloud].get_latest_imported_tags(
         config.project,
@@ -49,3 +49,7 @@ def delete_image(config, image_id):
 
 def publish_image(config):
     return ADAPTERS[config.cloud].publish_image(config)
+
+# supported actions
+def actions(config):
+    return ADAPTERS[config.cloud].ACTIONS

alpine-cloud-images/clouds/aws.py

@@ -30,6 +30,10 @@ class AWSCloudAdapter(CloudAdapterInterface):
         'bios': 'legacy-bios',
         'uefi': 'uefi',
     }
+    ACTIONS = [
+        'import',
+        'publish',
+    ]
 
     @property
     def sdk(self):
@@ -96,6 +100,7 @@ class AWSCloudAdapter(CloudAdapterInterface):
         tags = ImageTags(from_list=i.tags)
         return DictObj({k: tags.get(k, None) for k in self.IMAGE_INFO})
 
+    # TODO: deprectate/remove
     # get the latest imported image's tags for a given build key
     def get_latest_imported_tags(self, project, image_key):
         images = self._get_images_with_tags(
@@ -222,6 +227,10 @@ class AWSCloudAdapter(CloudAdapterInterface):
             tags.import_id = image_id
             tags.import_region = ec2c.meta.region_name
             image.create_tags(Tags=tags.as_list())
+            # update image config with import information
+            ic.imported = tags.imported
+            ic.import_id = tags.import_id
+            ic.import_region = tags.import_region
         except Exception:
             log.error('Unable to tag image:', exc_info=True)
             log.info('Removing image and snapshot')
@@ -229,9 +238,6 @@ class AWSCloudAdapter(CloudAdapterInterface):
             snapshot.delete()
             raise
 
-        # update ImageConfig with imported tag values, minus special AWS 'Name'
-        tags.pop('Name', None)
-        ic.__dict__ |= tags
 
     # delete an (unpublished) image
     def delete_image(self, image_id):
@@ -390,7 +396,9 @@ class AWSCloudAdapter(CloudAdapterInterface):
                 time.sleep(copy_wait)
                 copy_wait = 30
 
+        # update image config with published information
         ic.artifacts = artifacts
+        ic.published = datetime.utcnow().isoformat()
 
 
 def register(cloud, cred_provider=None):

alpine-cloud-images/clouds/interfaces/adapter.py

@@ -2,6 +2,8 @@
 
 class CloudAdapterInterface:
 
+    ACTIONS = []
+
     def __init__(self, cloud, cred_provider=None):
         self._sdk = None
         self._sessions = {}

alpine-cloud-images/configs/alpine.conf

@@ -55,7 +55,7 @@ Default {
 # profile build matrix
 Dimensions {
   version {
-    "3.18"  { include required("version/3.17.conf") }
+    "3.18"  { include required("version/3.18.conf") }
     "3.17"  { include required("version/3.17.conf") }
     "3.16"  { include required("version/3.16.conf") }
     "3.15"  { include required("version/3.15.conf") }
@@ -94,12 +94,4 @@ Mandatory {
 
   # final provisioning script
   scripts = [ cleanup ]
-
-  # TODO: remove this after testing
-  #access.PUBLIC = false
-  #regions {
-  #    ALL       = false
-  #    us-west-2 = true
-  #    us-east-1 = true
-  #}
 }

alpine-cloud-images/configs/bootstrap/cloudinit.conf

@@ -9,6 +9,7 @@ EXCLUDE = ["3.12", "3.13", "3.14"]
 packages {
   cloud-init          = true
   dhclient            = true  # offically supported, for now
+  dhcpcd              = null  # unsupported, for now
   openssh-server-pam  = true
   e2fsprogs-extra     = true  # for resize2fs
 }

alpine-cloud-images/configs/bootstrap/tiny.conf

@@ -6,9 +6,9 @@ bootstrap_url   = "https://gitlab.alpinelinux.org/alpine/cloud/tiny-cloud"
 WHEN {
   "3.13 3.14 3.15 3.16 3.17" {
     # tiny-cloud < 3.0.0 doesn't have --setup option
-    boot.tiny-cloud-early     = true
+    services.boot.tiny-cloud-early     = true
-    default.tiny-cloud        = true
+    services.default.tiny-cloud        = true
-    default.tiny-cloud-final  = true
+    services.default.tiny-cloud-final  = true
   }
   aws {
     packages.tiny-cloud-aws = true
@@ -16,7 +16,7 @@ WHEN {
       "3.12" {
           # fallback to the old tiny-ec2-bootstrap package
           packages.tiny-cloud-aws             = null
-          services.sysinit.tiny-cloud-early   = null
+          services.boot.tiny-cloud-early      = null
           services.default.tiny-cloud         = null
           services.default.tiny-cloud-final   = null
           packages.tiny-ec2-bootstrap         = true

alpine-cloud-images/configs/cloud/aws.conf

@@ -21,6 +21,14 @@ ntp_server  = 169.254.169.123
 access.PUBLIC = true
 regions.ALL   = true
 
+# limit edge publishing
+WHEN.edge {
+  access.PUBLIC     = false
+  regions.ALL       = false
+  regions.us-west-2 = true
+  regions.us-east-1 = true
+}
+
 cloud_region_url  = "https://{region}.console.aws.amazon.com/ec2/home#Images:visibility=public-images;imageId={image_id}",
 cloud_launch_url  = "https://{region}.console.aws.amazon.com/ec2/home#launchAmi={image_id}"
 

alpine-cloud-images/configs/version/base/5.conf

@@ -2,7 +2,5 @@
 
 include required("4.conf")
 
-packages {
+# start using dhcpcd for improved IPv6 experience
-  # start using dhcpcd for improved IPv6 experience
+packages.dhcpcd = true
-  dhcpcd = true
-}

alpine-cloud-images/gen_mksite_releases.py

@@ -117,8 +117,6 @@ for i_key, i_cfg in configs.get().items():
             'cloud_name': i_cfg.cloud_name,
         }
 
-    filters['regions'] = {}
-
     if arch not in filters['archs']:
         filters['archs'][arch] = {
             'arch': arch,
@@ -137,9 +135,31 @@ for i_key, i_cfg in configs.get().items():
             'bootstrap_name': i_cfg.bootstrap_name,
         }
 
-    if i_cfg.artifacts:
+    versions[version] |= {
+        'version': version,
+        'release': release,
+        'end_of_life': i_cfg.end_of_life,
+    }
+    versions[version]['images'][image_name] |= {
+        'image_name': image_name,
+        'arch': arch,
+        'firmware': firmware,
+        'bootstrap': bootstrap,
+        'published': i_cfg.published.split('T')[0],     # just the date
+    }
+    versions[version]['images'][image_name]['downloads'][cloud] |= {
+        'cloud': cloud,
+        'image_format': i_cfg.image_format,
+        'image_url':  i_cfg.download_url + '/' + (i_cfg.image_name)
+    }
+
+    # TODO: not all clouds will have artifacts
+    if i_cfg._get('artifacts'):
+        log.debug("ARTIFACTS: %s", i_cfg.artifacts)
         for region, image_id in {r: i_cfg.artifacts[r] for r in sorted(i_cfg.artifacts)}.items():
+            log.debug("REGION: %s", region)
             if region not in filters['regions']:
+                log.debug("not in filters['region']")
                 filters['regions'][region] = {
                     'region': region,
                     'clouds': [cloud],
@@ -148,23 +168,6 @@ for i_key, i_cfg in configs.get().items():
             if cloud not in filters['regions'][region]['clouds']:
                 filters['regions'][region]['clouds'].append(cloud)
 
-            versions[version] |= {
-                'version': version,
-                'release': release,
-                'end_of_life': i_cfg.end_of_life,
-            }
-            versions[version]['images'][image_name] |= {
-                'image_name': image_name,
-                'arch': arch,
-                'firmware': firmware,
-                'bootstrap': bootstrap,
-                'published': i_cfg.published.split('T')[0],     # just the date
-            }
-            versions[version]['images'][image_name]['downloads'][cloud] |= {
-                'cloud': cloud,
-                'image_format': i_cfg.image_format,
-                'image_url':  i_cfg.download_url + '/' + (i_cfg.image_name)
-            }
             versions[version]['images'][image_name]['regions'][region] |= {
                 'cloud': cloud,
                 'region': region,
@@ -199,7 +202,7 @@ for version in sorted(versions, reverse=True, key=lambda s: [int(u) for u in s.s
 
         images[image_name]['downloads'] = d
 
-        regions = images[image_name].pop('regions')
+        regions = images[image_name].pop('regions', [])
         r = []
         for region in sorted(regions):
             r.append(regions[region])

alpine-cloud-images/get-image-cache.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+# vim: ts=4 et:
+
+# NOTE: this is an experimental work-in-progress
+
+# Ensure we're using the Python virtual env with our installed dependencies
+import os
+import sys
+import textwrap
+
+NOTE = textwrap.dedent("""
+    Experimental:  Outputs image cache YAML on STDOUT for use with prune-images.py
+    """)
+
+sys.pycache_prefix = 'work/__pycache__'
+
+if not os.path.exists('work'):
+    print('FATAL: Work directory does not exist.', file=sys.stderr)
+    print(NOTE, file=sys.stderr)
+    exit(1)
+
+# Re-execute using the right virtual environment, if necessary.
+venv_args = [os.path.join('work', 'bin', 'python3')] + sys.argv
+if os.path.join(os.getcwd(), venv_args[0]) != sys.executable:
+    print("Re-executing with work environment's Python...\n", file=sys.stderr)
+    os.execv(venv_args[0], venv_args)
+
+# We're now in the right Python environment
+
+import argparse
+import logging
+import re
+import time
+from collections import defaultdict
+from ruamel.yaml import YAML
+
+import clouds
+
+
+### Constants & Variables
+
+CLOUDS = ['aws']
+LOGFORMAT = '%(asctime)s - %(levelname)s - %(message)s'
+
+RE_ALPINE = re.compile(r'^alpine-')
+RE_RELEASE = re.compile(r'-(edge|[\d\.]+)-')
+RE_REVISION = re.compile(r'-r?(\d+)$')
+RE_STUFF = re.compile(r'(edge|[\d+\.]+)(?:_rc(\d+))?-(.+)-r?(\d+)$')
+
+
+### Functions
+
+# allows us to set values deep within an object that might not be fully defined
+def dictfactory():
+    return defaultdict(dictfactory)
+
+
+# undo dictfactory() objects to normal objects
+def undictfactory(o):
+    if isinstance(o, defaultdict):
+        o = {k: undictfactory(v) for k, v in o.items()}
+    return o
+
+
+### Command Line & Logging
+
+parser = argparse.ArgumentParser(description=NOTE)
+parser.add_argument('--debug', action='store_true', help='enable debug output')
+parser.add_argument('--cloud', choices=CLOUDS, required=True, help='cloud provider')
+parser.add_argument('--region', help='specific region, instead of all regions')
+parser.add_argument(
+    '--use-broker', action='store_true',
+    help='use the identity broker to get credentials')
+args = parser.parse_args()
+
+log = logging.getLogger()
+log.setLevel(logging.DEBUG if args.debug else logging.INFO)
+console = logging.StreamHandler()
+logfmt = logging.Formatter(LOGFORMAT, datefmt='%FT%TZ')
+logfmt.converter = time.gmtime
+console.setFormatter(logfmt)
+log.addHandler(console)
+log.debug(args)
+
+# set up credential provider, if we're going to use it
+if args.use_broker:
+    clouds.set_credential_provider(debug=args.debug)
+
+# what region(s)?
+regions = clouds.ADAPTERS[args.cloud].regions
+if args.region:
+    if args.region not in regions:
+        log.error('invalid region: %s', args.region)
+        exit(1)
+    else:
+        regions = [args.region]
+
+filters = {
+    'Owners': ['self'],
+    'Filters': [
+        {'Name': 'state', 'Values': ['available']},
+    ]
+}
+
+data = dictfactory()
+now = time.gmtime()
+
+for region in sorted(regions):
+    # TODO: make more generic if we need to do this for other clouds someday
+    ec2r = clouds.ADAPTERS[args.cloud].session(region).resource('ec2')
+    images = sorted(ec2r.images.filter(**filters), key=lambda k: k.creation_date)
+    log.info(f'--- {region} : {len(images)} ---')
+    version = release = revision = None
+
+    for image in images:
+        latest = data[region]['latest']     # shortcut
+
+        # information about the image
+        id = image.id
+        name = image.name
+
+        # only consider images named /^alpine-/
+        if not RE_ALPINE.search(image.name):
+            log.warning(f'IGNORING {region}\t{id}\t{name}')
+            continue
+
+        # parse image name for more information
+        # NOTE: we can't rely on tags, because they may not have been set successfully
+        m = RE_STUFF.search(name)
+        if not m:
+            log.error(f'!PARSE\t{region}\t{id}\t{name}')
+            continue
+
+        release = m.group(1)
+        rc = m.group(2)
+        version = '.'.join(release.split('.')[0:2])
+        variant = m.group(3)
+        revision = m.group(4)
+        variant_key = '-'.join([version, variant])
+        release_key = revision if release == 'edge' else '-'.join([release, revision])
+
+        last_launched_attr = image.describe_attribute(Attribute='lastLaunchedTime')['LastLaunchedTime']
+        last_launched = last_launched_attr.get('Value', 'Never')
+
+        eol = time.strptime(image.deprecation_time ,'%Y-%m-%dT%H:%M:%S.%fZ') < now
+
+        # keep track of images
+        data[region]['images'][id] = {
+            'name': name,
+            'release': release,
+            'version': version,
+            'variant': variant,
+            'revision': revision,
+            'variant_key': variant_key,
+            'release_key': release_key,
+            'created': image.creation_date,
+            'launched': last_launched,
+            'deprecated': image.deprecation_time,
+            'rc': rc is not None,
+            'eol': eol,
+            'private': not image.public,
+            'snapshot_id': image.block_device_mappings[0]['Ebs']['SnapshotId']
+        }
+
+        # keep track of the latest release_key per variant_key
+        if variant_key not in latest or (release > latest[variant_key]['release']) or (release == latest[variant_key]['release'] and [revision > latest[variant_key]['revision']]):
+            data[region]['latest'][variant_key] = {
+                'release': release,
+                'revision': revision,
+                'release_key': release_key
+            }
+
+        log.info(f'{region}\t{not image.public}\t{eol}\t{last_launched.split("T")[0]}\t{name}')
+
+# instantiate YAML
+yaml = YAML()
+yaml.explicit_start = True
+
+# TODO?  dump out to a file instead of STDOUT?
+yaml.dump(undictfactory(data), sys.stdout)
+
+total = 0
+for region, rdata in sorted(data.items()):
+    count = len(rdata['images'])
+    log.info(f'{region} : {count} images')
+    total += count
+
+log.info(f'TOTAL : {total} images')

alpine-cloud-images/image_config.py

@@ -28,6 +28,15 @@ class ImageConfig():
     STEPS = [
         'local', 'upload', 'import', 'publish', 'release'
     ]
+    # we expect these to be available
+    DEFAULT_OBJ = {
+        'built': None,
+        'uploaded': None,
+        'imported': None,
+        'published': None,
+        'released': None,
+        'artifacts': None,
+    }
 
     def __init__(self, config_key, obj={}, log=None, yaml=None):
         self._log = log
@@ -35,7 +44,7 @@ class ImageConfig():
         self._storage = None
         self.config_key = str(config_key)
         tags = obj.pop('tags', None)
-        self.__dict__ |= self._deep_dict(obj)
+        self.__dict__ |= self.DEFAULT_OBJ | self._deep_dict(obj)
         # ensure tag values are str() when loading
         if tags:
             self.tags = tags
@@ -266,14 +275,18 @@ class ImageConfig():
 
         return self.STEPS.index(s) <= self.STEPS.index(step)
 
+    def load_local_metadata(self):
+        metadata_path = self.local_dir / self.metadata_file
+        if metadata_path.exists():
+            self._log.debug('Loading image metadata from %s', metadata_path)
+            loaded = self._yaml.load(metadata_path)
+            loaded.pop('name', None)    # don't overwrite 'name' format string!
+            loaded.pop('Name', None)    # remove special AWS tag
+            self.__dict__ |= loaded
 
-    # TODO: this needs to be sorted out for 'upload' and 'release' steps
     def refresh_state(self, step, revise=False):
         log = self._log
         actions = {}
-        revision = 0
-        step_state = step == 'state'
-        step_rollback = step == 'rollback'
         undo = {}
 
         # enable initial set of possible actions based on specified step
@@ -281,93 +294,61 @@ class ImageConfig():
             if self._is_step_or_earlier(s, step):
                 actions[s] = True
 
-        # pick up any updated image metadata
+        # sets the latest revision metadata (from storage and local)
-        self.load_metadata()
+        self.load_metadata(step)
 
-        # TODO: check storage and/or cloud - use this instead of remote_image
+        # if we're rolling back, figure out what we need to undo first
-        # latest_revision = self.get_latest_revision()
+        if step == 'rollback':
+            if self.released or self.published:
+                undo['ROLLBACK_BLOCKED'] = True
 
-        if (step_rollback or revise) and self.local_image.exists():
+            else:
-            undo['local'] = True
+                if self.imported and 'import' in clouds.actions(self):
+                    undo['import'] = True
+                    self.imported = None
 
-
-
-        if step_rollback:
-            if self.local_image.exists():
-                undo['local'] = True
-
-            if not self.published or self.released:
                 if self.uploaded:
                     undo['upload'] = True
+                    self.uploaded = None
 
-                if self.imported:
+                if self.built and self.local_dir.exists():
-                    undo['import'] = True
+                    undo['local'] = True
+                    self.built = None
 
-        # TODO: rename to 'remote_tags'?
+        # handle --revise option, if necessary
-        # if we load remote tags into state automatically, shouldn't that info already be in self?
+        if revise and (self.published or self.released):
-        remote_image = clouds.get_latest_imported_tags(self)
+            # get rid of old metadata
-        log.debug('\n%s', remote_image)
+            (self.local_dir / self.metadata_file).unlink()
+            self.revision = int(self.revision) + 1
+            self.__dict__ |= self.DEFAULT_OBJ
+            self.__dict__.pop('import_id', None)
+            self.__dict__.pop('import_region', None)
 
-        if revise:
+            # do we already have it built locally?
-            if self.local_image.exists():
+            if self.image_path.exists():
-                # remove previously built local image artifacts
+                # then we should use its metadata
-                log.warning('%s existing local image dir %s',
+                self.load_local_metadata()
-                    'Would remove' if step_state else 'Removing',
-                    self.local_dir)
-                if not step_state:
-                    shutil.rmtree(self.local_dir)
 
-            if remote_image and remote_image.get('published', None):
+            else:
-                log.warning('%s image revision for %s',
+                undo['local'] = True
-                    'Would bump' if step_state else 'Bumping',
-                    self.image_key)
-                revision = int(remote_image.revision) + 1
 
-            elif remote_image and remote_image.get('imported', None):
+        # after all that, let's figure out what's to be done!
-                # remove existing imported (but unpublished) image
+        if self.built:
-                log.warning('%s unpublished remote image %s',
-                    'Would remove' if step_state else 'Removing',
-                    remote_image.import_id)
-                if not step_state:
-                    clouds.delete_image(self, remote_image.import_id)
-
-            remote_image = None
-
-        elif remote_image:
-            if remote_image.get('imported', None):
-                # already imported, don't build/upload/import again
-                log.debug('%s - already imported', self.image_key)
-                actions.pop('local', None)
-                actions.pop('upload', None)
-                actions.pop('import', None)
-
-            if remote_image.get('published', None):
-                # NOTE: re-publishing can update perms or push to new regions
-                log.debug('%s - already published', self.image_key)
-
-        if self.local_image.exists():
-            # local image's already built, don't rebuild
-            log.debug('%s - already locally built', self.image_key)
             actions.pop('local', None)
 
-        else:
+        if self.uploaded:
-            self.built = None
+            actions.pop('upload', None)
 
-        # merge remote_image data into image state
+        if self.imported or 'import' not in clouds.actions(self):
-        if remote_image:
+            actions.pop('import', None)
-            self.__dict__ |= dict(remote_image)
 
-        else:
+        # NOTE: always publish (if cloud allows) to support new regions
-            self.__dict__ |= {
+        if 'publish' not in clouds.actions(self):
-                'revision': revision,
+            actions.pop('publish', None)
-                'uploaded': None,
+
-                'imported': None,
+        # don't re-publish again if we're targeting the release step
-                'import_id': None,
+        elif step == 'release' and self.published:
-                'import_region': None,
+            actions.pop('publish', None)
-                'published': None,
-                'artifacts': None,
-                'released': None,
-            }
 
         # remove remaining actions not possible based on specified step
         for s in self.STEPS:
@@ -375,7 +356,26 @@ class ImageConfig():
                 actions.pop(s, None)
 
         self.actions = list(actions)
-        log.info('%s/%s = %s', self.cloud, self.image_name, self.actions)
+        log.info('%s/%s = [%s]', self.cloud, self.image_name, ' '.join(self.actions))
+
+        if undo:
+            act = "Would undo" if step == 'state' else "Undoing"
+            log.warning('%s: [%s]', act, ' '.join(undo.keys()))
+
+        if step != 'state':
+            if 'import' in undo:
+                log.warning('Deleting imported image: %s', self.import_id)
+                clouds.delete_image(self, self.import_id)
+                self.import_id = None
+                self.import_region = None
+
+            if 'upload' in undo:
+                log.warning('Removing uploaded image from storage')
+                self.remove_image()
+
+            if 'local' in undo:
+                log.warning('Removing local build directory')
+                shutil.rmtree(self.local_dir)
 
         self.state_updated = datetime.utcnow().isoformat()
 
@@ -388,16 +388,11 @@ class ImageConfig():
 
     def _save_checksum(self, file):
         self._log.info("Calculating checksum for '%s'", file)
-        sha256_hash = hashlib.sha256()
         sha512_hash = hashlib.sha512()
         with open(file, 'rb') as f:
             for block in iter(lambda: f.read(4096), b''):
-                sha256_hash.update(block)
                 sha512_hash.update(block)
 
-        with open(str(file) + '.sha256', 'w') as f:
-            print(sha256_hash.hexdigest(), file=f)
-
         with open(str(file) + '.sha512', 'w') as f:
             print(sha512_hash.hexdigest(), file=f)
 
@@ -415,16 +410,30 @@ class ImageConfig():
     def upload_image(self):
         self.storage.store(
             self.image_file,
-            self.image_file + '.sha256',
             self.image_file + '.sha512'
         )
         self.uploaded = datetime.utcnow().isoformat()
 
+    def retrieve_image(self):
+        self._log.info('Retrieving %s from storage', self.image_file)
+        self.storage.retrieve(
+            self.image_file
+        )
+
+    def remove_image(self):
+        self.storage.remove(
+            self.image_file,
+            self.image_file + '.sha512',
+            self.metadata_file,
+            self.metadata_file + '.sha512'
+        )
+
+    def release_image(self):
+        self.released = datetime.utcnow().isoformat()
+
     def save_metadata(self, action):
         os.makedirs(self.local_dir, exist_ok=True)
         self._log.info('Saving image metadata')
-        # TODO: save metadata updated timestamp as metadata?
-        # TODO: def self.metadata to return what we consider metadata?
         metadata = dict(self.tags)
         self.metadata_updated = datetime.utcnow().isoformat()
         metadata |= {
@@ -437,29 +446,36 @@ class ImageConfig():
         if action != 'local' and self.storage:
             self.storage.store(
                 self.metadata_file,
-                self.metadata_file + '.sha256',
                 self.metadata_file + '.sha512'
             )
 
-    def load_metadata(self):
+    def load_metadata(self, step):
-        # TODO: what if we have fresh configs, but the image is already uploaded/imported?
+        new = True
-        # we'll need to get revision first somehow
+        if step != 'final':
-        if 'revision' not in self.__dict__:
+            # what's the latest uploaded revision?
-            return
+            revision_glob = self.name.format(**(self.__dict__ | {'revision': '*'}))
+            try:
+                revision_yamls = self.storage.list(revision_glob + '.yaml', err_ok=True)
+                new = not revision_yamls    # empty list is still new
 
-        # TODO: revision = '*' for now - or only if unknown?
+            except RuntimeError:
+                pass
+
+            latest_revision = 0
+            if not new:
+                for y in revision_yamls:
+                    yr = int(y.rstrip('.yaml').rsplit('r', 1)[1])
+                    if yr > latest_revision:
+                        latest_revision = yr
+
+            self.revision = latest_revision
 
-        # get a list of local matching <name>-r*.yaml?
         metadata_path = self.local_dir / self.metadata_file
-        if metadata_path.exists():
+        if step != 'final' and not new and not metadata_path.exists():
-            self._log.info('Loading image metadata from %s', metadata_path)
+            try:
-            self.__dict__ |= self._yaml.load(metadata_path).items()
+                self.storage.retrieve(self.metadata_file)
+            except RuntimeError as e:
+                # TODO: don't we already log an error/warning?
+                self._log.warning(f'Unable to retrieve from storage: {metadata_path}')
 
-        # get a list of storage  matching <name>-r*.yaml
+        self.load_local_metadata()  # if it exists
-        #else:
-            # retrieve metadata (and image?) from storage_url
-            # else:
-                # retrieve metadata from imported image
-
-        # if there's no stored metadata, we are in transition,
-        #   get a list of imported images matching <name>-r*.yaml

alpine-cloud-images/image_storage.py

@@ -11,22 +11,25 @@ from urllib.parse import urlparse
 from image_tags import DictObj
 
 
-def run(cmd, log, errmsg=None, errvals=[]):
+def run(cmd, log, errmsg=None, errvals=[], err_ok=False):
     # ensure command and error values are lists of strings
     cmd = [str(c) for c in cmd]
     errvals = [str(ev) for ev in errvals]
 
     log.debug('COMMAND: %s', ' '.join(cmd))
-    p = Popen(cmd, stdout=PIPE, stdin=PIPE, encoding='utf8')
+    p = Popen(cmd, stdout=PIPE, stdin=PIPE, stderr=PIPE, encoding='utf8')
     out, err = p.communicate()
     if p.returncode:
         if errmsg:
-            log.error(errmsg, *errvals)
+            if err_ok:
+                log.debug(errmsg, *errvals)
 
-        log.error('COMMAND: %s', ' '.join(cmd))
+            else:
-        log.error('EXIT: %d', p.returncode)
+                log.error(errmsg, *errvals)
-        log.error('STDOUT:\n%s', out)
+
-        log.error('STDERR:\n%s', err)
+        log.debug('EXIT: %d / COMMAND: %s', p.returncode, ' '.join(cmd))
+        log.debug('STDOUT:\n%s', out)
+        log.debug('STDERR:\n%s', err)
         raise RuntimeError
 
     return out, err
@@ -105,7 +108,7 @@ class ImageStorage():
         dest.mkdir(parents=True, exist_ok=True)
         if self.scheme == 'file':
             for file in files:
-                log.info('Retrieving %s', src / file)
+                log.debug('Retrieving %s', src / file)
                 shutil.copy2(src / file, dest / file)
 
             return
@@ -115,7 +118,7 @@ class ImageStorage():
         scp = self.scp
         src_files = []
         for file in files:
-            log.info('Retrieving %s', url + '/' + file)
+            log.debug('Retrieving %s', url + '/' + file)
             src_files.append(scp.user + ':'.join([host, str(src / file)]))
 
         run(
@@ -124,7 +127,7 @@ class ImageStorage():
         )
 
     # TODO: optional files=[]?
-    def list(self, match=None):
+    def list(self, match=None, err_ok=False):
         log = self.log
         path = self.remote
         if not match:
@@ -133,27 +136,27 @@ class ImageStorage():
         files = []
         if self.scheme == 'file':
             path.mkdir(parents=True, exist_ok=True)
-            log.info('Listing of %s files in %s', match, path)
+            log.debug('Listing of %s files in %s', match, path)
             files = sorted(glob(str(path / match)), key=os.path.getmtime, reverse=True)
 
         else:
             url = self.url
             host = self.host
             ssh = self.ssh
-            log.info('Listing %s files at %s', match, url)
+            log.debug('Listing %s files at %s', match, url)
             run(
                 ['ssh'] + ssh.port + ssh.user + [host, 'mkdir', '-p', path],
-                log=log, errmsg='Unable to create path'
+                log=log, errmsg='Unable to create path', err_ok=err_ok
             )
             out, _ = run(
                 ['ssh'] + ssh.port + ssh.user + [host, 'ls', '-1drt', path / match],
-                log=log, errmsg='Failed to list files'
+                log=log, errmsg='Failed to list files', err_ok=err_ok
             )
             files = out.splitlines()
 
         return [os.path.basename(f) for f in files]
 
-    def remove(self, files):
+    def remove(self, *files):
         log = self.log
         if not files:
             log.debug('No files to remove')
@@ -163,7 +166,7 @@ class ImageStorage():
         if self.scheme == 'file':
             for file in files:
                 path = dest / file
-                log.info('Removing %s', path)
+                log.debug('Removing %s', path)
                 if path.exists():
                     path.unlink()
 
@@ -174,7 +177,7 @@ class ImageStorage():
         ssh = self.ssh
         dest_files = []
         for file in files:
-            log.info('Removing %s', url + '/' + file)
+            log.debug('Removing %s', url + '/' + file)
             dest_files.append(dest / file)
 
         run(

alpine-cloud-images/prune-images.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+# vim: ts=4 et:
+
+# NOTE: this is an experimental work-in-progress
+
+# Ensure we're using the Python virtual env with our installed dependencies
+import os
+import sys
+import textwrap
+
+NOTE = textwrap.dedent("""
+    Experimental: Given an image cache YAML file, figure out what needs to be pruned.
+    """)
+
+sys.pycache_prefix = 'work/__pycache__'
+
+if not os.path.exists('work'):
+    print('FATAL: Work directory does not exist.', file=sys.stderr)
+    print(NOTE, file=sys.stderr)
+    exit(1)
+
+# Re-execute using the right virtual environment, if necessary.
+venv_args = [os.path.join('work', 'bin', 'python3')] + sys.argv
+if os.path.join(os.getcwd(), venv_args[0]) != sys.executable:
+    print("Re-executing with work environment's Python...\n", file=sys.stderr)
+    os.execv(venv_args[0], venv_args)
+
+# We're now in the right Python environment
+
+import argparse
+import logging
+import re
+import time
+from collections import defaultdict
+from ruamel.yaml import YAML
+from pathlib import Path
+
+import clouds
+
+
+### Constants & Variables
+
+ACTIONS = ['list', 'prune']
+CLOUDS = ['aws']
+SELECTIONS = ['keep-last', 'unused', 'ALL']
+LOGFORMAT = '%(asctime)s - %(levelname)s - %(message)s'
+
+RE_ALPINE = re.compile(r'^alpine-')
+RE_RELEASE = re.compile(r'-(edge|[\d\.]+)-')
+RE_REVISION = re.compile(r'-r?(\d+)$')
+RE_STUFF = re.compile(r'(edge|[\d+\.]+)-(.+)-r?(\d+)$')
+
+### Functions
+
+# allows us to set values deep within an object that might not be fully defined
+def dictfactory():
+    return defaultdict(dictfactory)
+
+
+# undo dictfactory() objects to normal objects
+def undictfactory(o):
+    if isinstance(o, defaultdict):
+        o = {k: undictfactory(v) for k, v in o.items()}
+    return o
+
+
+### Command Line & Logging
+
+parser = argparse.ArgumentParser(description=NOTE)
+parser.add_argument('--debug', action='store_true', help='enable debug output')
+parser.add_argument('--really', action='store_true', help='really prune images')
+parser.add_argument('--cloud', choices=CLOUDS, required=True, help='cloud provider')
+parser.add_argument('--region', help='specific region, instead of all regions')
+# what to prune...
+parser.add_argument('--private', action='store_true')
+parser.add_argument('--edge-eol', action='store_true')
+parser.add_argument('--rc', action='store_true')
+parser.add_argument('--eol-unused-not-latest', action='store_true')
+parser.add_argument('--eol-not-latest', action='store_true')
+parser.add_argument('--unused-not-latest', action='store_true')
+parser.add_argument(
+    '--use-broker', action='store_true',
+    help='use the identity broker to get credentials')
+parser.add_argument('cache_file')
+args = parser.parse_args()
+
+log = logging.getLogger()
+log.setLevel(logging.DEBUG if args.debug else logging.INFO)
+console = logging.StreamHandler()
+logfmt = logging.Formatter(LOGFORMAT, datefmt='%FT%TZ')
+logfmt.converter = time.gmtime
+console.setFormatter(logfmt)
+log.addHandler(console)
+log.debug(args)
+
+# set up credential provider, if we're going to use it
+if args.use_broker:
+    clouds.set_credential_provider(debug=args.debug)
+
+# what region(s)?
+regions = clouds.ADAPTERS[args.cloud].regions
+if args.region:
+    if args.region not in regions:
+        log.error('invalid region: %s', args.region)
+        exit(1)
+    else:
+        regions = [args.region]
+
+filters = {
+    'Owners': ['self'],
+    'Filters': [
+        {'Name': 'state', 'Values': ['available']},
+    ]
+}
+
+initial = dictfactory()
+variants = dictfactory()
+removes = dictfactory()
+summary = dictfactory()
+latest = {}
+now = time.gmtime()
+
+# load cache
+yaml = YAML()
+log.info(f'loading image cache from {args.cache_file}')
+cache = yaml.load(Path(args.cache_file))
+log.info(f'loaded image cache')
+
+
+for region in sorted(regions):
+    latest = cache[region]['latest']
+    images = cache[region]['images']
+    log.info(f'--- {region} : {len(images)} ---')
+
+    for id, image in images.items():
+        name = image['name']
+
+        if args.private and image['private']:
+            log.info(f"{region}\tPRIVATE\t{name}")
+            removes[region][id] = image
+            summary[region]['PRIVATE'][id] = name
+            continue
+
+        if args.edge_eol and image['version'] == 'edge' and image['eol']:
+            log.info(f"{region}\tEDGE-EOL\t{name}")
+            removes[region][id] = image
+            summary[region]['EDGE-EOL'][id] = name
+            continue
+
+        if args.rc and image['rc']:
+            log.info(f"{region}\tRC\t{name}")
+            removes[region][id] = image
+            summary[region]['RC'][id] = name
+            continue
+
+        unused = image['launched'] == 'Never'
+        release_key = image['release_key']
+        variant_key = image['variant_key']
+        if variant_key not in latest:
+            log.warning(f"variant key '{variant_key}' not in latest, skipping.")
+            summary[region]['__WTF__'][id] = name
+            continue
+
+        latest_release_key = latest[variant_key]['release_key']
+        not_latest = release_key != latest_release_key
+
+        if args.eol_unused_not_latest and image['eol'] and unused and not_latest:
+            log.info(f"{region}\tEOL-UNUSED-NOT-LATEST\t{name}")
+            removes[region][id] = image
+            summary[region]['EOL-UNUSED-NOT-LATEST'][id] = name
+            continue
+
+        if args.eol_not_latest and image['eol'] and not_latest:
+            log.info(f"{region}\tEOL-NOT-LATEST\t{name}")
+            removes[region][id] = image
+            summary[region]['EOL-NOT-LATEST'][id] = name
+            continue
+
+        if args.unused_not_latest and unused and not_latest:
+            log.info(f"{region}\tUNUSED-NOT-LATEST\t{name}")
+            removes[region][id] = image
+            summary[region]['UNUSED-NOT-LATEST'][id] = name
+            continue
+
+        log.debug(f"{region}\t__KEPT__\t{name}")
+        summary[region]['__KEPT__'][id] = name
+
+totals = {}
+log.info('SUMMARY')
+for region, reasons in sorted(summary.items()):
+    log.info(f"\t{region}")
+    for reason, images in sorted(reasons.items()):
+        count = len(images)
+        log.info(f"\t\t{count}\t{reason}")
+        if reason not in totals:
+            totals[reason] = 0
+
+        totals[reason] += count
+
+log.info('TOTALS')
+for reason, count in sorted(totals.items()):
+    log.info(f"\t{count}\t{reason}")
+
+if args.really:
+    log.warning('Please confirm you wish to actually prune these images...')
+    r = input("(yes/NO): ")
+    print()
+    if r.lower() != 'yes':
+        args.really = False
+
+if not args.really:
+    log.warning("Not really pruning any images.")
+    exit(0)
+
+# do the pruning...
+
+for region, images in sorted(removes.items()):
+    ec2r = clouds.ADAPTERS[args.cloud].session(region).resource('ec2')
+    for id, image in images.items():
+        name = image['name']
+        snapshot_id = image['snapshot_id']
+        try:
+            log.info(f'Deregistering: {region}/{id}: {name}')
+            ec2r.Image(id).deregister()
+            log.info(f"Deleting: {region}/{snapshot_id}: {name}")
+            ec2r.Snapshot(snapshot_id).delete()
+
+        except Exception as e:
+            log.warning(f"Failed: {e}")
+            pass
+
+log.info('DONE')

alpine-cloud-images/scripts/setup-tiny

@@ -9,6 +9,10 @@ einfo() {
     printf '\n\033[1;7;36m> %s <\033[0m\n' "$@" >&2     # bold reversed cyan
 }
 
+greater_or_equal() {
+    return $(echo "$1 $2" | awk '{print ($1 < $2)}')
+}
+
 if [ "$VERSION" = "3.12" ]; then
     # tiny-cloud-network requires ifupdown-ng, not in 3.12
     einfo "Configuring Tiny EC2 Bootstrap..."
@@ -25,19 +29,8 @@ else
 
     # tiny-cloud >= 3.0.0 sets up init scripts with /sbin/tiny-cloud --setup
     if [ -f "$TARGET/sbin/tiny-cloud" ]; then
-        # fixed in tiny-cloud >3.0.1
+        chroot "$TARGET" /sbin/tiny-cloud --enable
-        #chroot "$TARGET" /sbin/tiny-cloud --enable
+    elif greater_or_equal "$VERSION" 3.18; then
-        # logic directly implemented here, for now
-        echo -- "- removing tiny-cloud* from all runlevels"
-		rm -f "$TARGET"/etc/runlevels/*/tiny-cloud*
-		ln -s /etc/init.d/tiny-cloud-boot "$TARGET"/etc/runlevels/boot
-        echo -- "+ tiny-cloud-boot service added to boot runlevel"
-        for p in early main final; do
-            ln -s "/etc/init.d/tiny-cloud-$p" "$TARGET"/etc/runlevels/default
-            echo -- "+ tiny-cloud-$p service added to default runlevel"
-        done
-    # TODO: will need to update this for >3.18
-    elif [ "$VERSION" = "3.18" ]; then
         # 3.18 has tiny-cloud 3.0.0, and we didn't find what we expected
         echo "Error: /sbin/tiny-cloud not found" >&2
         exit 1