Authoritative EOL / Publish Updates Tags & Description

Implement alpine lib as a class
* get versions/releases/EOLs from authoritative source
* methods to build appropriate URLs
* fallback to old method of determining release for RC versions
* compute edge & RC EOLs here instead of elsewhere

Remove end_of_life from configs, and don't return it from imported images.

Always update image tags and descriptions when re/publishing images.

Fix image description URL...  :P

alpine.py

@@ -1,35 +1,103 @@
 # vim: ts=4 et:
 
+import json
 import re
+from datetime import datetime, timedelta
 from urllib.request import urlopen
 
 
-# constants and vars
+class Alpine():
-CDN_URL = 'https://dl-cdn.alpinelinux.org/alpine'
 
+    DEFAULT_RELEASES_URL = 'https://alpinelinux.org/releases.json'
+    DEFAULT_CDN_URL = 'https://dl-cdn.alpinelinux.org/alpine'
 
-# TODO:  also get EOL from authoritative source
+    def __init__(self, releases_url=None, cdn_url=None):
+        self.now = datetime.utcnow()
+        self.release_today = self.now.strftime('%Y%m%d')
+        self.eol_tomorrow = (self.now + timedelta(days=1)).strftime('%F')
+        self.latest = None
+        self.versions = {}
+        self.releases_url = releases_url or self.DEFAULT_RELEASES_URL
+        self.cdn_url = cdn_url or self.DEFAULT_CDN_URL
 
-def get_version_release(alpine_version):
+        # get all Alpine versions, and their EOL and latest release
-    apk_ver = get_apk_version(alpine_version, 'main', 'x86_64', 'alpine-base')
+        res = urlopen(self.releases_url)
-    release = apk_ver.split('-')[0]
+        r = json.load(res)
-    version = '.'.join(release.split('.')[:2])
+        branches = sorted(
-    return {'version': version, 'release': release}
+            r['release_branches'], reverse=True,
+            key=lambda x: x.get('branch_date', '0000-00-00')
+        )
+        for b in branches:
+            ver = b['rel_branch'].lstrip('v')
+            if not self.latest:
+                self.latest = ver
 
+            rel = None
+            if releases := b.get('releases', None):
+                rel = sorted(
+                    releases, reverse=True, key=lambda x: x['date']
+                )[0]['version']
+            elif ver == 'edge':
+                # edge "releases" is today's YYYYMMDD
+                rel = self.release_today
 
-# TODO?  maybe download and parse APKINDEX instead?
+            self.versions[ver] = {
-# also check out https://dl-cdn.alpinelinux.org/alpine/v3.15/releases/x86_64/latest-releases.yaml
+                'version': ver,
-def get_apk_version(alpine_version, repo, arch, apk):
+                'release': rel,
-    repo_url = f"{CDN_URL}/{alpine_version}/{repo}/{arch}"
+                'end_of_life': b.get('eol_date', self.eol_tomorrow),
-    apks_re = re.compile(f'"{apk}-(\\d.*)\\.apk"')
+                'arches': b.get('arches'),
-    res = urlopen(repo_url)
+            }
-    for line in map(lambda x: x.decode('utf8'), res):
-        if not line.startswith('<a href="'):
-            continue
 
-        match = apks_re.search(line)
+    def _ver(self, ver=None):
-        if match:
+        if not ver or ver == 'latest' or ver == 'latest-stable':
-            return match.group(1)
+            ver = self.latest
 
-    # didn't find it?
+        return ver
-    raise RuntimeError(f"Unable to find {apk} APK via {repo_url}")
+
+    def repo_url(self, repo, arch, ver=None):
+        ver = self._ver(ver)
+        if ver != 'edge':
+            ver = 'v' + ver
+
+        return f"{self.cdn_url}/{ver}/{repo}/{arch}"
+
+    def virt_iso_url(self, arch, ver=None):
+        ver = self._ver(ver)
+        rel = self.versions[ver]['release']
+        return f"{self.cdn_url}/v{ver}/releases/{arch}/alpine-virt-{rel}-{arch}.iso"
+
+    def version_info(self, ver=None):
+        ver = self._ver(ver)
+        if ver not in self.versions:
+            # perhaps a release candidate?
+            apk_ver = self.apk_version('main', 'x86_64', 'alpine-base', ver=ver)
+            rel = apk_ver.split('-')[0]
+            ver = '.'.join(rel.split('.')[:2])
+            self.versions[ver] = {
+                'version': ver,
+                'release': rel,
+                'end_of_life': self.eol_tomorrow,
+                'arches': self.versions['edge']['arches'],  # reasonable assumption
+            }
+
+        return self.versions[ver]
+
+    # TODO?  maybe implement apk_info() to read from APKINDEX, but for now
+    #   this apk_version() seems faster and gets what we need
+
+    def apk_version(self, repo, arch, apk, ver=None):
+        ver = self._ver(ver)
+        repo_url = self.repo_url(repo, arch, ver=ver)
+        apks_re = re.compile(f'"{apk}-(\\d.*)\\.apk"')
+        print(repo_url)
+        res = urlopen(repo_url)
+        for line in map(lambda x: x.decode('utf8'), res):
+            if not line.startswith('<a href="'):
+                continue
+
+            match = apks_re.search(line)
+            if match:
+                return match.group(1)
+
+        # didn't find it?
+        raise RuntimeError(f"Unable to find {apk} APK via {repo_url}")

build

@@ -40,8 +40,8 @@ import time
 from subprocess import Popen, PIPE
 from urllib.request import urlopen
 
-import alpine
 import clouds
+from alpine import Alpine
 from image_configs import ImageConfigManager
 
 
@@ -55,8 +55,7 @@ OVMF_FIRMWARE = {
     'aarch64': 'usr/share/OVMF/QEMU_EFI.fd',
     'x86_64': 'usr/share/OVMF/OVMF.fd'
 }
-ISO_URL_FORMAT = f"{alpine.CDN_URL}/" \
+alpine = Alpine()
-    'v{version}/releases/{{arch}}/alpine-virt-{release}-{{arch}}.iso'
 
 
 ### Functions
@@ -162,8 +161,8 @@ def install_qemu_firmware():
 
     os.makedirs(firm_dir)
     for arch, bin in OVMF_FIRMWARE.items():
-        v = alpine.get_apk_version('latest-stable', 'community', arch, 'ovmf')
+        v = alpine.apk_version('community', arch, 'ovmf')
-        ovmf_url = f"{alpine.CDN_URL}/latest-stable/community/{arch}/ovmf-{v}.apk"
+        ovmf_url = f"{alpine.repo_url('community', arch)}/ovmf-{v}.apk"
         data = urlopen(ovmf_url).read()
 
         # Python tarfile library can't extract from APKs
@@ -235,7 +234,7 @@ if args.use_broker:
 
 ### Setup Configs
 
-latest = alpine.get_version_release('latest-stable')
+latest = alpine.version_info()
 log.info('Latest Alpine version %s and release %s', latest['version'], latest['release'])
 
 if args.clean:
@@ -248,7 +247,7 @@ image_configs = ImageConfigManager(
     conf_path='work/configs/images.conf',
     yaml_path='work/images.yaml',
     log='build',
-    iso_url_format=ISO_URL_FORMAT.format(**latest)
+    alpine=alpine,
 )
 
 log.info('Configuration Complete')

clouds/aws.py

@@ -16,7 +16,6 @@ from image_configs import Tags, DictObj
 class AWSCloudAdapter(CloudAdapterInterface):
     IMAGE_INFO = [
         'revision', 'imported', 'import_id', 'import_region', 'published',
-        'end_of_life',
     ]
     CRED_MAP = {
         'access_key': 'aws_access_key_id',
@@ -268,7 +267,9 @@ class AWSCloudAdapter(CloudAdapterInterface):
         source_region = source_image.import_region
         log.info('Publishing source: %s/%s', source_region, source_id)
         source = self.session().resource('ec2').Image(source_id)
-        source_tags = Tags(from_list=source.tags)
+
+        # we may be updating tags, get them from image config
+        tags = ic.tags
 
         # sort out published image access permissions
         perms = {'groups': [], 'users': []}
@@ -334,15 +335,11 @@ class AWSCloudAdapter(CloudAdapterInterface):
                     if image.state == 'available':
                         # tag image
                         log.info('%s: Adding tags to %s', r, image.id)
-                        tags = Tags(from_list=image.tags)
+                        image_tags = Tags(from_list=image.tags)
                         fresh = False
-                        if 'published' not in tags:
+                        if 'published' not in image_tags:
                             fresh = True
 
-                        if not tags:
-                            # fallback to source image's tags
-                            tags = Tags(source_tags)
-
                         if fresh:
                             tags.published = datetime.utcnow().isoformat()
 
@@ -352,7 +349,13 @@ class AWSCloudAdapter(CloudAdapterInterface):
                         snapshot = self.session(r).resource('ec2').Snapshot(
                             image.block_device_mappings[0]['Ebs']['SnapshotId']
                         )
-                        snapshot.create_tags(Tags=image.tags)
+                        snapshot.create_tags(Tags=tags.as_list())
+
+                        # update image description to match description in tags
+                        log.info('%s: Updating description to %s', r, tags.description)
+                        image.modify_attribute(
+                            Description={'Value': tags.description},
+                        )
 
                         # apply launch perms
                         log.info('%s: Applying launch perms to %s', r, image.id)
@@ -369,7 +372,7 @@ class AWSCloudAdapter(CloudAdapterInterface):
                         log.info('%s: Setting EOL deprecation time on %s', r, image.id)
                         ec2c.enable_image_deprecation(
                             ImageId=image.id,
-                            DeprecateAt=f"{source_image.end_of_life}T23:59:59Z"
+                            DeprecateAt=f"{tags.end_of_life}T23:59:59Z"
                         )
 
                         artifacts[r] = image.id

configs/alpine.conf

@@ -65,7 +65,7 @@ Dimensions {
 # all build configs merge these at the very end
 Mandatory {
   name        = [ "r{revision}" ]
-  description = [ - https://alpine.linux.org/cloud ]
+  description = [ - https://alpinelinux.org/cloud ]
 
   motd {
     motd_change = "You may change this message by editing /etc/motd."

configs/version/3.11.conf

@@ -2,7 +2,5 @@
 
 include required("base/1.conf")
 
-end_of_life = "2021-11-01"
-
 # Alpine 3.11 doesn't support aarch64
 EXCLUDE = [ aarch64 ]

configs/version/3.12.conf

@@ -1,5 +1,3 @@
 # vim: ts=2 et:
 
-include required("base/1.conf")
+include required("base/1.conf")
-
-end_of_life = "2022-05-01"

configs/version/3.13.conf

@@ -1,5 +1,3 @@
 # vim: ts=2 et:
 
-include required("base/2.conf")
+include required("base/2.conf")
-
-end_of_life = "2022-11-01"

configs/version/3.14.conf

@@ -1,5 +1,3 @@
 # vim: ts=2 et:
 
-include required("base/2.conf")
+include required("base/2.conf")
-
-end_of_life = "2023-05-01"

configs/version/3.15.conf

@@ -2,8 +2,6 @@
 
 include required("base/3.conf")
 
-end_of_life = "2023-11-01"
-
 motd {
   sudo_deprecated = "NOTE: 'sudo' has been deprecated, please use 'doas' instead."
 }

image_configs.py

@@ -7,24 +7,22 @@ import pyhocon
 import shutil
 
 from copy import deepcopy
-from datetime import datetime, timedelta
+from datetime import datetime
 from pathlib import Path
 from ruamel.yaml import YAML
 
-from alpine import get_version_release
 import clouds
 
 
 class ImageConfigManager():
 
-    def __init__(self, conf_path, yaml_path, log=__name__, iso_url_format=None):
+    def __init__(self, conf_path, yaml_path, log=__name__, alpine=None):
         self.conf_path = Path(conf_path)
         self.yaml_path = Path(yaml_path)
         self.log = logging.getLogger(log)
-        self.iso_url_format = iso_url_format
+        self.alpine = alpine
 
         self.now = datetime.utcnow()
-        self.tomorrow = self.now + timedelta(days=1)
         self._configs = {}
 
         self.yaml = YAML()
@@ -134,7 +132,7 @@ class ImageConfigManager():
 
             # clean stuff up
             image_config._normalize()
-            image_config.qemu['iso_url'] = self.iso_url_format.format(arch=image_config.arch)
+            image_config.qemu['iso_url'] = self.alpine.virt_iso_url(arch=image_config.arch)
 
             # we've resolved everything, add tags attribute to config
             self._configs[config_key] = image_config
@@ -143,11 +141,9 @@ class ImageConfigManager():
 
     # set current version release
     def _set_version_release(self, v, c):
-        if v == 'edge':
+        info = self.alpine.version_info(v)
-            c.put('release', self.now.strftime('%Y%m%d'))
+        c.put('release', info['release'])
-            c.put('end_of_life', self.tomorrow.strftime('%F'))
+        c.put('end_of_life', info['end_of_life'])
-        else:
-            c.put('release', get_version_release(f"v{v}")['release'])
 
         # release is also appended to name & description arrays
         c.put('name', [c.release])
@@ -427,11 +423,6 @@ class ImageConfig():
                 'import_region': None,
                 'published': None,
             }
-            self.end_of_life = self.__dict__.pop(
-                'end_of_life',
-                # EOL is tomorrow, if otherwise unset
-                (datetime.utcnow() + timedelta(days=1)).strftime('%F')
-            )
 
         # update artifacts, if we've got 'em
         artifacts_yaml = self.local_dir / 'artifacts.yaml'