Authoritative EOL / Publish Updates Tags & Description
Implement alpine lib as a class * get versions/releases/EOLs from authoritative source * methods to build appropriate URLs * fallback to old method of determining release for RC versions * compute edge & RC EOLs here instead of elsewhere Remove end_of_life from configs, and don't return it from imported images. Always update image tags and descriptions when re/publishing images. Fix image description URL... :P
This commit is contained in:
parent
31b84a9dd1
commit
a8fae241f0
112
alpine.py
112
alpine.py
@ -1,35 +1,103 @@
|
|||||||
# vim: ts=4 et:
|
# vim: ts=4 et:
|
||||||
|
|
||||||
|
import json
|
||||||
import re
|
import re
|
||||||
|
from datetime import datetime, timedelta
|
||||||
from urllib.request import urlopen
|
from urllib.request import urlopen
|
||||||
|
|
||||||
|
|
||||||
# constants and vars
|
class Alpine():
|
||||||
CDN_URL = 'https://dl-cdn.alpinelinux.org/alpine'
|
|
||||||
|
|
||||||
|
DEFAULT_RELEASES_URL = 'https://alpinelinux.org/releases.json'
|
||||||
|
DEFAULT_CDN_URL = 'https://dl-cdn.alpinelinux.org/alpine'
|
||||||
|
|
||||||
# TODO: also get EOL from authoritative source
|
def __init__(self, releases_url=None, cdn_url=None):
|
||||||
|
self.now = datetime.utcnow()
|
||||||
|
self.release_today = self.now.strftime('%Y%m%d')
|
||||||
|
self.eol_tomorrow = (self.now + timedelta(days=1)).strftime('%F')
|
||||||
|
self.latest = None
|
||||||
|
self.versions = {}
|
||||||
|
self.releases_url = releases_url or self.DEFAULT_RELEASES_URL
|
||||||
|
self.cdn_url = cdn_url or self.DEFAULT_CDN_URL
|
||||||
|
|
||||||
def get_version_release(alpine_version):
|
# get all Alpine versions, and their EOL and latest release
|
||||||
apk_ver = get_apk_version(alpine_version, 'main', 'x86_64', 'alpine-base')
|
res = urlopen(self.releases_url)
|
||||||
release = apk_ver.split('-')[0]
|
r = json.load(res)
|
||||||
version = '.'.join(release.split('.')[:2])
|
branches = sorted(
|
||||||
return {'version': version, 'release': release}
|
r['release_branches'], reverse=True,
|
||||||
|
key=lambda x: x.get('branch_date', '0000-00-00')
|
||||||
|
)
|
||||||
|
for b in branches:
|
||||||
|
ver = b['rel_branch'].lstrip('v')
|
||||||
|
if not self.latest:
|
||||||
|
self.latest = ver
|
||||||
|
|
||||||
|
rel = None
|
||||||
|
if releases := b.get('releases', None):
|
||||||
|
rel = sorted(
|
||||||
|
releases, reverse=True, key=lambda x: x['date']
|
||||||
|
)[0]['version']
|
||||||
|
elif ver == 'edge':
|
||||||
|
# edge "releases" is today's YYYYMMDD
|
||||||
|
rel = self.release_today
|
||||||
|
|
||||||
# TODO? maybe download and parse APKINDEX instead?
|
self.versions[ver] = {
|
||||||
# also check out https://dl-cdn.alpinelinux.org/alpine/v3.15/releases/x86_64/latest-releases.yaml
|
'version': ver,
|
||||||
def get_apk_version(alpine_version, repo, arch, apk):
|
'release': rel,
|
||||||
repo_url = f"{CDN_URL}/{alpine_version}/{repo}/{arch}"
|
'end_of_life': b.get('eol_date', self.eol_tomorrow),
|
||||||
apks_re = re.compile(f'"{apk}-(\\d.*)\\.apk"')
|
'arches': b.get('arches'),
|
||||||
res = urlopen(repo_url)
|
}
|
||||||
for line in map(lambda x: x.decode('utf8'), res):
|
|
||||||
if not line.startswith('<a href="'):
|
|
||||||
continue
|
|
||||||
|
|
||||||
match = apks_re.search(line)
|
def _ver(self, ver=None):
|
||||||
if match:
|
if not ver or ver == 'latest' or ver == 'latest-stable':
|
||||||
return match.group(1)
|
ver = self.latest
|
||||||
|
|
||||||
# didn't find it?
|
return ver
|
||||||
raise RuntimeError(f"Unable to find {apk} APK via {repo_url}")
|
|
||||||
|
def repo_url(self, repo, arch, ver=None):
|
||||||
|
ver = self._ver(ver)
|
||||||
|
if ver != 'edge':
|
||||||
|
ver = 'v' + ver
|
||||||
|
|
||||||
|
return f"{self.cdn_url}/{ver}/{repo}/{arch}"
|
||||||
|
|
||||||
|
def virt_iso_url(self, arch, ver=None):
|
||||||
|
ver = self._ver(ver)
|
||||||
|
rel = self.versions[ver]['release']
|
||||||
|
return f"{self.cdn_url}/v{ver}/releases/{arch}/alpine-virt-{rel}-{arch}.iso"
|
||||||
|
|
||||||
|
def version_info(self, ver=None):
|
||||||
|
ver = self._ver(ver)
|
||||||
|
if ver not in self.versions:
|
||||||
|
# perhaps a release candidate?
|
||||||
|
apk_ver = self.apk_version('main', 'x86_64', 'alpine-base', ver=ver)
|
||||||
|
rel = apk_ver.split('-')[0]
|
||||||
|
ver = '.'.join(rel.split('.')[:2])
|
||||||
|
self.versions[ver] = {
|
||||||
|
'version': ver,
|
||||||
|
'release': rel,
|
||||||
|
'end_of_life': self.eol_tomorrow,
|
||||||
|
'arches': self.versions['edge']['arches'], # reasonable assumption
|
||||||
|
}
|
||||||
|
|
||||||
|
return self.versions[ver]
|
||||||
|
|
||||||
|
# TODO? maybe implement apk_info() to read from APKINDEX, but for now
|
||||||
|
# this apk_version() seems faster and gets what we need
|
||||||
|
|
||||||
|
def apk_version(self, repo, arch, apk, ver=None):
|
||||||
|
ver = self._ver(ver)
|
||||||
|
repo_url = self.repo_url(repo, arch, ver=ver)
|
||||||
|
apks_re = re.compile(f'"{apk}-(\\d.*)\\.apk"')
|
||||||
|
print(repo_url)
|
||||||
|
res = urlopen(repo_url)
|
||||||
|
for line in map(lambda x: x.decode('utf8'), res):
|
||||||
|
if not line.startswith('<a href="'):
|
||||||
|
continue
|
||||||
|
|
||||||
|
match = apks_re.search(line)
|
||||||
|
if match:
|
||||||
|
return match.group(1)
|
||||||
|
|
||||||
|
# didn't find it?
|
||||||
|
raise RuntimeError(f"Unable to find {apk} APK via {repo_url}")
|
||||||
|
13
build
13
build
@ -40,8 +40,8 @@ import time
|
|||||||
from subprocess import Popen, PIPE
|
from subprocess import Popen, PIPE
|
||||||
from urllib.request import urlopen
|
from urllib.request import urlopen
|
||||||
|
|
||||||
import alpine
|
|
||||||
import clouds
|
import clouds
|
||||||
|
from alpine import Alpine
|
||||||
from image_configs import ImageConfigManager
|
from image_configs import ImageConfigManager
|
||||||
|
|
||||||
|
|
||||||
@ -55,8 +55,7 @@ OVMF_FIRMWARE = {
|
|||||||
'aarch64': 'usr/share/OVMF/QEMU_EFI.fd',
|
'aarch64': 'usr/share/OVMF/QEMU_EFI.fd',
|
||||||
'x86_64': 'usr/share/OVMF/OVMF.fd'
|
'x86_64': 'usr/share/OVMF/OVMF.fd'
|
||||||
}
|
}
|
||||||
ISO_URL_FORMAT = f"{alpine.CDN_URL}/" \
|
alpine = Alpine()
|
||||||
'v{version}/releases/{{arch}}/alpine-virt-{release}-{{arch}}.iso'
|
|
||||||
|
|
||||||
|
|
||||||
### Functions
|
### Functions
|
||||||
@ -162,8 +161,8 @@ def install_qemu_firmware():
|
|||||||
|
|
||||||
os.makedirs(firm_dir)
|
os.makedirs(firm_dir)
|
||||||
for arch, bin in OVMF_FIRMWARE.items():
|
for arch, bin in OVMF_FIRMWARE.items():
|
||||||
v = alpine.get_apk_version('latest-stable', 'community', arch, 'ovmf')
|
v = alpine.apk_version('community', arch, 'ovmf')
|
||||||
ovmf_url = f"{alpine.CDN_URL}/latest-stable/community/{arch}/ovmf-{v}.apk"
|
ovmf_url = f"{alpine.repo_url('community', arch)}/ovmf-{v}.apk"
|
||||||
data = urlopen(ovmf_url).read()
|
data = urlopen(ovmf_url).read()
|
||||||
|
|
||||||
# Python tarfile library can't extract from APKs
|
# Python tarfile library can't extract from APKs
|
||||||
@ -235,7 +234,7 @@ if args.use_broker:
|
|||||||
|
|
||||||
### Setup Configs
|
### Setup Configs
|
||||||
|
|
||||||
latest = alpine.get_version_release('latest-stable')
|
latest = alpine.version_info()
|
||||||
log.info('Latest Alpine version %s and release %s', latest['version'], latest['release'])
|
log.info('Latest Alpine version %s and release %s', latest['version'], latest['release'])
|
||||||
|
|
||||||
if args.clean:
|
if args.clean:
|
||||||
@ -248,7 +247,7 @@ image_configs = ImageConfigManager(
|
|||||||
conf_path='work/configs/images.conf',
|
conf_path='work/configs/images.conf',
|
||||||
yaml_path='work/images.yaml',
|
yaml_path='work/images.yaml',
|
||||||
log='build',
|
log='build',
|
||||||
iso_url_format=ISO_URL_FORMAT.format(**latest)
|
alpine=alpine,
|
||||||
)
|
)
|
||||||
|
|
||||||
log.info('Configuration Complete')
|
log.info('Configuration Complete')
|
||||||
|
@ -16,7 +16,6 @@ from image_configs import Tags, DictObj
|
|||||||
class AWSCloudAdapter(CloudAdapterInterface):
|
class AWSCloudAdapter(CloudAdapterInterface):
|
||||||
IMAGE_INFO = [
|
IMAGE_INFO = [
|
||||||
'revision', 'imported', 'import_id', 'import_region', 'published',
|
'revision', 'imported', 'import_id', 'import_region', 'published',
|
||||||
'end_of_life',
|
|
||||||
]
|
]
|
||||||
CRED_MAP = {
|
CRED_MAP = {
|
||||||
'access_key': 'aws_access_key_id',
|
'access_key': 'aws_access_key_id',
|
||||||
@ -268,7 +267,9 @@ class AWSCloudAdapter(CloudAdapterInterface):
|
|||||||
source_region = source_image.import_region
|
source_region = source_image.import_region
|
||||||
log.info('Publishing source: %s/%s', source_region, source_id)
|
log.info('Publishing source: %s/%s', source_region, source_id)
|
||||||
source = self.session().resource('ec2').Image(source_id)
|
source = self.session().resource('ec2').Image(source_id)
|
||||||
source_tags = Tags(from_list=source.tags)
|
|
||||||
|
# we may be updating tags, get them from image config
|
||||||
|
tags = ic.tags
|
||||||
|
|
||||||
# sort out published image access permissions
|
# sort out published image access permissions
|
||||||
perms = {'groups': [], 'users': []}
|
perms = {'groups': [], 'users': []}
|
||||||
@ -334,15 +335,11 @@ class AWSCloudAdapter(CloudAdapterInterface):
|
|||||||
if image.state == 'available':
|
if image.state == 'available':
|
||||||
# tag image
|
# tag image
|
||||||
log.info('%s: Adding tags to %s', r, image.id)
|
log.info('%s: Adding tags to %s', r, image.id)
|
||||||
tags = Tags(from_list=image.tags)
|
image_tags = Tags(from_list=image.tags)
|
||||||
fresh = False
|
fresh = False
|
||||||
if 'published' not in tags:
|
if 'published' not in image_tags:
|
||||||
fresh = True
|
fresh = True
|
||||||
|
|
||||||
if not tags:
|
|
||||||
# fallback to source image's tags
|
|
||||||
tags = Tags(source_tags)
|
|
||||||
|
|
||||||
if fresh:
|
if fresh:
|
||||||
tags.published = datetime.utcnow().isoformat()
|
tags.published = datetime.utcnow().isoformat()
|
||||||
|
|
||||||
@ -352,7 +349,13 @@ class AWSCloudAdapter(CloudAdapterInterface):
|
|||||||
snapshot = self.session(r).resource('ec2').Snapshot(
|
snapshot = self.session(r).resource('ec2').Snapshot(
|
||||||
image.block_device_mappings[0]['Ebs']['SnapshotId']
|
image.block_device_mappings[0]['Ebs']['SnapshotId']
|
||||||
)
|
)
|
||||||
snapshot.create_tags(Tags=image.tags)
|
snapshot.create_tags(Tags=tags.as_list())
|
||||||
|
|
||||||
|
# update image description to match description in tags
|
||||||
|
log.info('%s: Updating description to %s', r, tags.description)
|
||||||
|
image.modify_attribute(
|
||||||
|
Description={'Value': tags.description},
|
||||||
|
)
|
||||||
|
|
||||||
# apply launch perms
|
# apply launch perms
|
||||||
log.info('%s: Applying launch perms to %s', r, image.id)
|
log.info('%s: Applying launch perms to %s', r, image.id)
|
||||||
@ -369,7 +372,7 @@ class AWSCloudAdapter(CloudAdapterInterface):
|
|||||||
log.info('%s: Setting EOL deprecation time on %s', r, image.id)
|
log.info('%s: Setting EOL deprecation time on %s', r, image.id)
|
||||||
ec2c.enable_image_deprecation(
|
ec2c.enable_image_deprecation(
|
||||||
ImageId=image.id,
|
ImageId=image.id,
|
||||||
DeprecateAt=f"{source_image.end_of_life}T23:59:59Z"
|
DeprecateAt=f"{tags.end_of_life}T23:59:59Z"
|
||||||
)
|
)
|
||||||
|
|
||||||
artifacts[r] = image.id
|
artifacts[r] = image.id
|
||||||
|
@ -65,7 +65,7 @@ Dimensions {
|
|||||||
# all build configs merge these at the very end
|
# all build configs merge these at the very end
|
||||||
Mandatory {
|
Mandatory {
|
||||||
name = [ "r{revision}" ]
|
name = [ "r{revision}" ]
|
||||||
description = [ - https://alpine.linux.org/cloud ]
|
description = [ - https://alpinelinux.org/cloud ]
|
||||||
|
|
||||||
motd {
|
motd {
|
||||||
motd_change = "You may change this message by editing /etc/motd."
|
motd_change = "You may change this message by editing /etc/motd."
|
||||||
|
@ -2,7 +2,5 @@
|
|||||||
|
|
||||||
include required("base/1.conf")
|
include required("base/1.conf")
|
||||||
|
|
||||||
end_of_life = "2021-11-01"
|
|
||||||
|
|
||||||
# Alpine 3.11 doesn't support aarch64
|
# Alpine 3.11 doesn't support aarch64
|
||||||
EXCLUDE = [ aarch64 ]
|
EXCLUDE = [ aarch64 ]
|
||||||
|
@ -1,5 +1,3 @@
|
|||||||
# vim: ts=2 et:
|
# vim: ts=2 et:
|
||||||
|
|
||||||
include required("base/1.conf")
|
include required("base/1.conf")
|
||||||
|
|
||||||
end_of_life = "2022-05-01"
|
|
@ -1,5 +1,3 @@
|
|||||||
# vim: ts=2 et:
|
# vim: ts=2 et:
|
||||||
|
|
||||||
include required("base/2.conf")
|
include required("base/2.conf")
|
||||||
|
|
||||||
end_of_life = "2022-11-01"
|
|
@ -1,5 +1,3 @@
|
|||||||
# vim: ts=2 et:
|
# vim: ts=2 et:
|
||||||
|
|
||||||
include required("base/2.conf")
|
include required("base/2.conf")
|
||||||
|
|
||||||
end_of_life = "2023-05-01"
|
|
@ -2,8 +2,6 @@
|
|||||||
|
|
||||||
include required("base/3.conf")
|
include required("base/3.conf")
|
||||||
|
|
||||||
end_of_life = "2023-11-01"
|
|
||||||
|
|
||||||
motd {
|
motd {
|
||||||
sudo_deprecated = "NOTE: 'sudo' has been deprecated, please use 'doas' instead."
|
sudo_deprecated = "NOTE: 'sudo' has been deprecated, please use 'doas' instead."
|
||||||
}
|
}
|
@ -7,24 +7,22 @@ import pyhocon
|
|||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from ruamel.yaml import YAML
|
from ruamel.yaml import YAML
|
||||||
|
|
||||||
from alpine import get_version_release
|
|
||||||
import clouds
|
import clouds
|
||||||
|
|
||||||
|
|
||||||
class ImageConfigManager():
|
class ImageConfigManager():
|
||||||
|
|
||||||
def __init__(self, conf_path, yaml_path, log=__name__, iso_url_format=None):
|
def __init__(self, conf_path, yaml_path, log=__name__, alpine=None):
|
||||||
self.conf_path = Path(conf_path)
|
self.conf_path = Path(conf_path)
|
||||||
self.yaml_path = Path(yaml_path)
|
self.yaml_path = Path(yaml_path)
|
||||||
self.log = logging.getLogger(log)
|
self.log = logging.getLogger(log)
|
||||||
self.iso_url_format = iso_url_format
|
self.alpine = alpine
|
||||||
|
|
||||||
self.now = datetime.utcnow()
|
self.now = datetime.utcnow()
|
||||||
self.tomorrow = self.now + timedelta(days=1)
|
|
||||||
self._configs = {}
|
self._configs = {}
|
||||||
|
|
||||||
self.yaml = YAML()
|
self.yaml = YAML()
|
||||||
@ -134,7 +132,7 @@ class ImageConfigManager():
|
|||||||
|
|
||||||
# clean stuff up
|
# clean stuff up
|
||||||
image_config._normalize()
|
image_config._normalize()
|
||||||
image_config.qemu['iso_url'] = self.iso_url_format.format(arch=image_config.arch)
|
image_config.qemu['iso_url'] = self.alpine.virt_iso_url(arch=image_config.arch)
|
||||||
|
|
||||||
# we've resolved everything, add tags attribute to config
|
# we've resolved everything, add tags attribute to config
|
||||||
self._configs[config_key] = image_config
|
self._configs[config_key] = image_config
|
||||||
@ -143,11 +141,9 @@ class ImageConfigManager():
|
|||||||
|
|
||||||
# set current version release
|
# set current version release
|
||||||
def _set_version_release(self, v, c):
|
def _set_version_release(self, v, c):
|
||||||
if v == 'edge':
|
info = self.alpine.version_info(v)
|
||||||
c.put('release', self.now.strftime('%Y%m%d'))
|
c.put('release', info['release'])
|
||||||
c.put('end_of_life', self.tomorrow.strftime('%F'))
|
c.put('end_of_life', info['end_of_life'])
|
||||||
else:
|
|
||||||
c.put('release', get_version_release(f"v{v}")['release'])
|
|
||||||
|
|
||||||
# release is also appended to name & description arrays
|
# release is also appended to name & description arrays
|
||||||
c.put('name', [c.release])
|
c.put('name', [c.release])
|
||||||
@ -427,11 +423,6 @@ class ImageConfig():
|
|||||||
'import_region': None,
|
'import_region': None,
|
||||||
'published': None,
|
'published': None,
|
||||||
}
|
}
|
||||||
self.end_of_life = self.__dict__.pop(
|
|
||||||
'end_of_life',
|
|
||||||
# EOL is tomorrow, if otherwise unset
|
|
||||||
(datetime.utcnow() + timedelta(days=1)).strftime('%F')
|
|
||||||
)
|
|
||||||
|
|
||||||
# update artifacts, if we've got 'em
|
# update artifacts, if we've got 'em
|
||||||
artifacts_yaml = self.local_dir / 'artifacts.yaml'
|
artifacts_yaml = self.local_dir / 'artifacts.yaml'
|
||||||
|
Loading…
Reference in New Issue
Block a user