Merge branch 'fixes/authoritative-EOLs_etc' into 'main'
Authoritative EOL / Publish Updates Tags & Description See merge request tomalok/alpine-cloud-images!129
This commit is contained in:
Jake Buchholz Göktürk
commit
0cf623f7a5
112
alpine.py
112
alpine.py
|
|
@ -1,35 +1,103 @@
|
|||
# vim: ts=4 et:
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timedelta
|
||||
from urllib.request import urlopen
|
||||
|
||||
|
||||
# constants and vars
|
||||
CDN_URL = 'https://dl-cdn.alpinelinux.org/alpine'
|
||||
class Alpine():
|
||||
|
||||
DEFAULT_RELEASES_URL = 'https://alpinelinux.org/releases.json'
|
||||
DEFAULT_CDN_URL = 'https://dl-cdn.alpinelinux.org/alpine'
|
||||
|
||||
# TODO: also get EOL from authoritative source
|
||||
def __init__(self, releases_url=None, cdn_url=None):
|
||||
self.now = datetime.utcnow()
|
||||
self.release_today = self.now.strftime('%Y%m%d')
|
||||
self.eol_tomorrow = (self.now + timedelta(days=1)).strftime('%F')
|
||||
self.latest = None
|
||||
self.versions = {}
|
||||
self.releases_url = releases_url or self.DEFAULT_RELEASES_URL
|
||||
self.cdn_url = cdn_url or self.DEFAULT_CDN_URL
|
||||
|
||||
def get_version_release(alpine_version):
|
||||
apk_ver = get_apk_version(alpine_version, 'main', 'x86_64', 'alpine-base')
|
||||
release = apk_ver.split('-')[0]
|
||||
version = '.'.join(release.split('.')[:2])
|
||||
return {'version': version, 'release': release}
|
||||
# get all Alpine versions, and their EOL and latest release
|
||||
res = urlopen(self.releases_url)
|
||||
r = json.load(res)
|
||||
branches = sorted(
|
||||
r['release_branches'], reverse=True,
|
||||
key=lambda x: x.get('branch_date', '0000-00-00')
|
||||
)
|
||||
for b in branches:
|
||||
ver = b['rel_branch'].lstrip('v')
|
||||
if not self.latest:
|
||||
self.latest = ver
|
||||
|
||||
rel = None
|
||||
if releases := b.get('releases', None):
|
||||
rel = sorted(
|
||||
releases, reverse=True, key=lambda x: x['date']
|
||||
)[0]['version']
|
||||
elif ver == 'edge':
|
||||
# edge "releases" is today's YYYYMMDD
|
||||
rel = self.release_today
|
||||
|
||||
# TODO? maybe download and parse APKINDEX instead?
|
||||
# also check out https://dl-cdn.alpinelinux.org/alpine/v3.15/releases/x86_64/latest-releases.yaml
|
||||
def get_apk_version(alpine_version, repo, arch, apk):
|
||||
repo_url = f"{CDN_URL}/{alpine_version}/{repo}/{arch}"
|
||||
apks_re = re.compile(f'"{apk}-(\\d.*)\\.apk"')
|
||||
res = urlopen(repo_url)
|
||||
for line in map(lambda x: x.decode('utf8'), res):
|
||||
if not line.startswith('<a href="'):
|
||||
continue
|
||||
self.versions[ver] = {
|
||||
'version': ver,
|
||||
'release': rel,
|
||||
'end_of_life': b.get('eol_date', self.eol_tomorrow),
|
||||
'arches': b.get('arches'),
|
||||
}
|
||||
|
||||
match = apks_re.search(line)
|
||||
if match:
|
||||
return match.group(1)
|
||||
def _ver(self, ver=None):
|
||||
if not ver or ver == 'latest' or ver == 'latest-stable':
|
||||
ver = self.latest
|
||||
|
||||
# didn't find it?
|
||||
raise RuntimeError(f"Unable to find {apk} APK via {repo_url}")
|
||||
return ver
|
||||
|
||||
def repo_url(self, repo, arch, ver=None):
|
||||
ver = self._ver(ver)
|
||||
if ver != 'edge':
|
||||
ver = 'v' + ver
|
||||
|
||||
return f"{self.cdn_url}/{ver}/{repo}/{arch}"
|
||||
|
||||
def virt_iso_url(self, arch, ver=None):
|
||||
ver = self._ver(ver)
|
||||
rel = self.versions[ver]['release']
|
||||
return f"{self.cdn_url}/v{ver}/releases/{arch}/alpine-virt-{rel}-{arch}.iso"
|
||||
|
||||
def version_info(self, ver=None):
|
||||
ver = self._ver(ver)
|
||||
if ver not in self.versions:
|
||||
# perhaps a release candidate?
|
||||
apk_ver = self.apk_version('main', 'x86_64', 'alpine-base', ver=ver)
|
||||
rel = apk_ver.split('-')[0]
|
||||
ver = '.'.join(rel.split('.')[:2])
|
||||
self.versions[ver] = {
|
||||
'version': ver,
|
||||
'release': rel,
|
||||
'end_of_life': self.eol_tomorrow,
|
||||
'arches': self.versions['edge']['arches'], # reasonable assumption
|
||||
}
|
||||
|
||||
return self.versions[ver]
|
||||
|
||||
# TODO? maybe implement apk_info() to read from APKINDEX, but for now
|
||||
# this apk_version() seems faster and gets what we need
|
||||
|
||||
def apk_version(self, repo, arch, apk, ver=None):
|
||||
ver = self._ver(ver)
|
||||
repo_url = self.repo_url(repo, arch, ver=ver)
|
||||
apks_re = re.compile(f'"{apk}-(\\d.*)\\.apk"')
|
||||
print(repo_url)
|
||||
res = urlopen(repo_url)
|
||||
for line in map(lambda x: x.decode('utf8'), res):
|
||||
if not line.startswith('<a href="'):
|
||||
continue
|
||||
|
||||
match = apks_re.search(line)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
# didn't find it?
|
||||
raise RuntimeError(f"Unable to find {apk} APK via {repo_url}")
|
||||
|
|
|
|||
13
build
13
build
|
|
@ -40,8 +40,8 @@ import time
|
|||
from subprocess import Popen, PIPE
|
||||
from urllib.request import urlopen
|
||||
|
||||
import alpine
|
||||
import clouds
|
||||
from alpine import Alpine
|
||||
from image_configs import ImageConfigManager
|
||||
|
||||
|
||||
|
|
@ -55,8 +55,7 @@ OVMF_FIRMWARE = {
|
|||
'aarch64': 'usr/share/OVMF/QEMU_EFI.fd',
|
||||
'x86_64': 'usr/share/OVMF/OVMF.fd'
|
||||
}
|
||||
ISO_URL_FORMAT = f"{alpine.CDN_URL}/" \
|
||||
'v{version}/releases/{{arch}}/alpine-virt-{release}-{{arch}}.iso'
|
||||
alpine = Alpine()
|
||||
|
||||
|
||||
### Functions
|
||||
|
|
@ -162,8 +161,8 @@ def install_qemu_firmware():
|
|||
|
||||
os.makedirs(firm_dir)
|
||||
for arch, bin in OVMF_FIRMWARE.items():
|
||||
v = alpine.get_apk_version('latest-stable', 'community', arch, 'ovmf')
|
||||
ovmf_url = f"{alpine.CDN_URL}/latest-stable/community/{arch}/ovmf-{v}.apk"
|
||||
v = alpine.apk_version('community', arch, 'ovmf')
|
||||
ovmf_url = f"{alpine.repo_url('community', arch)}/ovmf-{v}.apk"
|
||||
data = urlopen(ovmf_url).read()
|
||||
|
||||
# Python tarfile library can't extract from APKs
|
||||
|
|
@ -235,7 +234,7 @@ if args.use_broker:
|
|||
|
||||
### Setup Configs
|
||||
|
||||
latest = alpine.get_version_release('latest-stable')
|
||||
latest = alpine.version_info()
|
||||
log.info('Latest Alpine version %s and release %s', latest['version'], latest['release'])
|
||||
|
||||
if args.clean:
|
||||
|
|
@ -248,7 +247,7 @@ image_configs = ImageConfigManager(
|
|||
conf_path='work/configs/images.conf',
|
||||
yaml_path='work/images.yaml',
|
||||
log='build',
|
||||
iso_url_format=ISO_URL_FORMAT.format(**latest)
|
||||
alpine=alpine,
|
||||
)
|
||||
|
||||
log.info('Configuration Complete')
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ from image_configs import Tags, DictObj
|
|||
class AWSCloudAdapter(CloudAdapterInterface):
|
||||
IMAGE_INFO = [
|
||||
'revision', 'imported', 'import_id', 'import_region', 'published',
|
||||
'end_of_life',
|
||||
]
|
||||
CRED_MAP = {
|
||||
'access_key': 'aws_access_key_id',
|
||||
|
|
@ -268,7 +267,9 @@ class AWSCloudAdapter(CloudAdapterInterface):
|
|||
source_region = source_image.import_region
|
||||
log.info('Publishing source: %s/%s', source_region, source_id)
|
||||
source = self.session().resource('ec2').Image(source_id)
|
||||
source_tags = Tags(from_list=source.tags)
|
||||
|
||||
# we may be updating tags, get them from image config
|
||||
tags = ic.tags
|
||||
|
||||
# sort out published image access permissions
|
||||
perms = {'groups': [], 'users': []}
|
||||
|
|
@ -334,15 +335,11 @@ class AWSCloudAdapter(CloudAdapterInterface):
|
|||
if image.state == 'available':
|
||||
# tag image
|
||||
log.info('%s: Adding tags to %s', r, image.id)
|
||||
tags = Tags(from_list=image.tags)
|
||||
image_tags = Tags(from_list=image.tags)
|
||||
fresh = False
|
||||
if 'published' not in tags:
|
||||
if 'published' not in image_tags:
|
||||
fresh = True
|
||||
|
||||
if not tags:
|
||||
# fallback to source image's tags
|
||||
tags = Tags(source_tags)
|
||||
|
||||
if fresh:
|
||||
tags.published = datetime.utcnow().isoformat()
|
||||
|
||||
|
|
@ -352,7 +349,13 @@ class AWSCloudAdapter(CloudAdapterInterface):
|
|||
snapshot = self.session(r).resource('ec2').Snapshot(
|
||||
image.block_device_mappings[0]['Ebs']['SnapshotId']
|
||||
)
|
||||
snapshot.create_tags(Tags=image.tags)
|
||||
snapshot.create_tags(Tags=tags.as_list())
|
||||
|
||||
# update image description to match description in tags
|
||||
log.info('%s: Updating description to %s', r, tags.description)
|
||||
image.modify_attribute(
|
||||
Description={'Value': tags.description},
|
||||
)
|
||||
|
||||
# apply launch perms
|
||||
log.info('%s: Applying launch perms to %s', r, image.id)
|
||||
|
|
@ -369,7 +372,7 @@ class AWSCloudAdapter(CloudAdapterInterface):
|
|||
log.info('%s: Setting EOL deprecation time on %s', r, image.id)
|
||||
ec2c.enable_image_deprecation(
|
||||
ImageId=image.id,
|
||||
DeprecateAt=f"{source_image.end_of_life}T23:59:59Z"
|
||||
DeprecateAt=f"{tags.end_of_life}T23:59:59Z"
|
||||
)
|
||||
|
||||
artifacts[r] = image.id
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ Dimensions {
|
|||
# all build configs merge these at the very end
|
||||
Mandatory {
|
||||
name = [ "r{revision}" ]
|
||||
description = [ - https://alpine.linux.org/cloud ]
|
||||
description = [ - https://alpinelinux.org/cloud ]
|
||||
|
||||
motd {
|
||||
motd_change = "You may change this message by editing /etc/motd."
|
||||
|
|
|
|||
|
|
@ -2,7 +2,5 @@
|
|||
|
||||
include required("base/1.conf")
|
||||
|
||||
end_of_life = "2021-11-01"
|
||||
|
||||
# Alpine 3.11 doesn't support aarch64
|
||||
EXCLUDE = [ aarch64 ]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# vim: ts=2 et:
|
||||
|
||||
include required("base/1.conf")
|
||||
|
||||
end_of_life = "2022-05-01"
|
||||
include required("base/1.conf")
|
||||
|
|
@ -1,5 +1,3 @@
|
|||
# vim: ts=2 et:
|
||||
|
||||
include required("base/2.conf")
|
||||
|
||||
end_of_life = "2022-11-01"
|
||||
include required("base/2.conf")
|
||||
|
|
@ -1,5 +1,3 @@
|
|||
# vim: ts=2 et:
|
||||
|
||||
include required("base/2.conf")
|
||||
|
||||
end_of_life = "2023-05-01"
|
||||
include required("base/2.conf")
|
||||
|
|
@ -2,8 +2,6 @@
|
|||
|
||||
include required("base/3.conf")
|
||||
|
||||
end_of_life = "2023-11-01"
|
||||
|
||||
motd {
|
||||
sudo_deprecated = "NOTE: 'sudo' has been deprecated, please use 'doas' instead."
|
||||
}
|
||||
|
|
@ -7,24 +7,22 @@ import pyhocon
|
|||
import shutil
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from ruamel.yaml import YAML
|
||||
|
||||
from alpine import get_version_release
|
||||
import clouds
|
||||
|
||||
|
||||
class ImageConfigManager():
|
||||
|
||||
def __init__(self, conf_path, yaml_path, log=__name__, iso_url_format=None):
|
||||
def __init__(self, conf_path, yaml_path, log=__name__, alpine=None):
|
||||
self.conf_path = Path(conf_path)
|
||||
self.yaml_path = Path(yaml_path)
|
||||
self.log = logging.getLogger(log)
|
||||
self.iso_url_format = iso_url_format
|
||||
self.alpine = alpine
|
||||
|
||||
self.now = datetime.utcnow()
|
||||
self.tomorrow = self.now + timedelta(days=1)
|
||||
self._configs = {}
|
||||
|
||||
self.yaml = YAML()
|
||||
|
|
@ -134,7 +132,7 @@ class ImageConfigManager():
|
|||
|
||||
# clean stuff up
|
||||
image_config._normalize()
|
||||
image_config.qemu['iso_url'] = self.iso_url_format.format(arch=image_config.arch)
|
||||
image_config.qemu['iso_url'] = self.alpine.virt_iso_url(arch=image_config.arch)
|
||||
|
||||
# we've resolved everything, add tags attribute to config
|
||||
self._configs[config_key] = image_config
|
||||
|
|
@ -143,11 +141,9 @@ class ImageConfigManager():
|
|||
|
||||
# set current version release
|
||||
def _set_version_release(self, v, c):
|
||||
if v == 'edge':
|
||||
c.put('release', self.now.strftime('%Y%m%d'))
|
||||
c.put('end_of_life', self.tomorrow.strftime('%F'))
|
||||
else:
|
||||
c.put('release', get_version_release(f"v{v}")['release'])
|
||||
info = self.alpine.version_info(v)
|
||||
c.put('release', info['release'])
|
||||
c.put('end_of_life', info['end_of_life'])
|
||||
|
||||
# release is also appended to name & description arrays
|
||||
c.put('name', [c.release])
|
||||
|
|
@ -427,11 +423,6 @@ class ImageConfig():
|
|||
'import_region': None,
|
||||
'published': None,
|
||||
}
|
||||
self.end_of_life = self.__dict__.pop(
|
||||
'end_of_life',
|
||||
# EOL is tomorrow, if otherwise unset
|
||||
(datetime.utcnow() + timedelta(days=1)).strftime('%F')
|
||||
)
|
||||
|
||||
# update artifacts, if we've got 'em
|
||||
artifacts_yaml = self.local_dir / 'artifacts.yaml'
|
||||
|
|
|
|||
Loading…
Reference in New Issue