diff --git a/Makefile b/Makefile index 3aa7464..e6f879c 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ __check_defined = \ .PHONY: amis prune release-readme clean -amis: build/packer.json build/profile/$(PROFILE) build build/setup-ami $(NVME_SCRIPTS) +amis: build/packer.json build/profile/$(PROFILE) build @:$(call check_defined, PROFILE, target profile name) build/builder make-amis $(PROFILE) $(BUILDS) @@ -54,7 +54,7 @@ build/packer.json: packer.conf build .PHONY: build/profile/$(PROFILE) build/profile/$(PROFILE): build $(CORE_PROFILES) $(TARGET_PROFILES) @:$(call check_defined, PROFILE, target profile name) - build/builder resolve-profile $(PROFILE) + build/builder resolve-profiles $(PROFILE) clean: rm -rf build diff --git a/scripts/builder.py b/scripts/builder.py new file mode 100644 index 0000000..f4e1472 --- /dev/null +++ b/scripts/builder.py @@ -0,0 +1,607 @@ +import io +import os +import re +import sys +import glob +import json +import time +import shutil +import logging +import argparse +import textwrap +import subprocess +import urllib.error + +from collections import defaultdict +from datetime import datetime, timedelta +from distutils.version import StrictVersion +from urllib.request import Request, urlopen + +import yaml +import boto3 +import pyhocon + + +class IdentityBrokerClient: + + _DEFAULT_ENDPOINT = "https://aws-access.crute.us/api/account" + _DEFAULT_ACCOUNT = "alpine-amis-user" + + def __init__(self, endpoint=None, key=None, account=None): + self.endpoint = endpoint or self._DEFAULT_ENDPOINT + self.account = account or self._DEFAULT_ACCOUNT + self.key = key + self._logger = logging.getLogger(__class__.__name__) + + if override_endpoint := os.environ.get("IDENTITY_BROKER_ENDPOINT"): + self.endpoint = override_endpoint + + if not self.key: + self.key = os.environ.get("IDENTITY_BROKER_API_KEY") + + if not self.key: + raise Exception("No identity broker key found") + + def _get(self, path): + while True: # to handle rate limits + try: + res = urlopen(Request(path, headers={"X-API-Key": self.key})) + except urllib.error.HTTPError as ex: + if ex.headers.get("Location") == "/logout": + raise Exception("Identity broker token is expired") + + if res.status == 429: + self._logger.warning( + "Rate-limited by identity broker, sleeping 30 seconds") + time.sleep(30) + continue + + if res.status not in {200, 429}: + raise Exception(res.reason) + + return json.load(res) + + def get_credentials_url(self): + for account in self._get(self.endpoint): + if account["short_name"] == self.account: + return account["credentials_url"] + + raise Exception("No account found") + + def get_regions(self): + out = {} + + for region in self._get(self.get_credentials_url()): + if region["enabled"]: + out[region["name"]] = region["credentials_url"] + + return out + + def get_credentials(self, region): + return self._get(self.get_regions()[region]) + + def _boto3_session_from_creds(self, creds, region): + return boto3.session.Session( + aws_access_key_id=creds["access_key"], + aws_secret_access_key=creds["secret_key"], + aws_session_token=creds["session_token"], + region_name=region) + + def boto3_session_for_region(self, region): + return self._boto3_session_from_creds( + self.get_credentials(region), region) + + def iter_regions(self): + for region, cred_url in self.get_regions().items(): + yield self._boto3_session_from_creds(self._get(cred_url), region) + + +class ReleaseReadmeUpdater: + + SECTION_TPL = textwrap.dedent(""" + ### Alpine Linux {release} ({date}) +
click to show/hide

+ + {rows} + +

+ """) + + AMI_TPL = ( + " [{id}](https://{r}.console.aws.amazon.com/ec2/home" + "#Images:visibility=public-images;imageId={id}) " + "([launch](https://{r}.console.aws.amazon.com/ec2/home" + "#launchAmi={id})) |" + ) + + def __init__(self, repo_root, profile, archs=None): + self.repo_root = repo_root + self.profile = profile + self.archs = archs or ["x86_64", "aarch64"] + + @staticmethod + def extract_ver(x): + return StrictVersion("0.0" if x["release"] == "edge" else x["release"]) + + def get_sorted_releases(self, release_data): + sections = defaultdict(lambda: { + "release": "", + "built": {}, + "name": {}, + "ami": defaultdict(dict) + }) + + for build, releases in release_data.items(): + for release, amis in releases.items(): + for name, info in amis.items(): + arch = info["arch"] + built = info["build_time"] + ver = sections[info["version"]] + + if arch not in ver["built"] or ver["built"][arch] < built: + ver["release"] = release + ver["name"][arch] = name + ver["built"][arch] = built + + for region, ami in info["artifacts"].items(): + ver["ami"][region][arch] = ami + + return sorted(sections.values(), key=self.extract_ver, reverse=True) + + def make_ami_list(self, sorted_releases): + ami_list = "## AMIs\n" + + for info in sorted_releases: + rows = ["| Region |", "| ------ |"] + + for arch in self.archs: + if arch in info["name"]: + rows[0] += f" {info['name'][arch]} |" + rows[1] += " --- |" + + for region, amis in info["ami"].items(): + row = f"| {region} |" + for arch in self.archs: + if arch in amis: + row += self.AMI_TPL.format(r=region, id=amis[arch]) + rows.append(row) + + ami_list += self.SECTION_TPL.format( + release=info["release"].capitalize(), + date=datetime.utcfromtimestamp( + max(info["built"].values())).date(), + rows="\n".join(rows)) + + return ami_list + + def update_markdown(self): + release_dir = os.path.join(self.repo_root, "releases") + profile_file = os.path.join(release_dir, f"{self.profile}.yaml") + + with open(profile_file, "r") as data: + sorted_releases = self.get_sorted_releases(yaml.safe_load(data)) + + readme_md = os.path.join(release_dir, "README.md") + + with open(readme_md, "r") as file: + readme = file.read() + + with open(readme_md, "w") as file: + file.write( + re.sub("## AMIs.*\Z", self.make_ami_list(sorted_releases), + readme, flags=re.S)) + + +class GenReleaseReadme: + """Update release README + """ + + command_name = "gen-release-readme" + + @staticmethod + def add_args(parser): + parser.add_argument("profile", help="name of profile to update") + + def run(self, args, root): + ReleaseReadmeUpdater(root, args.profile).update_markdown() + + +class MakeAMIs: + """Build Packer JSON variable files from HOCON build profiles + """ + + command_name = "make-amis" + + @staticmethod + def add_args(parser): + parser.add_argument("profile", help="name of profile to build") + parser.add_argument("builds", nargs="*", + help="name of builds within a profile to build") + + def run(self, args, root): + os.chdir(os.path.join(root, "build")) + + builds = args.builds or os.listdir( + os.path.join("profile", args.profile)) + + for build in builds: + print(f"\n*** Building {args.profile}/{build} ***\n\n") + + build_dir = os.path.join("profile", args.profile, build) + if not os.path.exists(build_dir): + print(f"Build dir '{build_dir}' does not exist") + break + + out = io.StringIO() + + res = subprocess.Popen([ + os.environ.get("PACKER", "packer"), + "build", + f"-var-file={build_dir}/vars.json", + "packer.json" + ], stdout=subprocess.PIPE, encoding="utf-8") + + while res.poll() is None: + text = res.stdout.readline() + out.write(text) + print(text, end="") + + if res.returncode == 0: + subprocess.run([os.path.join(root, "build", "builder"), + "update-releases", args.profile, build]) + else: + if "is used by an existing AMI" in out.getvalue(): + continue + else: + sys.exit(res.returncode) + + print("\n=== DONE ===\n") + + +class PruneAMIs: + """Prune AMIs from AWS + """ + + command_name = "prune-amis" + + @staticmethod + def add_args(parser): + LEVEL_HELP = textwrap.dedent("""\ + revision - keep only the latest revision per release + release - keep only the latest release per version + version - keep only the versions that aren't end-of-life + """) + + parser.add_argument( + "level", choices=["revision", "release", "version"], + help=LEVEL_HELP) + parser.add_argument("profile", help="profile to prune") + parser.add_argument( + "build", nargs="?", help="build within profile to prune") + + @staticmethod + def delete_image(ec2, image): + ec2.deregister_image(ImageId=image["ImageId"]) + + for blockdev in image["BlockDeviceMappings"]: + if "Ebs" not in blockdev: + continue + + ec2.delete_snapshot(SnapshotId=blockdev["Ebs"]["SnapshotId"]) + + def run(self, args, root): + now = datetime.utcnow() + release_yaml = os.path.join(root, "releases", f"{args.profile}.yaml") + + with open(release_yaml, "r") as data: + before = yaml.safe_load(data) + + known = defaultdict(list) + prune = defaultdict(list) + after = defaultdict(lambda: defaultdict(dict)) + + # for all builds in the profile... + for build_name, releases in before.items(): + # this is not the build that was specified + if args.build is not None and args.build != build_name: + print(f"< skipping {args.profile}/{build_name}") + # ensure its release data remains intact + after[build_name] = before[build_name] + continue + else: + print(f"> PRUNING {args.profile}/{build_name} for {args.level}") + + criteria = {} + + # scan releases for pruning criteria + for release, amis in releases.items(): + for ami_name, info in amis.items(): + version = info["version"] + built = info["build_time"] + + if eol := info.get("end_of_life"): + eol = datetime.fromisoformat(info["end_of_life"]) + + for region, ami_id in info["artifacts"].items(): + known[region].append(ami_id) + + if args.level == "revision": + # find build timestamp of most recent revision, per release + if release not in criteria or built > criteria[release]: + criteria[release] = built + elif args.level == "release": + # find build timestamp of most recent revision, per version + if version not in criteria or built > criteria[version]: + criteria[version] = built + elif args.level == "version": + # find latest EOL date, per version + if ( + version not in criteria or + (not criteria[version]) or + (eol and eol > criteria[version]) + ): + criteria[version] = eol + + # rescan again to determine what doesn't make the cut + for release, amis in releases.items(): + for ami_name, info in amis.items(): + version = info["version"] + + if eol := info.get("end_of_life"): + eol = datetime.fromisoformat(info["end_of_life"]) + + if args.level == "revision": + if info["build_time"] < criteria[release]: + for region, ami_id in info["artifacts"].items(): + prune[region].append(ami_id) + elif args.level == "release": + if info["build_time"] < criteria[version]: + for region, ami_id in info["artifacts"].items(): + prune[region].append(ami_id) + elif args.level == "version": + if criteria[version] and ( + (version != "edge" and criteria[version] < now) or + (version == "edge" and ((not eol) or (eol < now))) + ): + for region, ami_id in info["artifacts"].items(): + prune[region].append(ami_id) + else: + after[build_name][release][ami_name] = info + + for session in IdentityBrokerClient().iter_regions(): + region = session.region_name + + print(f"* scanning: {region} ...") + + ec2 = session.client("ec2") + for image in ec2.describe_images(Owners=["self"])["Images"]: + image_name, image_id = image["Name"], image["ImageId"] + + if region in prune and image["ImageId"] in prune[region]: + print(f"REMOVE: {image_name} = {image_id}") + self.delete_image(image) + elif region in known and image["ImageId"] in known[region]: + print(f"KEEP: {image_name} = {image_id}") + else: + print(f"UNKNOWN: {image_name} = {image_id}") + + # update releases/.yaml + with open(release_yaml, "w") as data: + yaml.dump(after, data, sort_keys=False) + + +class ConfigBuilder: + + now = datetime.utcnow() + tomorrow = now + timedelta(days=1) + + @staticmethod + def unquote(x): + return x.strip('"') + + @staticmethod + def force_iso_date(input): + return datetime.fromisoformat(input).isoformat(timespec="seconds") + + @classmethod + def resolve_tomorrow(cls, input): + return cls.tomorrow.isoformat(timespec="seconds") + + @classmethod + def resolve_now(cls, input): + return cls.now.strftime("%Y%m%d%H%M%S") + + @classmethod + def fold_comma(cls, input): + return ",".join([cls.unquote(k) for k in input.keys()]) + + @classmethod + def fold_space(cls, input): + return " ".join([cls.unquote(k) for k in input.keys()]) + + @classmethod + def fold_repos(cls, input): + return "\n".join( + f"@{v} {cls.unquote(k)}" if isinstance(v, str) else cls.unquote(k) + for k, v in input.items()) + + @staticmethod + def fold_packages(input): + return " ".join( + f"{k}@{v}" if isinstance(v, str) else k + for k, v in input.items()) + + @staticmethod + def fold_services(input): + return " ".join( + "{}={}".format(k, ",".join(v.keys())) + for k, v in input.items()) + + def __init__(self, config_path, out_dir): + self.config_path = config_path + self.out_dir = out_dir + + self._keys_to_transform = { + "ami_access" : self.fold_comma, + "ami_regions" : self.fold_comma, + "kernel_modules" : self.fold_comma, + "kernel_options" : self.fold_space, + "repos" : self.fold_repos, + "pkgs" : self.fold_packages, + "svcs" : self.fold_services, + "revision" : self.resolve_now, + "end_of_life" : lambda x: \ + self.force_iso_date(self.resolve_tomorrow(x)), + } + + def build_all(self): + for file in glob.glob(os.path.join(self.config_path, "*.conf")): + profile = os.path.splitext(os.path.split(file)[-1])[0] + self.build_profile(profile) + + def build_profile(self, profile): + build_config = pyhocon.ConfigFactory.parse_file( + os.path.join(self.config_path, f"{profile}.conf")) + + for build, cfg in build_config["BUILDS"].items(): + build_dir = os.path.join(self.out_dir, profile, build) + + # Always start fresh + shutil.rmtree(build_dir, ignore_errors=True) + os.makedirs(build_dir) + + cfg["profile"] = profile + cfg["profile_build"] = build + + # Order of operations is important here + for k, v in cfg.items(): + transform = self._keys_to_transform.get(k) + if transform: + cfg[k] = transform(v) + + if isinstance(v, str) and "{var." in v: + cfg[k] = v.format(var=cfg) + + with open(os.path.join(build_dir, "vars.json"), "w") as out: + json.dump(cfg, out, indent=4, separators=(",", ": ")) + + +class ResolveProfiles: + """Build Packer JSON variable files from HOCON build profiles + """ + + command_name = "resolve-profiles" + + @staticmethod + def add_args(parser): + parser.add_argument( + "profile", help="name of profile to build", nargs="*") + + def run(self, args, root): + builder = ConfigBuilder( + os.path.join(root, "profiles"), + os.path.join(root, "build", "profile")) + + if args.profile: + for profile in args.profile: + builder.build_profile(profile) + else: + builder.build_all() + + +class UpdateReleases: + """Update release YAML + """ + + command_name = "update-releases" + + @staticmethod + def add_args(parser): + parser.add_argument("profile", help="name of profile to update") + parser.add_argument("build", help="name of build to update") + + @staticmethod + def parse_ids(ids): + parsed = re.split(":|,", ids) + return dict(zip(parsed[0::2], parsed[1::2])) + + def run(self, args, root): + release_dir = os.path.join(root, "releases") + if not os.path.exists(release_dir): + os.makedirs(release_dir) + + release_yaml = os.path.join(release_dir, f"{args.profile}.yaml") + releases = {} + if os.path.exists(release_yaml): + with open(release_yaml, "r") as data: + releases = yaml.safe_load(data) + + manifest_json = os.path.join( + root, "build", "profile", args.profile, args.build, + "manifest.json") + with open(manifest_json, "r") as data: + manifest = json.load(data) + + data = manifest["builds"][0]["custom_data"] + release = data["release"] + + if args.build not in releases: + releases[args.build] = {} + + if release not in releases[args.build]: + releases[args.build][release] = {} + + releases[args.build][release][data["ami_name"]] = { + "description": data["ami_desc"], + "profile": args.profile, + "profile_build": args.build, + "version": data["version"], + "release": release, + "arch": data["arch"], + "revision": data["revision"], + "end_of_life": data["end_of_life"], + "build_time": manifest["builds"][0]["build_time"], + "artifacts": self.parse_ids(manifest["builds"][0]["artifact_id"]), + } + + with open(release_yaml, "w") as data: + yaml.dump(releases, data, sort_keys=False) + + +def find_repo_root(): + path = os.getcwd() + + while ".git" not in set(os.listdir(path)) and path != "/": + path = os.path.dirname(path) + + if path == "/": + raise Exception("No repo found, stopping at /") + + return path + + +def main(): + dispatch = {} + + parser = argparse.ArgumentParser() + subs = parser.add_subparsers(dest="command_name", required=True) + + for command in sys.modules[__name__].__dict__.values(): + if not hasattr(command, "command_name"): + continue + + dispatch[command.command_name] = command() + + doc = getattr(command, "__doc__", "") + subparser = subs.add_parser( + command.command_name, help=doc, description=doc) + + if add_args := getattr(command, "add_args", None): + command.add_args(subparser) + + args = parser.parse_args() + dispatch[args.command_name].run(args, find_repo_root()) + + +if __name__ == "__main__": + main() diff --git a/scripts/gen-release-readme.py.in b/scripts/gen-release-readme.py.in deleted file mode 100644 index c2af953..0000000 --- a/scripts/gen-release-readme.py.in +++ /dev/null @@ -1,131 +0,0 @@ -@PYTHON@ -# vim: ts=4 et: - -import os -import re -import argparse -import textwrap -from datetime import datetime -from collections import defaultdict -from distutils.version import StrictVersion - -import yaml - - -def find_repo_root(): - path = os.getcwd() - - while ".git" not in set(os.listdir(path)) and path != "/": - path = os.path.dirname(path) - - if path == "/": - raise Exception("No repo found, stopping at /") - - return path - - -class ReleaseReadmeUpdater: - - SECTION_TPL = textwrap.dedent(""" - ### Alpine Linux {release} ({date}) -
click to show/hide

- - {rows} - -

- """) - - AMI_TPL = ( - " [{id}](https://{r}.console.aws.amazon.com/ec2/home" - "#Images:visibility=public-images;imageId={id}) " - "([launch](https://{r}.console.aws.amazon.com/ec2/home" - "#launchAmi={id})) |" - ) - - def __init__(self, profile, archs=None): - self.profile = profile - self.archs = archs or ["x86_64", "aarch64"] - - def get_sorted_releases(self, release_data): - sections = defaultdict(lambda: { - "release": "", - "built": {}, - "name": {}, - "ami": defaultdict(dict) - }) - - for build, releases in release_data.items(): - for release, amis in releases.items(): - for name, info in amis.items(): - arch = info["arch"] - built = info["build_time"] - ver = sections[info["version"]] - - if arch not in ver["built"] or ver["built"][arch] < built: - ver["release"] = release - ver["name"][arch] = name - ver["built"][arch] = built - - for region, ami in info["artifacts"].items(): - ver["ami"][region][arch] = ami - - extract_ver = lambda x: StrictVersion( - "0.0" if x["release"] == "edge" else x["release"]) - - return sorted(sections.values(), key=extract_ver, reverse=True) - - def make_ami_list(self, sorted_releases): - ami_list = "## AMIs\n" - - for info in sorted_releases: - rows = ["| Region |", "| ------ |"] - - for arch in self.archs: - if arch in info["name"]: - rows[0] += f" {info['name'][arch]} |" - rows[1] += " --- |" - - for region, amis in info["ami"].items(): - row = f"| {region} |" - for arch in self.archs: - if arch in amis: - row += self.AMI_TPL.format(r=region, id=amis[arch]) - rows.append(row) - - ami_list += self.SECTION_TPL.format( - release=info["release"].capitalize(), - date=datetime.utcfromtimestamp( - max(info["built"].values())).date(), - rows="\n".join(rows)) - - return ami_list - - def update_markdown(self): - release_dir = os.path.join(find_repo_root(), "releases") - profile_file = os.path.join(release_dir, f"{self.profile}.yaml") - - with open(profile_file, "r") as data: - sorted_releases = self.get_sorted_releases(yaml.safe_load(data)) - - readme_md = os.path.join(release_dir, "README.md") - - with open(readme_md, "r") as file: - readme = file.read() - - with open(readme_md, "w") as file: - file.write( - re.sub("## AMIs.*\Z", self.make_ami_list(sorted_releases), - readme, flags=re.S)) - - -def main(): - parser = argparse.ArgumentParser(description="Update release README") - parser.add_argument("profile", help="name of profile to update") - args = parser.parse_args() - - ReleaseReadmeUpdater(args.profile).update_markdown() - - - -if __name__ == "__main__": - main() diff --git a/scripts/make-amis.py.in b/scripts/make-amis.py.in deleted file mode 100644 index c7f9f98..0000000 --- a/scripts/make-amis.py.in +++ /dev/null @@ -1,68 +0,0 @@ -@PYTHON@ -# vim: set ts=4 et: - -import os -import io -import sys -import argparse -import subprocess - - -def find_repo_root(): - path = os.getcwd() - - while ".git" not in set(os.listdir(path)) and path != "/": - path = os.path.dirname(path) - - if path == "/": - raise Exception("No repo found, stopping at /") - - return path - - -def main(args): - parser = argparse.ArgumentParser(description="Build Packer JSON variable " - "files from HOCON build profiles") - parser.add_argument("profile", help="name of profile to build") - parser.add_argument("builds", nargs="*", - help="name of builds within a profile to build") - args = parser.parse_args() - - os.chdir(os.path.join(find_repo_root(), "build")) - - builds = args.builds or os.listdir(os.path.join("profile", args.profile)) - for build in builds: - print(f"\n*** Building {args.profile}/{build} ***\n\n") - - build_dir = os.path.join("profile", args.profile, build) - if not os.path.exists(build_dir): - print(f"Build dir '{build_dir}' does not exist") - break - - out = io.StringIO() - - res = subprocess.Popen([ - os.environ.get("PACKER", "packer"), - "build", - f"-var-file={build_dir}/vars.json", - "packer.json" - ], stdout=subprocess.PIPE, encoding="utf-8") - - while res.poll() is None: - text = res.stdout.readline() - out.write(text) - print(text, end="") - - if res.returncode == 0: - subprocess.run(["./update-release.py", args.profile, build]) - else: - if "is used by an existing AMI" in out.getvalue(): - continue - else: - sys.exit(res.returncode) - - print("\n=== DONE ===\n") - - -if __name__ == "__main__": - main(sys.argv) diff --git a/scripts/prune-amis.py.in b/scripts/prune-amis.py.in deleted file mode 100644 index 06ef567..0000000 --- a/scripts/prune-amis.py.in +++ /dev/null @@ -1,168 +0,0 @@ -#@PYTHON@ -# vim: ts=4 et: - -import os -import sys -import argparse -from datetime import datetime - -import yaml -import boto3 -from botocore.exceptions import ClientError - -LEVEL_HELP = """\ -revision - keep only the latest revision per release -release - keep only the latest release per version -version - keep only the versions that aren't end-of-life -""" - - -def find_repo_root(): - path = os.getcwd() - - while ".git" not in set(os.listdir(path)) and path != "/": - path = os.path.dirname(path) - - if path == "/": - raise Exception("No repo found, stopping at /") - - return path - - -def main(args): - parser = argparse.ArgumentParser( - description="Prune AMIs from AWS", - formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument( - "level", choices=["revision", "release", "version"], help=LEVEL_HELP) - parser.add_argument("profile", help="profile to prune") - parser.add_argument( - "build", nargs="?", help="build within profile to prune") - args = parser.parse_args() - - now = datetime.utcnow() - - release_yaml = os.path.join( - find_repo_root() "releases", f"{args.profile}.yaml") - - with open(release_yaml, "r") as data: - before = yaml.safe_load(data) - - known = {} - prune = {} - after = {} - - # for all builds in the profile... - for build_name, releases in before.items(): - - # this is not the build that was specified - if args.build is not None and args.build != build_name: - print(f"< skipping {args.profile}/{build_name}") - # ensure its release data remains intact - after[build_name] = before[build_name] - continue - else: - print(f"> PRUNING {args.profile}/{build_name} for {args.level}") - - criteria = {} - - # scan releases for pruning criteria - for release, amis in releases.items(): - for ami_name, info in amis.items(): - version = info["version"] - built = info["build_time"] - - if info["end_of_life"]: - eol = datetime.fromisoformat(info["end_of_life"]) - else: - eol = None - - for region, ami_id in info["artifacts"].items(): - if region not in known: - known[region] = [] - known[region].append(ami_id) - - if args.level == "revision": - # find build timestamp of most recent revision, per release - if release not in criteria or built > criteria[release]: - criteria[release] = built - elif args.level == "release": - # find build timestamp of most recent revision, per version - if version not in criteria or built > criteria[version]: - criteria[version] = built - elif args.level == "version": - # find latest EOL date, per version - if (version not in criteria or not criteria[version]) or ( - eol and eol > criteria[version]): - criteria[version] = eol - - # rescan again to determine what doesn't make the cut - for release, amis in releases.items(): - for ami_name, info in amis.items(): - version = info["version"] - built = info["build_time"] - - if info["end_of_life"]: - eol = datetime.fromisoformat(info["end_of_life"]) - else: - eol = None - - if ((args.level == "revision" and built < criteria[release]) or - (args.level == "release" and built < criteria[version]) or - (args.level == "version" and criteria[version] and ( - (version != "edge" and criteria[version] < now) or - (version == "edge" and ((not eol) or (eol < now))) - ))): - for region, ami_id in info["artifacts"].items(): - if region not in prune: - prune[region] = [] - - prune[region].append(ami_id) - else: - if build_name not in after: - after[build_name] = {} - - if release not in after[build_name]: - after[build_name][release] = {} - - after[build_name][release][ami_name] = info - - # scan all regions for AMIs - AWS = boto3.session.Session() - for region in AWS.get_available_regions("ec2"): - print(f"* scanning: {region} ...") - EC2 = AWS.client("ec2", region_name=region) - - try: - for image in EC2.describe_images(Owners=["self"])["Images"]: - - action = "? UNKNOWN" - if region in prune and image["ImageId"] in prune[region]: - action = "- REMOVING" - elif region in known and image["ImageId"] in known[region]: - action = "+ KEEPING" - - print(f" {action}: {image['Name']}\n = {image['ImageId']}", - end="", flush=True) - - if action[0] == "-": - EC2.deregister_image(ImageId=image["ImageId"]) - - for blockdev in image["BlockDeviceMappings"]: - if "Ebs" in blockdev: - print(", {blockdev['Ebs']['SnapshotId']}", - end="", flush=True) - if action[0] == "-": - EC2.delete_snapshot( - SnapshotId=blockdev["Ebs"]["SnapshotId"]) - print() - except ClientError as e: - print(e) - - # update releases/.yaml - with open(release_yaml, "w") as data: - yaml.dump(after, data, sort_keys=False) - - -if __name__ == "__main__": - main(sys.argv) diff --git a/scripts/resolve-profile.py.in b/scripts/resolve-profile.py.in deleted file mode 100644 index 2905423..0000000 --- a/scripts/resolve-profile.py.in +++ /dev/null @@ -1,133 +0,0 @@ -@PYTHON@ -# vim: set ts=4 et: - -import os -import sys -import json -import shutil -import argparse -from datetime import datetime, timedelta - -from pyhocon import ConfigFactory - - -# Just group together our transforms -class Transforms: - - NOW = datetime.utcnow() - TOMORROW = NOW + timedelta(days=1) - - unquote = lambda x: x.strip('"') - - @staticmethod - def force_iso_date(input): - return datetime.fromisoformat(input).isoformat(timespec="seconds") - - @classmethod - def resolve_tomorrow(cls, input): - return cls.TOMORROW.isoformat(timespec="seconds") - - @classmethod - def resolve_now(cls, input): - return cls.NOW.strftime("%Y%m%d%H%M%S") - - @classmethod - def fold_comma(cls, input): - return ",".join([cls.unquote(k) for k in input.keys()]) - - @classmethod - def fold_space(cls, input): - return " ".join([cls.unquote(k) for k in input.keys()]) - - @classmethod - def fold_repos(cls, input): - return "\n".join( - f"@{v} {cls.unquote(k)}" if isinstance(v, str) else cls.unquote(k) - for k, v in input.items()) - - @staticmethod - def fold_packages(input): - return " ".join( - f"{k}@{v}" if isinstance(v, str) else k - for k, v in input.items()) - - @staticmethod - def fold_services(input): - return " ".join( - "{}={}".format(k, ",".join(v.keys())) - for k, v in input.items()) - - -class ConfigBuilder: - - _CFG_TRANSFORMS = { - "ami_access" : Transforms.fold_comma, - "ami_regions" : Transforms.fold_comma, - "kernel_modules" : Transforms.fold_comma, - "kernel_options" : Transforms.fold_space, - "repos" : Transforms.fold_repos, - "pkgs" : Transforms.fold_packages, - "svcs" : Transforms.fold_services, - "revision" : Transforms.resolve_now, - "end_of_life" : lambda x: \ - Transforms.force_iso_date(Transforms.resolve_tomorrow(x)), - } - - def __init__(self, config_path, out_dir): - self.config_path = config_path - self.out_dir = out_dir - - def build(self, profile): - build_config = ConfigFactory.parse_file(self.config_path) - - for build, cfg in build_config["BUILDS"].items(): - build_dir = os.path.join(self.out_dir, build) - - # Always start fresh - shutil.rmtree(build_dir, ignore_errors=True) - os.makedirs(build_dir) - - cfg["profile"] = profile - cfg["profile_build"] = build - - # Order of operations is important here - for k, v in cfg.items(): - transform = self._CFG_TRANSFORMS.get(k) - if transform: - cfg[k] = transform(v) - - if isinstance(v, str) and "{var." in v: - cfg[k] = v.format(var=cfg) - - with open(os.path.join(build_dir, "vars.json"), "w") as out: - json.dump(cfg, out, indent=4, separators=(",", ": ")) - - -def find_repo_root(): - path = os.getcwd() - - while ".git" not in set(os.listdir(path)) and path != "/": - path = os.path.dirname(path) - - if path == "/": - raise Exception("No repo found, stopping at /") - - return path - - -def main(args): - parser = argparse.ArgumentParser(description="Build Packer JSON variable " - "files from HOCON build profiles") - parser.add_argument("profile", help="name of profile to build") - args = parser.parse_args() - - root = find_repo_root() - - ConfigBuilder( - os.path.join(root, "profiles", f"{args.profile}.conf"), - os.path.join(root, "build", "profile", args.profile) - ).build(args.profile) - - -if __name__ == "__main__": - main(sys.argv) diff --git a/scripts/update-release.py.in b/scripts/update-release.py.in deleted file mode 100644 index b8f4d00..0000000 --- a/scripts/update-release.py.in +++ /dev/null @@ -1,80 +0,0 @@ -@PYTHON@ -# vim: set ts=4 et: - -import os -import re -import sys -import json -import argparse - -import yaml - - -def find_repo_root(): - path = os.getcwd() - - while ".git" not in set(os.listdir(path)) and path != "/": - path = os.path.dirname(path) - - if path == "/": - raise Exception("No repo found, stopping at /") - - return path - - -def parse_artifact_ids(ids): - parsed = re.split(":|,", ids) - return dict(zip(parsed[0::2], parsed[1::2])) - - -def main(args): - parser = argparse.ArgumentParser(description="Update release YAML") - parser.add_argument("profile", help="name of profile to update") - parser.add_argument("build", help="name of build to update") - args = parser.parse_args() - - root = find_repo_root() - - release_dir = os.path.join(root, "releases") - if not os.path.exists(release_dir): - os.makedirs(release_dir) - - release_yaml = os.path.join(release_dir, f"{args.profile}.yaml") - releases = {} - if os.path.exists(release_yaml): - with open(release_yaml, "r") as data: - releases = yaml.safe_load(data) - - manifest_json = os.path.join( - root, "build", "profile", args.profile, args.build, "manifest.json") - with open(manifest_json, "r") as data: - manifest = json.load(data) - - data = manifest["builds"][0]["custom_data"] - release = data["release"] - - if args.build not in releases: - releases[args.build] = {} - - if release not in releases[args.build]: - releases[args.build][release] = {} - - releases[args.build][release][data["ami_name"]] = { - "description": data["ami_desc"], - "profile": args.profile, - "profile_build": args.build, - "version": data["version"], - "release": release, - "arch": data["arch"], - "revision": data["revision"], - "end_of_life": data["end_of_life"], - "build_time": manifest["builds"][0]["build_time"], - "artifacts": parse_artifact_ids(manifest["builds"][0]["artifact_id"]), - } - - with open(release_yaml, "w") as data: - yaml.dump(releases, data, sort_keys=False) - - -if __name__ == "__main__": - main(sys.argv)