From 8f563aa4f1b33651a87399e73bfcf82fef858605 Mon Sep 17 00:00:00 2001 From: Jake Buchholz Date: Mon, 3 Jun 2019 17:35:07 -0700 Subject: [PATCH] more changes based on review... * Makefile - improve/fix check for required make vars * resolve-profile.py.in + build a list of all regions & probe to see which ones are enabled (unknown if special subscription regions like ap-northeast-3 would show up in this list) + expand 'ami_regions' 'ALL' meta key to all enabled regions, 'ALL' key's value is preserved (that is, a value of None or False will disable all regions) + warn/remove regions in profile config that are found to be disabled. + improve checks for [None, False] values --- Makefile | 19 ++++++++++---- profiles/alpine.conf | 19 +------------- profiles/test.conf | 4 +++ scripts/resolve-profile.py.in | 48 +++++++++++++++++++++++++++++++++-- 4 files changed, 65 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 8247c9b..851ed7e 100644 --- a/Makefile +++ b/Makefile @@ -12,20 +12,29 @@ LEVEL := PACKER := packer export PACKER -require_var = $(if $(value $1),,$(error $1=... required)) + +check_defined = \ + $(strip $(foreach 1,$1, \ + $(call __check_defined,$1,$(strip $(value 2))))) +__check_defined = \ + $(if $(value $1),, \ + $(error Undefined $1$(if $2, ($2))$(if $(value @), \ + required by target `$@'))) + .PHONY: amis prune release-readme clean amis: build build/packer.json build/profile/$(PROFILE) build/update-release.py - @:$(call require_var, PROFILE) + @:$(call check_defined, PROFILE, target profile name) build/make-amis $(PROFILE) $(BUILDS) prune: build build/prune-amis.py - @:$(call require_var, LEVEL) - @:$(call require_var, PROFILE) + @:$(call check_defined, LEVEL, pruning level) + @:$(call check_defined, PROFILE, target profile name) build/prune-amis.py $(LEVEL) $(PROFILE) $(BUILD) release-readme: build build/gen-release-readme.py + @:$(call check_defined, PROFILE, target profile name) @:$(call require_var, PROFILE) build/gen-release-readme.py $(PROFILE) @@ -39,7 +48,7 @@ build/packer.json: build packer.conf build/.py3/bin/pyhocon -i packer.conf -f json > build/packer.json build/profile/$(PROFILE): build build/resolve-profile.py $(CORE_PROFILES) $(TARGET_PROFILES) - @:$(call require_var, PROFILE) + @:$(call check_defined, PROFILE, target profile name) build/resolve-profile.py $(PROFILE) %.py: %.py.in build diff --git a/profiles/alpine.conf b/profiles/alpine.conf index 3727753..b5ada55 100644 --- a/profiles/alpine.conf +++ b/profiles/alpine.conf @@ -18,24 +18,7 @@ alpine { all = true # these AMIs are publicly available } ami_regions { - #ap-east-1 = true # needs to be enabled first - ap-northeast-1 = true - ap-northeast-2 = true - #ap-northeast-3 = false # available by subscription only - ap-southeast-1 = true - ap-southeast-2 = true - ap-south-1 = true - ca-central-1 = true - eu-central-1 = true - eu-north-1 = true - eu-west-1 = true - eu-west-2 = true - eu-west-3 = true - sa-east-1 = true - us-east-1 = true - us-east-2 = true - us-west-1 = true - us-west-2 = true + ALL = true } } diff --git a/profiles/test.conf b/profiles/test.conf index eb7f2a3..b4ecab8 100644 --- a/profiles/test.conf +++ b/profiles/test.conf @@ -15,6 +15,10 @@ test { ami_desc_prefix = "Alpine Test " build_region = "us-west-2" build_subnet = "subnet-033a30d7b5220d177" + ami_regions { + us-west-1 = true + ap-east-1 = true + } } # Build definitions diff --git a/scripts/resolve-profile.py.in b/scripts/resolve-profile.py.in index d784cf3..5aca4d7 100644 --- a/scripts/resolve-profile.py.in +++ b/scripts/resolve-profile.py.in @@ -5,6 +5,8 @@ import json import os import shutil import sys +import boto3 +from botocore.exceptions import ClientError from datetime import datetime, timedelta from pyhocon import ConfigFactory @@ -47,13 +49,39 @@ def fold(fdict, ffmt): fkey = fkey.strip('"') # complex keys may be in quotes if fval is True: folded += ffmt[0] + fkey - elif not (fval is None or fval is False): + elif fval not in [None, False]: folded += ffmt.format(fkey, fval) return folded[1:] +# list of AWS regions, and whether they're enabled +all_regions = {} +AWS = boto3.session.Session() +sys.stderr.write("\n>>> Determining region availability...") +sys.stderr.flush() +for region in AWS.get_available_regions('ec2'): + ec2 = AWS.client('ec2', region_name=region) + try: + ec2.describe_regions() + except ClientError as e: + if e.response['Error']['Code'] == 'AuthFailure': + sys.stderr.write('-') + sys.stderr.flush() + all_regions[region] = False + continue + elif e.response['Error']['Code'] == 'UnauthorizedOperation': + # have access to the region, but not to ec2:DescribeRegions + pass + else: + raise + sys.stderr.write('+') + sys.stderr.flush() + all_regions[region] = True +sys.stderr.write("\n") + # parse/resolve HOCON profile's builds' config for build, cfg in BUILDS.items(): + sys.stderr.write(f">>> Resolving configuration for '{build}'\n") build_dir = os.path.join(PROFILE_DIR, build) # make a fresh profile build directory @@ -75,6 +103,22 @@ for build, cfg in BUILDS.items(): if cfg['revision'] == '@NOW@': cfg['revision'] = NOW.strftime('%Y%m%d%H%M%S') + # 'ALL' region expansion (or retraction) + if 'ALL' in cfg['ami_regions']: + all_val = cfg['ami_regions']['ALL'] + if all_val not in [None, False]: + cfg['ami_regions'] = all_regions + else: + cfg['ami_regions'] = {} + else: + # warn/remove disabled regions + for region, enabled in all_regions.items(): + if enabled is not False or region not in cfg['ami_regions']: + continue + if cfg['ami_regions'][region] not in [None, False]: + sys.stderr.write(f"*** WARNING: skipping disabled region {region}\n") + cfg['ami_regions'][region] = False + # fold dict vars to scalars for foldkey, foldfmt in FOLD_DICTS.items(): cfg[foldkey] = fold(cfg[foldkey], foldfmt) @@ -85,7 +129,7 @@ for build, cfg in BUILDS.items(): if lvl is True: # service in default runlevel lvls['default'].append(svc) - elif not (lvl is None or lvl is False): + elif lvl not in [None, False]: # service in specified runlevel (skip svc when false/null) if lvl not in lvls.keys(): lvls[lvl] = []