more changes based on review...

* Makefile - improve/fix check for required make vars
* resolve-profile.py.in
+ build a list of all regions & probe to see which ones are enabled (unknown if special subscription regions like ap-northeast-3 would show up in this list)
+ expand 'ami_regions' 'ALL' meta key to all enabled regions, 'ALL' key's value is preserved (that is, a value of None or False will disable all regions)
+ warn/remove regions in profile config that are found to be disabled.
+ improve checks for [None, False] values
This commit is contained in:
Jake Buchholz 2019-06-03 17:35:07 -07:00 committed by Mike Crute
parent d28214fc50
commit 8f563aa4f1
4 changed files with 65 additions and 25 deletions

View File

@ -12,20 +12,29 @@ LEVEL :=
PACKER := packer PACKER := packer
export PACKER export PACKER
require_var = $(if $(value $1),,$(error $1=... required))
check_defined = \
$(strip $(foreach 1,$1, \
$(call __check_defined,$1,$(strip $(value 2)))))
__check_defined = \
$(if $(value $1),, \
$(error Undefined $1$(if $2, ($2))$(if $(value @), \
required by target `$@')))
.PHONY: amis prune release-readme clean .PHONY: amis prune release-readme clean
amis: build build/packer.json build/profile/$(PROFILE) build/update-release.py amis: build build/packer.json build/profile/$(PROFILE) build/update-release.py
@:$(call require_var, PROFILE) @:$(call check_defined, PROFILE, target profile name)
build/make-amis $(PROFILE) $(BUILDS) build/make-amis $(PROFILE) $(BUILDS)
prune: build build/prune-amis.py prune: build build/prune-amis.py
@:$(call require_var, LEVEL) @:$(call check_defined, LEVEL, pruning level)
@:$(call require_var, PROFILE) @:$(call check_defined, PROFILE, target profile name)
build/prune-amis.py $(LEVEL) $(PROFILE) $(BUILD) build/prune-amis.py $(LEVEL) $(PROFILE) $(BUILD)
release-readme: build build/gen-release-readme.py release-readme: build build/gen-release-readme.py
@:$(call check_defined, PROFILE, target profile name)
@:$(call require_var, PROFILE) @:$(call require_var, PROFILE)
build/gen-release-readme.py $(PROFILE) build/gen-release-readme.py $(PROFILE)
@ -39,7 +48,7 @@ build/packer.json: build packer.conf
build/.py3/bin/pyhocon -i packer.conf -f json > build/packer.json build/.py3/bin/pyhocon -i packer.conf -f json > build/packer.json
build/profile/$(PROFILE): build build/resolve-profile.py $(CORE_PROFILES) $(TARGET_PROFILES) build/profile/$(PROFILE): build build/resolve-profile.py $(CORE_PROFILES) $(TARGET_PROFILES)
@:$(call require_var, PROFILE) @:$(call check_defined, PROFILE, target profile name)
build/resolve-profile.py $(PROFILE) build/resolve-profile.py $(PROFILE)
%.py: %.py.in build %.py: %.py.in build

View File

@ -18,24 +18,7 @@ alpine {
all = true # these AMIs are publicly available all = true # these AMIs are publicly available
} }
ami_regions { ami_regions {
#ap-east-1 = true # needs to be enabled first ALL = true
ap-northeast-1 = true
ap-northeast-2 = true
#ap-northeast-3 = false # available by subscription only
ap-southeast-1 = true
ap-southeast-2 = true
ap-south-1 = true
ca-central-1 = true
eu-central-1 = true
eu-north-1 = true
eu-west-1 = true
eu-west-2 = true
eu-west-3 = true
sa-east-1 = true
us-east-1 = true
us-east-2 = true
us-west-1 = true
us-west-2 = true
} }
} }

View File

@ -15,6 +15,10 @@ test {
ami_desc_prefix = "Alpine Test " ami_desc_prefix = "Alpine Test "
build_region = "us-west-2" build_region = "us-west-2"
build_subnet = "subnet-033a30d7b5220d177" build_subnet = "subnet-033a30d7b5220d177"
ami_regions {
us-west-1 = true
ap-east-1 = true
}
} }
# Build definitions # Build definitions

View File

@ -5,6 +5,8 @@ import json
import os import os
import shutil import shutil
import sys import sys
import boto3
from botocore.exceptions import ClientError
from datetime import datetime, timedelta from datetime import datetime, timedelta
from pyhocon import ConfigFactory from pyhocon import ConfigFactory
@ -47,13 +49,39 @@ def fold(fdict, ffmt):
fkey = fkey.strip('"') # complex keys may be in quotes fkey = fkey.strip('"') # complex keys may be in quotes
if fval is True: if fval is True:
folded += ffmt[0] + fkey folded += ffmt[0] + fkey
elif not (fval is None or fval is False): elif fval not in [None, False]:
folded += ffmt.format(fkey, fval) folded += ffmt.format(fkey, fval)
return folded[1:] return folded[1:]
# list of AWS regions, and whether they're enabled
all_regions = {}
AWS = boto3.session.Session()
sys.stderr.write("\n>>> Determining region availability...")
sys.stderr.flush()
for region in AWS.get_available_regions('ec2'):
ec2 = AWS.client('ec2', region_name=region)
try:
ec2.describe_regions()
except ClientError as e:
if e.response['Error']['Code'] == 'AuthFailure':
sys.stderr.write('-')
sys.stderr.flush()
all_regions[region] = False
continue
elif e.response['Error']['Code'] == 'UnauthorizedOperation':
# have access to the region, but not to ec2:DescribeRegions
pass
else:
raise
sys.stderr.write('+')
sys.stderr.flush()
all_regions[region] = True
sys.stderr.write("\n")
# parse/resolve HOCON profile's builds' config # parse/resolve HOCON profile's builds' config
for build, cfg in BUILDS.items(): for build, cfg in BUILDS.items():
sys.stderr.write(f">>> Resolving configuration for '{build}'\n")
build_dir = os.path.join(PROFILE_DIR, build) build_dir = os.path.join(PROFILE_DIR, build)
# make a fresh profile build directory # make a fresh profile build directory
@ -75,6 +103,22 @@ for build, cfg in BUILDS.items():
if cfg['revision'] == '@NOW@': if cfg['revision'] == '@NOW@':
cfg['revision'] = NOW.strftime('%Y%m%d%H%M%S') cfg['revision'] = NOW.strftime('%Y%m%d%H%M%S')
# 'ALL' region expansion (or retraction)
if 'ALL' in cfg['ami_regions']:
all_val = cfg['ami_regions']['ALL']
if all_val not in [None, False]:
cfg['ami_regions'] = all_regions
else:
cfg['ami_regions'] = {}
else:
# warn/remove disabled regions
for region, enabled in all_regions.items():
if enabled is not False or region not in cfg['ami_regions']:
continue
if cfg['ami_regions'][region] not in [None, False]:
sys.stderr.write(f"*** WARNING: skipping disabled region {region}\n")
cfg['ami_regions'][region] = False
# fold dict vars to scalars # fold dict vars to scalars
for foldkey, foldfmt in FOLD_DICTS.items(): for foldkey, foldfmt in FOLD_DICTS.items():
cfg[foldkey] = fold(cfg[foldkey], foldfmt) cfg[foldkey] = fold(cfg[foldkey], foldfmt)
@ -85,7 +129,7 @@ for build, cfg in BUILDS.items():
if lvl is True: if lvl is True:
# service in default runlevel # service in default runlevel
lvls['default'].append(svc) lvls['default'].append(svc)
elif not (lvl is None or lvl is False): elif lvl not in [None, False]:
# service in specified runlevel (skip svc when false/null) # service in specified runlevel (skip svc when false/null)
if lvl not in lvls.keys(): if lvl not in lvls.keys():
lvls[lvl] = [] lvls[lvl] = []