Refactor resolve-profile script
This is paving the way for identity broker improvements for opt-in regions. The output is functionally identical between the two scripts modulo the svcs change. Hopefully this makes the transformation process a little more clear.
This commit is contained in:
Mike Crute
parent
fe362af91f
commit
1fd42af98d
|
|
@ -56,25 +56,33 @@ pkgs {
|
|||
tzdata = true
|
||||
}
|
||||
svcs {
|
||||
devfs = "sysinit"
|
||||
dmesg = "sysinit"
|
||||
hwdrivers = "sysinit"
|
||||
mdev = "sysinit"
|
||||
acpid = "boot"
|
||||
bootmisc = "boot"
|
||||
hostname = "boot"
|
||||
hwclock = "boot"
|
||||
modules = "boot"
|
||||
swap = "boot"
|
||||
sysctl = "boot"
|
||||
syslog = "boot"
|
||||
chronyd = "default"
|
||||
networking = "default"
|
||||
sshd = "default"
|
||||
tiny-ec2-bootstrap = "default"
|
||||
killprocs = "shutdown"
|
||||
mount-ro = "shutdown"
|
||||
savecache = "shutdown"
|
||||
sysinit {
|
||||
devfs = true
|
||||
dmesg = true
|
||||
hwdrivers = true
|
||||
mdev = true
|
||||
}
|
||||
boot {
|
||||
acpid = true
|
||||
bootmisc = true
|
||||
hostname = true
|
||||
hwclock = true
|
||||
modules = true
|
||||
swap = true
|
||||
sysctl = true
|
||||
syslog = true
|
||||
}
|
||||
default {
|
||||
chronyd = true
|
||||
networking = true
|
||||
sshd = true
|
||||
tiny-ec2-bootstrap = true
|
||||
}
|
||||
shutdown {
|
||||
killprocs = true
|
||||
mount-ro = true
|
||||
savecache = true
|
||||
}
|
||||
}
|
||||
kernel_modules {
|
||||
sd-mod = true
|
||||
|
|
|
|||
|
|
@ -1,156 +1,133 @@
|
|||
@PYTHON@
|
||||
# vim: set ts=4 et:
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
import json
|
||||
import shutil
|
||||
import argparse
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from pyhocon import ConfigFactory
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
sys.exit("Usage: " + os.path.basename(__file__) + " <profile>")
|
||||
|
||||
PROFILE = sys.argv[1]
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
# path to the profile config file
|
||||
PROFILE_CONF = os.path.join(SCRIPT_DIR, '..', 'profiles', PROFILE + '.conf')
|
||||
|
||||
# load the profile's build configuration
|
||||
BUILDS = ConfigFactory.parse_file(PROFILE_CONF)['BUILDS']
|
||||
|
||||
# where we store the profile's builds' config/output
|
||||
PROFILE_DIR = os.path.join(SCRIPT_DIR, 'profile', PROFILE)
|
||||
if not os.path.exists(PROFILE_DIR):
|
||||
os.makedirs(PROFILE_DIR)
|
||||
|
||||
# fold these build config keys' dict to scalar
|
||||
FOLD_DICTS = {
|
||||
'ami_access': ',{0}',
|
||||
'ami_regions': ',{0}',
|
||||
'repos': "\n@{1} {0}",
|
||||
'pkgs': ' {0}@{1}',
|
||||
'kernel_modules': ',{0}',
|
||||
'kernel_options': ' {0}'
|
||||
}
|
||||
# Just group together our transforms
|
||||
class Transforms:
|
||||
|
||||
NOW = datetime.utcnow()
|
||||
ONE_DAY = timedelta(days=1)
|
||||
TOMORROW = NOW + timedelta(days=1)
|
||||
|
||||
unquote = lambda x: x.strip('"')
|
||||
|
||||
@staticmethod
|
||||
def force_iso_date(input):
|
||||
return datetime.fromisoformat(input).isoformat(timespec="seconds")
|
||||
|
||||
@classmethod
|
||||
def resolve_tomorrow(cls, input):
|
||||
return cls.TOMORROW.isoformat(timespec="seconds")
|
||||
|
||||
@classmethod
|
||||
def resolve_now(cls, input):
|
||||
return cls.NOW.strftime("%Y%m%d%H%M%S")
|
||||
|
||||
@classmethod
|
||||
def fold_comma(cls, input):
|
||||
return ",".join([cls.unquote(k) for k in input.keys()])
|
||||
|
||||
@classmethod
|
||||
def fold_space(cls, input):
|
||||
return " ".join([cls.unquote(k) for k in input.keys()])
|
||||
|
||||
@classmethod
|
||||
def fold_repos(cls, input):
|
||||
return "\n".join(
|
||||
f"@{v} {cls.unquote(k)}" if isinstance(v, str) else cls.unquote(k)
|
||||
for k, v in input.items())
|
||||
|
||||
@staticmethod
|
||||
def fold_packages(input):
|
||||
return " ".join(
|
||||
f"{k}@{v}" if isinstance(v, str) else k
|
||||
for k, v in input.items())
|
||||
|
||||
@staticmethod
|
||||
def fold_services(input):
|
||||
return " ".join(
|
||||
"{}={}".format(k, ",".join(v.keys()))
|
||||
for k, v in input.items())
|
||||
|
||||
|
||||
# func to fold dict down to scalar
|
||||
def fold(fdict, ffmt):
|
||||
folded = ''
|
||||
for fkey, fval in fdict.items():
|
||||
fkey = fkey.strip('"') # complex keys may be in quotes
|
||||
if fval is True:
|
||||
folded += ffmt[0] + fkey
|
||||
elif fval not in [None, False]:
|
||||
folded += ffmt.format(fkey, fval)
|
||||
return folded[1:]
|
||||
class ConfigBuilder:
|
||||
|
||||
_CFG_TRANSFORMS = {
|
||||
"ami_access" : Transforms.fold_comma,
|
||||
"ami_regions" : Transforms.fold_comma,
|
||||
"kernel_modules" : Transforms.fold_comma,
|
||||
"kernel_options" : Transforms.fold_space,
|
||||
"repos" : Transforms.fold_repos,
|
||||
"pkgs" : Transforms.fold_packages,
|
||||
"svcs" : Transforms.fold_services,
|
||||
"revision" : Transforms.resolve_now,
|
||||
"end_of_life" : lambda x: \
|
||||
Transforms.force_iso_date(Transforms.resolve_tomorrow(x)),
|
||||
}
|
||||
|
||||
# list of AWS regions, and whether they're enabled
|
||||
all_regions = {}
|
||||
AWS = boto3.session.Session()
|
||||
sys.stderr.write("\n>>> Determining region availability...")
|
||||
sys.stderr.flush()
|
||||
for region in AWS.get_available_regions('ec2'):
|
||||
ec2 = AWS.client('ec2', region_name=region)
|
||||
try:
|
||||
ec2.describe_regions()
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] == 'AuthFailure':
|
||||
sys.stderr.write('-')
|
||||
sys.stderr.flush()
|
||||
all_regions[region] = False
|
||||
continue
|
||||
elif e.response['Error']['Code'] == 'UnauthorizedOperation':
|
||||
# have access to the region, but not to ec2:DescribeRegions
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
sys.stderr.write('+')
|
||||
sys.stderr.flush()
|
||||
all_regions[region] = True
|
||||
sys.stderr.write("\n")
|
||||
def __init__(self, config_path, out_dir):
|
||||
self.config_path = config_path
|
||||
self.out_dir = out_dir
|
||||
|
||||
for region, available in all_regions.items():
|
||||
if available is False:
|
||||
sys.stderr.write(f"*** WARNING: skipping disabled region {region}\n")
|
||||
def build(self, profile):
|
||||
build_config = ConfigFactory.parse_file(self.config_path)
|
||||
|
||||
print()
|
||||
for build, cfg in build_config["BUILDS"].items():
|
||||
build_dir = os.path.join(self.out_dir, build)
|
||||
|
||||
# parse/resolve HOCON profile's builds' config
|
||||
for build, cfg in BUILDS.items():
|
||||
print(f">>> Resolving configuration for '{build}'")
|
||||
build_dir = os.path.join(PROFILE_DIR, build)
|
||||
|
||||
# make a fresh profile build directory
|
||||
if os.path.exists(build_dir):
|
||||
shutil.rmtree(build_dir)
|
||||
# Always start fresh
|
||||
shutil.rmtree(build_dir, ignore_errors=True)
|
||||
os.makedirs(build_dir)
|
||||
|
||||
# populate profile build vars
|
||||
cfg['profile'] = PROFILE
|
||||
cfg['profile_build'] = build
|
||||
cfg["profile"] = profile
|
||||
cfg["profile_build"] = build
|
||||
|
||||
# mostly edge-related temporal substitutions
|
||||
if cfg['end_of_life'] == '@TOMORROW@':
|
||||
cfg['end_of_life'] = (NOW + ONE_DAY).isoformat(timespec='seconds')
|
||||
elif cfg['end_of_life'] is not None:
|
||||
# to explicitly UTC-ify end_of_life
|
||||
cfg['end_of_life'] = datetime.fromisoformat(
|
||||
cfg['end_of_life'] + '+00:00').isoformat(timespec='seconds')
|
||||
if cfg['revision'] == '@NOW@':
|
||||
cfg['revision'] = NOW.strftime('%Y%m%d%H%M%S')
|
||||
# Order of operations is important here
|
||||
for k, v in cfg.items():
|
||||
transform = self._CFG_TRANSFORMS.get(k)
|
||||
if transform:
|
||||
cfg[k] = transform(v)
|
||||
|
||||
# 'ALL' region expansion (or retraction)
|
||||
if 'ALL' in cfg['ami_regions']:
|
||||
all_val = cfg['ami_regions']['ALL']
|
||||
if all_val not in [None, False]:
|
||||
cfg['ami_regions'] = all_regions
|
||||
else:
|
||||
cfg['ami_regions'] = {}
|
||||
else:
|
||||
# warn/remove disabled regions
|
||||
for region, enabled in all_regions.items():
|
||||
if enabled is not False or region not in cfg['ami_regions']:
|
||||
continue
|
||||
if cfg['ami_regions'][region] not in [None, False]:
|
||||
cfg['ami_regions'][region] = False
|
||||
if isinstance(v, str) and "{var." in v:
|
||||
cfg[k] = v.format(var=cfg)
|
||||
|
||||
# fold dict vars to scalars
|
||||
for foldkey, foldfmt in FOLD_DICTS.items():
|
||||
cfg[foldkey] = fold(cfg[foldkey], foldfmt)
|
||||
with open(os.path.join(build_dir, "vars.json"), "w") as out:
|
||||
json.dump(cfg, out, indent=4, separators=(",", ": "))
|
||||
|
||||
# fold 'svcs' dict to scalar
|
||||
lvls = {}
|
||||
for svc, lvl in cfg['svcs'].items():
|
||||
if lvl is True:
|
||||
# service in default runlevel
|
||||
lvls['default'].append(svc)
|
||||
elif lvl not in [None, False]:
|
||||
# service in specified runlevel (skip svc when false/null)
|
||||
if lvl not in lvls.keys():
|
||||
lvls[lvl] = []
|
||||
lvls[lvl].append(svc)
|
||||
cfg['svcs'] = ' '.join(
|
||||
str(lvl) + '=' + ','.join(
|
||||
str(svc) for svc in svcs
|
||||
) for lvl, svcs in lvls.items()
|
||||
)
|
||||
|
||||
# resolve ami_name and ami_desc
|
||||
cfg['ami_name'] = cfg['ami_name'].format(var=cfg)
|
||||
cfg['ami_desc'] = cfg['ami_desc'].format(var=cfg)
|
||||
def find_repo_root():
|
||||
path = os.getcwd()
|
||||
|
||||
# write build vars file
|
||||
with open(os.path.join(build_dir, 'vars.json'), 'w') as out:
|
||||
json.dump(cfg, out, indent=4, separators=(',', ': '))
|
||||
while ".git" not in set(os.listdir(path)) and path != "/":
|
||||
path = os.path.dirname(path)
|
||||
|
||||
print()
|
||||
if path == "/":
|
||||
raise Exception("No repo found, stopping at /")
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def main(args):
|
||||
parser = argparse.ArgumentParser(description="Build Packer JSON variable "
|
||||
"files from HOCON build profiles")
|
||||
parser.add_argument("profile", help="name of profile to build")
|
||||
args = parser.parse_args()
|
||||
|
||||
root = find_repo_root()
|
||||
|
||||
ConfigBuilder(
|
||||
os.path.join(root, "profiles", f"{args.profile}.conf"),
|
||||
os.path.join(root, "build", "profile", args.profile)
|
||||
).build(args.profile)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user