Refactor resolve-profile script
This is paving the way for identity broker improvements for opt-in regions. The output is functionally identical between the two scripts modulo the svcs change. Hopefully this makes the transformation process a little more clear.
This commit is contained in:
Mike Crute
parent
fe362af91f
commit
1fd42af98d
|
|
@ -56,25 +56,33 @@ pkgs {
|
||||||
tzdata = true
|
tzdata = true
|
||||||
}
|
}
|
||||||
svcs {
|
svcs {
|
||||||
devfs = "sysinit"
|
sysinit {
|
||||||
dmesg = "sysinit"
|
devfs = true
|
||||||
hwdrivers = "sysinit"
|
dmesg = true
|
||||||
mdev = "sysinit"
|
hwdrivers = true
|
||||||
acpid = "boot"
|
mdev = true
|
||||||
bootmisc = "boot"
|
}
|
||||||
hostname = "boot"
|
boot {
|
||||||
hwclock = "boot"
|
acpid = true
|
||||||
modules = "boot"
|
bootmisc = true
|
||||||
swap = "boot"
|
hostname = true
|
||||||
sysctl = "boot"
|
hwclock = true
|
||||||
syslog = "boot"
|
modules = true
|
||||||
chronyd = "default"
|
swap = true
|
||||||
networking = "default"
|
sysctl = true
|
||||||
sshd = "default"
|
syslog = true
|
||||||
tiny-ec2-bootstrap = "default"
|
}
|
||||||
killprocs = "shutdown"
|
default {
|
||||||
mount-ro = "shutdown"
|
chronyd = true
|
||||||
savecache = "shutdown"
|
networking = true
|
||||||
|
sshd = true
|
||||||
|
tiny-ec2-bootstrap = true
|
||||||
|
}
|
||||||
|
shutdown {
|
||||||
|
killprocs = true
|
||||||
|
mount-ro = true
|
||||||
|
savecache = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
kernel_modules {
|
kernel_modules {
|
||||||
sd-mod = true
|
sd-mod = true
|
||||||
|
|
|
||||||
|
|
@ -1,156 +1,133 @@
|
||||||
@PYTHON@
|
@PYTHON@
|
||||||
# vim: set ts=4 et:
|
# vim: set ts=4 et:
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
import shutil
|
|
||||||
import sys
|
import sys
|
||||||
import boto3
|
import json
|
||||||
from botocore.exceptions import ClientError
|
import shutil
|
||||||
|
import argparse
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
from pyhocon import ConfigFactory
|
from pyhocon import ConfigFactory
|
||||||
|
|
||||||
if len(sys.argv) != 2:
|
|
||||||
sys.exit("Usage: " + os.path.basename(__file__) + " <profile>")
|
|
||||||
|
|
||||||
PROFILE = sys.argv[1]
|
# Just group together our transforms
|
||||||
|
class Transforms:
|
||||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
|
||||||
|
|
||||||
# path to the profile config file
|
|
||||||
PROFILE_CONF = os.path.join(SCRIPT_DIR, '..', 'profiles', PROFILE + '.conf')
|
|
||||||
|
|
||||||
# load the profile's build configuration
|
|
||||||
BUILDS = ConfigFactory.parse_file(PROFILE_CONF)['BUILDS']
|
|
||||||
|
|
||||||
# where we store the profile's builds' config/output
|
|
||||||
PROFILE_DIR = os.path.join(SCRIPT_DIR, 'profile', PROFILE)
|
|
||||||
if not os.path.exists(PROFILE_DIR):
|
|
||||||
os.makedirs(PROFILE_DIR)
|
|
||||||
|
|
||||||
# fold these build config keys' dict to scalar
|
|
||||||
FOLD_DICTS = {
|
|
||||||
'ami_access': ',{0}',
|
|
||||||
'ami_regions': ',{0}',
|
|
||||||
'repos': "\n@{1} {0}",
|
|
||||||
'pkgs': ' {0}@{1}',
|
|
||||||
'kernel_modules': ',{0}',
|
|
||||||
'kernel_options': ' {0}'
|
|
||||||
}
|
|
||||||
|
|
||||||
NOW = datetime.utcnow()
|
NOW = datetime.utcnow()
|
||||||
ONE_DAY = timedelta(days=1)
|
TOMORROW = NOW + timedelta(days=1)
|
||||||
|
|
||||||
|
unquote = lambda x: x.strip('"')
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def force_iso_date(input):
|
||||||
|
return datetime.fromisoformat(input).isoformat(timespec="seconds")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def resolve_tomorrow(cls, input):
|
||||||
|
return cls.TOMORROW.isoformat(timespec="seconds")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def resolve_now(cls, input):
|
||||||
|
return cls.NOW.strftime("%Y%m%d%H%M%S")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def fold_comma(cls, input):
|
||||||
|
return ",".join([cls.unquote(k) for k in input.keys()])
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def fold_space(cls, input):
|
||||||
|
return " ".join([cls.unquote(k) for k in input.keys()])
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def fold_repos(cls, input):
|
||||||
|
return "\n".join(
|
||||||
|
f"@{v} {cls.unquote(k)}" if isinstance(v, str) else cls.unquote(k)
|
||||||
|
for k, v in input.items())
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def fold_packages(input):
|
||||||
|
return " ".join(
|
||||||
|
f"{k}@{v}" if isinstance(v, str) else k
|
||||||
|
for k, v in input.items())
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def fold_services(input):
|
||||||
|
return " ".join(
|
||||||
|
"{}={}".format(k, ",".join(v.keys()))
|
||||||
|
for k, v in input.items())
|
||||||
|
|
||||||
|
|
||||||
# func to fold dict down to scalar
|
class ConfigBuilder:
|
||||||
def fold(fdict, ffmt):
|
|
||||||
folded = ''
|
|
||||||
for fkey, fval in fdict.items():
|
|
||||||
fkey = fkey.strip('"') # complex keys may be in quotes
|
|
||||||
if fval is True:
|
|
||||||
folded += ffmt[0] + fkey
|
|
||||||
elif fval not in [None, False]:
|
|
||||||
folded += ffmt.format(fkey, fval)
|
|
||||||
return folded[1:]
|
|
||||||
|
|
||||||
|
_CFG_TRANSFORMS = {
|
||||||
|
"ami_access" : Transforms.fold_comma,
|
||||||
|
"ami_regions" : Transforms.fold_comma,
|
||||||
|
"kernel_modules" : Transforms.fold_comma,
|
||||||
|
"kernel_options" : Transforms.fold_space,
|
||||||
|
"repos" : Transforms.fold_repos,
|
||||||
|
"pkgs" : Transforms.fold_packages,
|
||||||
|
"svcs" : Transforms.fold_services,
|
||||||
|
"revision" : Transforms.resolve_now,
|
||||||
|
"end_of_life" : lambda x: \
|
||||||
|
Transforms.force_iso_date(Transforms.resolve_tomorrow(x)),
|
||||||
|
}
|
||||||
|
|
||||||
# list of AWS regions, and whether they're enabled
|
def __init__(self, config_path, out_dir):
|
||||||
all_regions = {}
|
self.config_path = config_path
|
||||||
AWS = boto3.session.Session()
|
self.out_dir = out_dir
|
||||||
sys.stderr.write("\n>>> Determining region availability...")
|
|
||||||
sys.stderr.flush()
|
|
||||||
for region in AWS.get_available_regions('ec2'):
|
|
||||||
ec2 = AWS.client('ec2', region_name=region)
|
|
||||||
try:
|
|
||||||
ec2.describe_regions()
|
|
||||||
except ClientError as e:
|
|
||||||
if e.response['Error']['Code'] == 'AuthFailure':
|
|
||||||
sys.stderr.write('-')
|
|
||||||
sys.stderr.flush()
|
|
||||||
all_regions[region] = False
|
|
||||||
continue
|
|
||||||
elif e.response['Error']['Code'] == 'UnauthorizedOperation':
|
|
||||||
# have access to the region, but not to ec2:DescribeRegions
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
raise
|
|
||||||
sys.stderr.write('+')
|
|
||||||
sys.stderr.flush()
|
|
||||||
all_regions[region] = True
|
|
||||||
sys.stderr.write("\n")
|
|
||||||
|
|
||||||
for region, available in all_regions.items():
|
def build(self, profile):
|
||||||
if available is False:
|
build_config = ConfigFactory.parse_file(self.config_path)
|
||||||
sys.stderr.write(f"*** WARNING: skipping disabled region {region}\n")
|
|
||||||
|
|
||||||
print()
|
for build, cfg in build_config["BUILDS"].items():
|
||||||
|
build_dir = os.path.join(self.out_dir, build)
|
||||||
|
|
||||||
# parse/resolve HOCON profile's builds' config
|
# Always start fresh
|
||||||
for build, cfg in BUILDS.items():
|
shutil.rmtree(build_dir, ignore_errors=True)
|
||||||
print(f">>> Resolving configuration for '{build}'")
|
|
||||||
build_dir = os.path.join(PROFILE_DIR, build)
|
|
||||||
|
|
||||||
# make a fresh profile build directory
|
|
||||||
if os.path.exists(build_dir):
|
|
||||||
shutil.rmtree(build_dir)
|
|
||||||
os.makedirs(build_dir)
|
os.makedirs(build_dir)
|
||||||
|
|
||||||
# populate profile build vars
|
cfg["profile"] = profile
|
||||||
cfg['profile'] = PROFILE
|
cfg["profile_build"] = build
|
||||||
cfg['profile_build'] = build
|
|
||||||
|
|
||||||
# mostly edge-related temporal substitutions
|
# Order of operations is important here
|
||||||
if cfg['end_of_life'] == '@TOMORROW@':
|
for k, v in cfg.items():
|
||||||
cfg['end_of_life'] = (NOW + ONE_DAY).isoformat(timespec='seconds')
|
transform = self._CFG_TRANSFORMS.get(k)
|
||||||
elif cfg['end_of_life'] is not None:
|
if transform:
|
||||||
# to explicitly UTC-ify end_of_life
|
cfg[k] = transform(v)
|
||||||
cfg['end_of_life'] = datetime.fromisoformat(
|
|
||||||
cfg['end_of_life'] + '+00:00').isoformat(timespec='seconds')
|
|
||||||
if cfg['revision'] == '@NOW@':
|
|
||||||
cfg['revision'] = NOW.strftime('%Y%m%d%H%M%S')
|
|
||||||
|
|
||||||
# 'ALL' region expansion (or retraction)
|
if isinstance(v, str) and "{var." in v:
|
||||||
if 'ALL' in cfg['ami_regions']:
|
cfg[k] = v.format(var=cfg)
|
||||||
all_val = cfg['ami_regions']['ALL']
|
|
||||||
if all_val not in [None, False]:
|
|
||||||
cfg['ami_regions'] = all_regions
|
|
||||||
else:
|
|
||||||
cfg['ami_regions'] = {}
|
|
||||||
else:
|
|
||||||
# warn/remove disabled regions
|
|
||||||
for region, enabled in all_regions.items():
|
|
||||||
if enabled is not False or region not in cfg['ami_regions']:
|
|
||||||
continue
|
|
||||||
if cfg['ami_regions'][region] not in [None, False]:
|
|
||||||
cfg['ami_regions'][region] = False
|
|
||||||
|
|
||||||
# fold dict vars to scalars
|
with open(os.path.join(build_dir, "vars.json"), "w") as out:
|
||||||
for foldkey, foldfmt in FOLD_DICTS.items():
|
json.dump(cfg, out, indent=4, separators=(",", ": "))
|
||||||
cfg[foldkey] = fold(cfg[foldkey], foldfmt)
|
|
||||||
|
|
||||||
# fold 'svcs' dict to scalar
|
|
||||||
lvls = {}
|
|
||||||
for svc, lvl in cfg['svcs'].items():
|
|
||||||
if lvl is True:
|
|
||||||
# service in default runlevel
|
|
||||||
lvls['default'].append(svc)
|
|
||||||
elif lvl not in [None, False]:
|
|
||||||
# service in specified runlevel (skip svc when false/null)
|
|
||||||
if lvl not in lvls.keys():
|
|
||||||
lvls[lvl] = []
|
|
||||||
lvls[lvl].append(svc)
|
|
||||||
cfg['svcs'] = ' '.join(
|
|
||||||
str(lvl) + '=' + ','.join(
|
|
||||||
str(svc) for svc in svcs
|
|
||||||
) for lvl, svcs in lvls.items()
|
|
||||||
)
|
|
||||||
|
|
||||||
# resolve ami_name and ami_desc
|
def find_repo_root():
|
||||||
cfg['ami_name'] = cfg['ami_name'].format(var=cfg)
|
path = os.getcwd()
|
||||||
cfg['ami_desc'] = cfg['ami_desc'].format(var=cfg)
|
|
||||||
|
|
||||||
# write build vars file
|
while ".git" not in set(os.listdir(path)) and path != "/":
|
||||||
with open(os.path.join(build_dir, 'vars.json'), 'w') as out:
|
path = os.path.dirname(path)
|
||||||
json.dump(cfg, out, indent=4, separators=(',', ': '))
|
|
||||||
|
|
||||||
print()
|
if path == "/":
|
||||||
|
raise Exception("No repo found, stopping at /")
|
||||||
|
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def main(args):
|
||||||
|
parser = argparse.ArgumentParser(description="Build Packer JSON variable "
|
||||||
|
"files from HOCON build profiles")
|
||||||
|
parser.add_argument("profile", help="name of profile to build")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
root = find_repo_root()
|
||||||
|
|
||||||
|
ConfigBuilder(
|
||||||
|
os.path.join(root, "profiles", f"{args.profile}.conf"),
|
||||||
|
os.path.join(root, "build", "profile", args.profile)
|
||||||
|
).build(args.profile)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main(sys.argv)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user