CloudBender/cloudbender/cli.py

492 lines
14 KiB
Python
Raw Permalink Normal View History

2018-11-22 18:31:59 +00:00
import os
import sys
2018-11-22 18:31:59 +00:00
import click
import functools
import re
2018-11-22 18:31:59 +00:00
2018-12-10 17:58:54 +00:00
from concurrent.futures import ThreadPoolExecutor, as_completed
2018-11-22 18:31:59 +00:00
from . import __version__
from .core import CloudBender
from .utils import setup_logging, get_docker_version
from .exceptions import InvalidProjectDir
from .pulumi import get_pulumi_version
2018-11-22 18:31:59 +00:00
import logging
2022-02-22 10:04:29 +00:00
2018-11-22 18:31:59 +00:00
logger = logging.getLogger(__name__)
2018-11-22 18:31:59 +00:00
@click.group()
@click.option(
"--profile",
"profile",
help="Use named AWS .config profile, overwrites any stack config",
)
@click.option(
"--region",
"region",
help="Use region, overwrites any stack config",
)
2018-11-22 18:31:59 +00:00
@click.option("--dir", "directory", help="Specify cloudbender project directory.")
@click.option("--debug", is_flag=True, help="Turn on debug logging.")
2018-11-22 18:31:59 +00:00
@click.pass_context
def cli(ctx, profile, region, debug, directory):
setup_logging(debug)
2018-11-22 18:31:59 +00:00
# Skip parsing all the things if we just want the versions
if ctx.invoked_subcommand == "version":
return
# Make sure our root is abs
if directory:
if not os.path.isabs(directory):
directory = os.path.normpath(os.path.join(os.getcwd(), directory))
elif os.getenv("CLOUDBENDER_PROJECT_ROOT"):
directory = os.getenv("CLOUDBENDER_PROJECT_ROOT")
else:
directory = os.getcwd()
# Read global config
try:
cb = CloudBender(directory, profile, region)
except InvalidProjectDir as e:
logger.error(e)
sys.exit(1)
# Only load stackgroups to get profile and region
if ctx.invoked_subcommand == "wrap":
cb.read_config(loadStacks=False)
else:
cb.read_config()
cb.dump_config()
ctx.obj = cb
2018-11-22 18:31:59 +00:00
@click.command()
def version():
"""Displays own version and all dependencies"""
logger.error(f"CloudBender: {__version__}")
# Pulumi
pulumi_version = get_pulumi_version()
if not pulumi_version:
logger.error(
"Pulumi: Error calling pulumi, see https://www.pulumi.com/docs/get-started/install/"
)
else:
logger.error(f"Pulumi: {pulumi_version}")
# Docker / podman version
docker_version = get_docker_version()
if not docker_version:
logger.error("Podman/Docker: Cannot call podman nor docker")
else:
logger.error(f"Podman/Docker: {docker_version}")
2018-11-22 18:31:59 +00:00
@click.command()
@click.argument("stack_names", nargs=-1)
2018-11-22 18:31:59 +00:00
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
2019-02-08 10:51:44 +00:00
@click.pass_obj
def render(cb, stack_names, multi):
2022-02-22 10:04:29 +00:00
"""Renders template and its parameters - CFN only"""
2018-11-22 18:31:59 +00:00
2019-02-08 10:51:44 +00:00
stacks = _find_stacks(cb, stack_names, multi)
_render(stacks)
2018-11-22 18:31:59 +00:00
@click.command()
@click.argument("stack_names", nargs=-1)
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
@click.pass_obj
def sync(cb, stack_names, multi):
2022-02-22 10:04:29 +00:00
"""Renders template and provisions it right away"""
stacks = _find_stacks(cb, stack_names, multi)
_render(stacks)
_provision(cb, stacks)
2018-11-22 18:31:59 +00:00
@click.command()
@click.argument("stack_names", nargs=-1)
2018-11-22 18:31:59 +00:00
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
2019-02-08 10:51:44 +00:00
@click.pass_obj
def validate(cb, stack_names, multi):
2022-02-22 10:04:29 +00:00
"""Validates already rendered templates using cfn-lint - CFN only"""
2019-02-08 10:51:44 +00:00
stacks = _find_stacks(cb, stack_names, multi)
2018-11-22 18:31:59 +00:00
for s in stacks:
ret = s.validate()
if ret:
sys.exit(ret)
2018-11-22 18:31:59 +00:00
2020-02-25 20:40:12 +00:00
@click.command()
@click.argument("stack_names", nargs=-1)
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
2022-02-22 10:04:29 +00:00
@click.option(
"--include", default=".*", help="regex matching wanted outputs, default '.*'"
)
@click.option(
"--values",
is_flag=True,
help="Only output values, most useful if only one outputs is returned",
)
2020-02-25 20:40:12 +00:00
@click.pass_obj
def outputs(cb, stack_names, multi, include, values):
2022-02-22 10:04:29 +00:00
"""Prints all stack outputs"""
2020-02-25 20:40:12 +00:00
stacks = _find_stacks(cb, stack_names, multi)
for s in stacks:
s.get_outputs()
for output in s.outputs.keys():
if re.search(include, output):
if values:
print("{}".format(output))
else:
print("{}={}".format(output, s.outputs[output]))
2020-02-25 20:40:12 +00:00
@click.command()
@click.argument("stack_names", nargs=-1)
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
@click.pass_obj
def docs(cb, stack_names, multi):
"""Outputs docs for stack(s). For Pulumi stacks prints out docstring. For CloudFormation templates render a markdown file. Same idea as helm-docs."""
stacks = _find_stacks(cb, stack_names, multi)
for s in stacks:
s.docs()
2019-01-30 13:00:06 +00:00
@click.command()
@click.argument("stack_name")
@click.argument("change_set_name")
2019-02-08 10:51:44 +00:00
@click.pass_obj
def create_change_set(cb, stack_name, change_set_name):
2022-02-22 10:04:29 +00:00
"""Creates a change set for an existing stack - CFN only"""
2019-02-08 10:51:44 +00:00
stacks = _find_stacks(cb, [stack_name])
2019-01-30 13:00:06 +00:00
for s in stacks:
s.create_change_set(change_set_name)
2021-09-20 14:19:14 +00:00
@click.command()
@click.argument("stack_name")
@click.pass_obj
def refresh(cb, stack_name):
2022-02-22 10:04:29 +00:00
"""Refreshes Pulumi stack / Drift detection"""
2021-09-20 14:19:14 +00:00
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
2022-02-22 10:04:29 +00:00
if s.mode == "pulumi":
2021-09-20 14:19:14 +00:00
s.refresh()
else:
2022-02-22 10:04:29 +00:00
logger.info("{} uses Cloudformation, refresh skipped.".format(s.stackname))
2021-09-20 14:19:14 +00:00
@click.command()
@click.argument("stack_name")
@click.argument("function", default="")
@click.argument("args", nargs=-1)
@click.pass_obj
def execute(cb, stack_name, function, args):
"""Executes custom Python function within an existing stack context"""
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
if s.mode == "pulumi":
ret = s.execute(function, args)
if ret:
raise click.Abort()
else:
logger.info(
"{} uses Cloudformation, no execute feature available.".format(
s.stackname
)
)
@click.command()
@click.argument("stack_name")
@click.option(
"-r",
"--remove-pending-operations",
2022-02-22 10:04:29 +00:00
is_flag=True,
help="All pending stack operations are removed and the stack will be re-imported",
)
@click.pass_obj
def export(cb, stack_name, remove_pending_operations=False):
2022-02-22 10:04:29 +00:00
"""Exports a Pulumi stack to repair state"""
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
2022-02-22 10:04:29 +00:00
if s.mode == "pulumi":
s.export(remove_pending_operations)
else:
2022-02-22 10:04:29 +00:00
logger.info("{} uses Cloudformation, export skipped.".format(s.stackname))
@click.command()
@click.argument("stack_name")
@click.pass_obj
def assimilate(cb, stack_name):
"""Imports potentially existing resources into Pulumi stack"""
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
if s.mode == "pulumi":
s.assimilate()
else:
logger.info(
"{} uses Cloudformation, cannot assimilate.".format(s.stackname)
)
2021-09-20 14:19:14 +00:00
@click.command()
@click.argument("stack_name")
@click.argument("key")
@click.argument("value")
@click.option("--secret", is_flag=True, help="Value is a secret")
@click.pass_obj
def set_config(cb, stack_name, key, value, secret=False):
2022-02-22 10:04:29 +00:00
"""Sets a config value, encrypts with stack key if secret"""
2021-09-20 14:19:14 +00:00
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
s.set_config(key, value, secret)
@click.command()
@click.argument("stack_name")
@click.argument("key")
@click.pass_obj
def get_config(cb, stack_name, key):
2022-02-22 10:04:29 +00:00
"""Get a config value, decrypted if secret"""
2021-09-20 14:19:14 +00:00
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
s.get_config(key)
@click.command()
@click.argument("stack_name")
@click.pass_obj
def preview(cb, stack_name):
2022-02-22 10:04:29 +00:00
"""Preview of Pulumi stack up operation"""
2021-09-20 14:19:14 +00:00
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
2022-02-22 10:04:29 +00:00
if s.mode == "pulumi":
2021-09-20 14:19:14 +00:00
s.preview()
else:
2022-02-22 10:04:29 +00:00
logger.warning(
"{} uses Cloudformation, use create-change-set for previews.".format(
s.stackname
)
)
2021-09-20 14:19:14 +00:00
2018-11-22 18:31:59 +00:00
@click.command()
@click.argument("stack_names", nargs=-1)
2018-11-22 18:31:59 +00:00
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
2019-02-08 10:51:44 +00:00
@click.pass_obj
def provision(cb, stack_names, multi):
2022-02-22 10:04:29 +00:00
"""Creates or updates stacks or stack groups"""
2019-02-08 10:51:44 +00:00
stacks = _find_stacks(cb, stack_names, multi)
_provision(cb, stacks)
2018-11-22 18:31:59 +00:00
@click.command()
@click.argument("stack_names", nargs=-1)
2018-11-22 18:31:59 +00:00
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
2019-02-08 10:51:44 +00:00
@click.pass_obj
def delete(cb, stack_names, multi):
2022-02-22 10:04:29 +00:00
"""Deletes stacks or stack groups"""
2019-02-08 10:51:44 +00:00
stacks = _find_stacks(cb, stack_names, multi)
2018-11-22 18:31:59 +00:00
# Reverse steps
2019-02-08 10:51:44 +00:00
steps = [s for s in sort_stacks(cb, stacks)]
2018-11-22 18:31:59 +00:00
delete_steps = steps[::-1]
for step in delete_steps:
if step:
with ThreadPoolExecutor(max_workers=len(step)) as group:
futures = []
for stack in step:
if stack.multi_delete:
futures.append(group.submit(stack.delete))
2018-12-10 17:58:54 +00:00
for future in as_completed(futures):
future.result()
2018-11-22 18:31:59 +00:00
@click.command()
@click.argument("stack_group", nargs=1, required=True)
@click.argument("cmd", nargs=-1, required=True)
@click.pass_obj
def wrap(cb, stack_group, cmd):
"""Execute custom external program"""
sg = cb.sg.get_stackgroup(stack_group)
cb.wrap(sg, " ".join(cmd))
2018-11-22 18:31:59 +00:00
@click.command()
2019-02-08 10:51:44 +00:00
@click.pass_obj
def clean(cb):
2022-02-22 10:04:29 +00:00
"""Deletes all previously rendered files locally"""
2018-11-22 18:31:59 +00:00
cb.clean()
2019-02-08 10:51:44 +00:00
def sort_stacks(cb, stacks):
2022-02-22 10:04:29 +00:00
"""Sort stacks by dependencies"""
2018-11-22 18:31:59 +00:00
data = {}
for s in stacks:
2022-02-22 10:04:29 +00:00
if s.mode == "pulumi":
2021-09-20 14:19:14 +00:00
data[s.id] = set()
continue
# To resolve dependencies we have to read each template
s.read_template_file()
2018-11-22 18:31:59 +00:00
deps = []
for d in s.dependencies:
# For now we assume deps are artifacts so we prepend them with our local profile and region to match stack.id
2022-02-22 10:04:29 +00:00
for dep_stack in cb.filter_stacks(
{"region": s.region, "profile": s.profile, "provides": d}
):
2018-11-22 18:31:59 +00:00
deps.append(dep_stack.id)
# also look for global services
2022-02-22 10:04:29 +00:00
for dep_stack in cb.filter_stacks(
{"region": "global", "profile": s.profile, "provides": d}
):
deps.append(dep_stack.id)
2018-12-10 17:58:54 +00:00
2018-11-22 18:31:59 +00:00
data[s.id] = set(deps)
logger.debug("Stack {} depends on {}".format(s.id, deps))
2018-11-22 18:31:59 +00:00
# Ignore self dependencies
2018-11-22 18:31:59 +00:00
for k, v in data.items():
v.discard(k)
2018-11-22 18:31:59 +00:00
2021-09-20 14:19:14 +00:00
if data:
2022-02-22 10:04:29 +00:00
extra_items_in_deps = functools.reduce(set.union, data.values()) - set(
data.keys()
)
2021-09-20 14:19:14 +00:00
data.update({item: set() for item in extra_items_in_deps})
2018-11-22 18:31:59 +00:00
while True:
ordered = set(item for item, dep in data.items() if not dep)
2018-11-22 18:31:59 +00:00
if not ordered:
break
# return list of stack objects rather than just names
result = []
for o in ordered:
for s in stacks:
if s.id == o:
result.append(s)
2018-11-22 18:31:59 +00:00
yield result
2022-02-22 10:04:29 +00:00
data = {
item: (dep - ordered) for item, dep in data.items() if item not in ordered
}
2018-11-22 18:31:59 +00:00
assert not data, "A cyclic dependency exists amongst %r" % data
2019-02-08 10:51:44 +00:00
def _find_stacks(cb, stack_names, multi=False):
2022-02-22 10:04:29 +00:00
"""search stacks by name"""
2018-11-22 18:31:59 +00:00
stacks = []
for s in stack_names:
stacks = stacks + cb.resolve_stacks(s)
2018-11-22 18:31:59 +00:00
if not multi and len(stacks) > 1:
2022-02-22 10:04:29 +00:00
logger.error(
"Found more than one stack matching name ({}). Please set --multi if that is what you want.".format(
", ".join(stack_names)
)
)
2018-11-22 18:31:59 +00:00
raise click.Abort()
if not stacks:
2022-02-22 10:04:29 +00:00
logger.error("Cannot find stack matching: {}".format(", ".join(stack_names)))
2018-11-22 18:31:59 +00:00
raise click.Abort()
return stacks
def _render(stacks):
2022-02-22 10:04:29 +00:00
"""Utility function to reuse code between tasks"""
for s in stacks:
2022-02-22 10:04:29 +00:00
if s.mode != "pulumi":
2021-09-20 14:19:14 +00:00
s.render()
s.write_template_file()
else:
2022-02-22 10:04:29 +00:00
logger.info("{} uses Pulumi, render skipped.".format(s.stackname))
def _anyPulumi(step):
for stack in step:
if stack.mode == "pulumi":
return True
return False
def _provision(cb, stacks):
2022-02-22 10:04:29 +00:00
"""Utility function to reuse code between tasks"""
for step in sort_stacks(cb, stacks):
if step:
2024-04-22 13:07:54 +00:00
# Pulumi is still not thread safe
if _anyPulumi(step):
_threads = 1
2024-04-23 13:14:29 +00:00
else:
2024-04-22 13:07:54 +00:00
_threads = len(step)
2024-04-23 13:14:29 +00:00
with ThreadPoolExecutor(max_workers=_threads) as group:
2023-12-04 16:24:55 +00:00
futures = []
for stack in step:
if stack.mode != "pulumi":
status = stack.get_status()
if not status:
2023-12-04 16:24:55 +00:00
futures.append(group.submit(stack.create))
else:
2023-12-04 16:24:55 +00:00
futures.append(group.submit(stack.update))
# Pulumi only needs "up"
else:
2023-12-04 16:24:55 +00:00
futures.append(group.submit(stack.create))
2023-12-04 16:24:55 +00:00
for future in as_completed(futures):
future.result()
cli.add_command(version)
2018-11-22 18:31:59 +00:00
cli.add_command(render)
cli.add_command(sync)
2018-11-22 18:31:59 +00:00
cli.add_command(validate)
cli.add_command(provision)
cli.add_command(delete)
cli.add_command(clean)
2019-01-30 13:00:06 +00:00
cli.add_command(create_change_set)
2020-02-25 20:40:12 +00:00
cli.add_command(outputs)
cli.add_command(docs)
2021-09-20 14:19:14 +00:00
cli.add_command(refresh)
cli.add_command(preview)
cli.add_command(set_config)
cli.add_command(get_config)
cli.add_command(export)
cli.add_command(assimilate)
cli.add_command(execute)
cli.add_command(wrap)
2022-02-22 10:04:29 +00:00
if __name__ == "__main__":
cli(obj={})