diff --git a/CHANGES.md b/CHANGES.md index af1338d..d2eaf47 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # Changelog +## 0.7.7 +- Add support for CLOUDBENDER_PROJECT_ROOT env variable to specify your root project +- Switch most os.path operations to pathlib to fix various corner cases caused by string matching + ## 0.7.6 - Added warning if rendered templates exceed max. inline size of 51200 bytes - Added optional removal of comments during include_raw processing to reduce user-data size diff --git a/cloudbender/core.py b/cloudbender/core.py index 7650a5b..ea44a87 100644 --- a/cloudbender/core.py +++ b/cloudbender/core.py @@ -1,4 +1,4 @@ -import os +import pathlib import logging from .utils import ensure_dir @@ -12,24 +12,24 @@ logger = logging.getLogger(__name__) class CloudBender(object): """ Config Class to handle recursive conf/* config tree """ def __init__(self, root_path): - self.root = root_path + self.root = pathlib.Path(root_path) self.sg = None self.all_stacks = [] self.ctx = { - "config_path": os.path.join(self.root, "config"), - "template_path": os.path.join(self.root, "cloudformation"), - "parameter_path": os.path.join(self.root, "parameters"), - "artifact_paths": [os.path.join(self.root, "artifacts")] + "config_path": self.root.joinpath("config"), + "template_path": self.root.joinpath("cloudformation"), + "parameter_path": self.root.joinpath("parameters"), + "artifact_paths": [self.root.joinpath("artifacts")] } - if not os.path.isdir(self.ctx['config_path']): - raise InvalidProjectDir("Check '{0}' exists and is a valid CloudBender project folder.".format(root_path)) + if not self.ctx['config_path'].is_dir(): + raise InvalidProjectDir("Check '{0}' exists and is a valid CloudBender project folder.".format(self.ctx['config_path'])) def read_config(self): """Load the /config.yaml, /*.yaml as stacks, sub-folders are sub-groups """ # Read top level config.yaml and extract CloudBender CTX - _config = read_config_file(os.path.join(self.ctx['config_path'], 'config.yaml')) + _config = read_config_file(self.ctx['config_path'].joinpath('config.yaml')) if _config and _config.get('CloudBender'): self.ctx.update(_config.get('CloudBender')) @@ -38,16 +38,17 @@ class CloudBender(object): if k in ['config_path', 'template_path', 'parameter_path', 'artifact_paths']: if isinstance(v, list): new_list = [] - for path in v: - if not os.path.isabs(path): - new_list.append(os.path.normpath(os.path.join(self.root, path))) + for p in v: + path = pathlib.Path(p) + if not path.is_absolute(): + new_list.append(self.root.joinpath(path)) else: new_list.append(path) self.ctx[k] = new_list elif isinstance(v, str): - if not os.path.isabs(v): - self.ctx[k] = os.path.normpath(os.path.join(self.root, v)) + if not v.is_absolute(): + self.ctx[k] = self.root.joinpath(v) if k in ['template_path', 'parameter_path']: ensure_dir(self.ctx[k]) diff --git a/cloudbender/exceptions.py b/cloudbender/exceptions.py index 3bbc901..2604c51 100644 --- a/cloudbender/exceptions.py +++ b/cloudbender/exceptions.py @@ -6,5 +6,5 @@ class ParameterIllegalValue(Exception): """My documentation""" -class InvalidProjectDir(BaseException): +class InvalidProjectDir(Exception): """My documentation""" diff --git a/cloudbender/jinja.py b/cloudbender/jinja.py index 71ed54a..e105fc4 100644 --- a/cloudbender/jinja.py +++ b/cloudbender/jinja.py @@ -179,7 +179,7 @@ def JinjaEnv(template_locations=[]): jinja_loaders = [] for _dir in template_locations: - jinja_loaders.append(jinja2.FileSystemLoader(_dir)) + jinja_loaders.append(jinja2.FileSystemLoader(str(_dir))) jenv.loader = jinja2.ChoiceLoader(jinja_loaders) jenv.globals['include_raw'] = include_raw_gz @@ -206,14 +206,14 @@ def read_config_file(path, variables={}): jinja_variables = copy.deepcopy(variables) jinja_variables['ENV'] = os.environ - if os.path.exists(path): + if path.exists(): logger.debug("Reading config file: {}".format(path)) try: jenv = jinja2.Environment( - loader=jinja2.FileSystemLoader(os.path.dirname(path)), + loader=jinja2.FileSystemLoader(str(path.parent)), undefined=jinja2.StrictUndefined, extensions=['jinja2.ext.loopcontrols']) - template = jenv.get_template(os.path.basename(path)) + template = jenv.get_template(path.name) rendered_template = template.render(jinja_variables) data = yaml.safe_load(rendered_template) if data: diff --git a/cloudbender/stack.py b/cloudbender/stack.py index 15cc7a6..7bf4a21 100644 --- a/cloudbender/stack.py +++ b/cloudbender/stack.py @@ -4,6 +4,7 @@ import hashlib import oyaml as yaml import json import time +import pathlib import pprint from datetime import datetime, timedelta @@ -37,7 +38,7 @@ class Stack(object): def __init__(self, name, template, path, rel_path, ctx): self.stackname = name self.template = template - self.path = path + self.path = pathlib.Path(path) self.rel_path = rel_path self.ctx = ctx diff --git a/cloudbender/stackgroup.py b/cloudbender/stackgroup.py index 2b8723c..5a3c007 100644 --- a/cloudbender/stackgroup.py +++ b/cloudbender/stackgroup.py @@ -1,5 +1,3 @@ -import os -import glob import logging import pprint @@ -15,7 +13,7 @@ class StackGroup(object): self.name = None self.ctx = ctx self.path = path - self.rel_path = os.path.relpath(path, ctx['config_path']) + self.rel_path = path.relative_to(ctx['config_path']) self.config = {} self.sgs = [] self.stacks = [] @@ -33,17 +31,17 @@ class StackGroup(object): s.dump_config() def read_config(self, parent_config={}): - if not os.path.isdir(self.path): + if not self.path.is_dir(): return None # First read config.yaml if present - _config = read_config_file(os.path.join(self.path, 'config.yaml'), parent_config.get('variables', {})) + _config = read_config_file(self.path.joinpath('config.yaml'), parent_config.get('variables', {})) # Stack Group name if not explicit via config is derived from subfolder, or in case of root object the parent folder if "stackgroupname" in _config: self.name = _config["stackgroupname"] elif not self.name: - self.name = os.path.split(self.path)[1] + self.name = self.path.stem # Merge config with parent config self.config = dict_merge(parent_config, _config) @@ -52,9 +50,9 @@ class StackGroup(object): logger.debug("StackGroup {} added.".format(self.name)) # Add stacks - stacks = [s for s in glob.glob(os.path.join(self.path, '*.yaml')) if not s.endswith("config.yaml")] + stacks = [s for s in self.path.glob('*.yaml') if not s.name == "config.yaml"] for stack_path in stacks: - stackname = os.path.basename(stack_path).split('.')[0] + stackname = stack_path.name.split('.')[0] template = stackname if stackname_prefix: stackname = stackname_prefix + stackname @@ -64,7 +62,7 @@ class StackGroup(object): self.stacks.append(new_stack) # Create StackGroups recursively - for sub_group in [f.path for f in os.scandir(self.path) if f.is_dir()]: + for sub_group in [s for s in self.path.iterdir() if s.is_dir()]: sg = StackGroup(sub_group, self.ctx) sg.read_config(self.config) @@ -77,12 +75,18 @@ class StackGroup(object): logger.debug("Looking for stack {} in group {}".format(name, self.name)) for s in self.stacks: - if not name or (s.stackname == name and match_by == 'name') or (s.path.endswith(name) and match_by == 'path'): - if self.rel_path: - logger.debug("Found stack {} in group {}".format(s.stackname, self.rel_path)) - else: - logger.debug("Found stack {}".format(s.stackname)) - stacks.append(s) + if name: + if match_by == 'name' and s.stackname != name: + continue + + if match_by == 'path' and not s.path.match(name): + continue + + if self.rel_path: + logger.debug("Found stack {} in group {}".format(s.stackname, self.rel_path)) + else: + logger.debug("Found stack {}".format(s.stackname)) + stacks.append(s) if recursive: for sg in self.sgs: @@ -94,11 +98,11 @@ class StackGroup(object): def get_stackgroup(self, name=None, recursive=True, match_by='name'): """ Returns stack group matching stackgroup_name or all if None """ - if not name or (self.name == name and match_by == 'name') or (self.path.endswith(name) and match_by == 'path'): + if not name or (self.name == name and match_by == 'name') or (self.path.match(name) and match_by == 'path'): logger.debug("Found stack_group {}".format(self.name)) return self - if name and name != 'config': + if name and self.name != 'config': logger.debug("Looking for stack_group {} in group {}".format(name, self.name)) if recursive: