ZeroDownTime/CloudBender
3
0
Fork 0
Code Issues 1 Pull Requests 4 Packages Projects Releases Wiki Activity

Fix path matching bugs by moving to pathlib

Browse Source
This commit is contained in:
Stefan Reimer 2019-12-09 13:32:39 +00:00
parent 6cadc18397
commit 7782c180f3
6 changed files with 47 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGES.md
View File

@ -1,5 +1,9 @@
# Changelog
## 0.7.7
- Add support for CLOUDBENDER_PROJECT_ROOT env variable to specify your root project
- Switch most os.path operations to pathlib to fix various corner cases caused by string matching
## 0.7.6
- Added warning if rendered templates exceed max. inline size of 51200 bytes
- Added optional removal of comments during include_raw processing to reduce user-data size

29
cloudbender/core.py
View File

@ -1,4 +1,4 @@
import os
import pathlib
import logging
from .utils import ensure_dir
@ -12,24 +12,24 @@ logger = logging.getLogger(__name__)
class CloudBender(object):
""" Config Class to handle recursive conf/* config tree """
def __init__(self, root_path):
self.root = root_path
self.root = pathlib.Path(root_path)
self.sg = None
self.all_stacks = []
self.ctx = {
"config_path": os.path.join(self.root, "config"),
"template_path": os.path.join(self.root, "cloudformation"),
"parameter_path": os.path.join(self.root, "parameters"),
"artifact_paths": [os.path.join(self.root, "artifacts")]
"config_path": self.root.joinpath("config"),
"template_path": self.root.joinpath("cloudformation"),
"parameter_path": self.root.joinpath("parameters"),
"artifact_paths": [self.root.joinpath("artifacts")]
}
if not os.path.isdir(self.ctx['config_path']):
raise InvalidProjectDir("Check '{0}' exists and is a valid CloudBender project folder.".format(root_path))
if not self.ctx['config_path'].is_dir():
raise InvalidProjectDir("Check '{0}' exists and is a valid CloudBender project folder.".format(self.ctx['config_path']))
def read_config(self):
"""Load the <path>/config.yaml, <path>/*.yaml as stacks, sub-folders are sub-groups """
# Read top level config.yaml and extract CloudBender CTX
_config = read_config_file(os.path.join(self.ctx['config_path'], 'config.yaml'))
_config = read_config_file(self.ctx['config_path'].joinpath('config.yaml'))
if _config and _config.get('CloudBender'):
self.ctx.update(_config.get('CloudBender'))
@ -38,16 +38,17 @@ class CloudBender(object):
if k in ['config_path', 'template_path', 'parameter_path', 'artifact_paths']:
if isinstance(v, list):
new_list = []
for path in v:
if not os.path.isabs(path):
new_list.append(os.path.normpath(os.path.join(self.root, path)))
for p in v:
path = pathlib.Path(p)
if not path.is_absolute():
new_list.append(self.root.joinpath(path))
else:
new_list.append(path)
self.ctx[k] = new_list
elif isinstance(v, str):
if not os.path.isabs(v):
self.ctx[k] = os.path.normpath(os.path.join(self.root, v))
if not v.is_absolute():
self.ctx[k] = self.root.joinpath(v)
if k in ['template_path', 'parameter_path']:
ensure_dir(self.ctx[k])

2
cloudbender/exceptions.py
View File

@ -6,5 +6,5 @@ class ParameterIllegalValue(Exception):
"""My documentation"""
class InvalidProjectDir(BaseException):
class InvalidProjectDir(Exception):
"""My documentation"""

8
cloudbender/jinja.py
View File

@ -179,7 +179,7 @@ def JinjaEnv(template_locations=[]):
jinja_loaders = []
for _dir in template_locations:
jinja_loaders.append(jinja2.FileSystemLoader(_dir))
jinja_loaders.append(jinja2.FileSystemLoader(str(_dir)))
jenv.loader = jinja2.ChoiceLoader(jinja_loaders)
jenv.globals['include_raw'] = include_raw_gz
@ -206,14 +206,14 @@ def read_config_file(path, variables={}):
jinja_variables = copy.deepcopy(variables)
jinja_variables['ENV'] = os.environ
if os.path.exists(path):
if path.exists():
logger.debug("Reading config file: {}".format(path))
try:
jenv = jinja2.Environment(
loader=jinja2.FileSystemLoader(os.path.dirname(path)),
loader=jinja2.FileSystemLoader(str(path.parent)),
undefined=jinja2.StrictUndefined,
extensions=['jinja2.ext.loopcontrols'])
template = jenv.get_template(os.path.basename(path))
template = jenv.get_template(path.name)
rendered_template = template.render(jinja_variables)
data = yaml.safe_load(rendered_template)
if data:

3
cloudbender/stack.py
View File

@ -4,6 +4,7 @@ import hashlib
import oyaml as yaml
import json
import time
import pathlib
import pprint
from datetime import datetime, timedelta
@ -37,7 +38,7 @@ class Stack(object):
def __init__(self, name, template, path, rel_path, ctx):
self.stackname = name
self.template = template
self.path = path
self.path = pathlib.Path(path)
self.rel_path = rel_path
self.ctx = ctx

28
cloudbender/stackgroup.py
View File

@ -1,5 +1,3 @@
import os
import glob
import logging
import pprint
@ -15,7 +13,7 @@ class StackGroup(object):
self.name = None
self.ctx = ctx
self.path = path
self.rel_path = os.path.relpath(path, ctx['config_path'])
self.rel_path = path.relative_to(ctx['config_path'])
self.config = {}
self.sgs = []
self.stacks = []
@ -33,17 +31,17 @@ class StackGroup(object):
s.dump_config()
def read_config(self, parent_config={}):
if not os.path.isdir(self.path):
if not self.path.is_dir():
return None
# First read config.yaml if present
_config = read_config_file(os.path.join(self.path, 'config.yaml'), parent_config.get('variables', {}))
_config = read_config_file(self.path.joinpath('config.yaml'), parent_config.get('variables', {}))
# Stack Group name if not explicit via config is derived from subfolder, or in case of root object the parent folder
if "stackgroupname" in _config:
self.name = _config["stackgroupname"]
elif not self.name:
self.name = os.path.split(self.path)[1]
self.name = self.path.stem
# Merge config with parent config
self.config = dict_merge(parent_config, _config)
@ -52,9 +50,9 @@ class StackGroup(object):
logger.debug("StackGroup {} added.".format(self.name))
# Add stacks
stacks = [s for s in glob.glob(os.path.join(self.path, '*.yaml')) if not s.endswith("config.yaml")]
stacks = [s for s in self.path.glob('*.yaml') if not s.name == "config.yaml"]
for stack_path in stacks:
stackname = os.path.basename(stack_path).split('.')[0]
stackname = stack_path.name.split('.')[0]
template = stackname
if stackname_prefix:
stackname = stackname_prefix + stackname
@ -64,7 +62,7 @@ class StackGroup(object):
self.stacks.append(new_stack)
# Create StackGroups recursively
for sub_group in [f.path for f in os.scandir(self.path) if f.is_dir()]:
for sub_group in [s for s in self.path.iterdir() if s.is_dir()]:
sg = StackGroup(sub_group, self.ctx)
sg.read_config(self.config)
@ -77,7 +75,13 @@ class StackGroup(object):
logger.debug("Looking for stack {} in group {}".format(name, self.name))
for s in self.stacks:
if not name or (s.stackname == name and match_by == 'name') or (s.path.endswith(name) and match_by == 'path'):
if name:
if match_by == 'name' and s.stackname != name:
continue
if match_by == 'path' and not s.path.match(name):
continue
if self.rel_path:
logger.debug("Found stack {} in group {}".format(s.stackname, self.rel_path))
else:
@ -94,11 +98,11 @@ class StackGroup(object):
def get_stackgroup(self, name=None, recursive=True, match_by='name'):
""" Returns stack group matching stackgroup_name or all if None """
if not name or (self.name == name and match_by == 'name') or (self.path.endswith(name) and match_by == 'path'):
if not name or (self.name == name and match_by == 'name') or (self.path.match(name) and match_by == 'path'):
logger.debug("Found stack_group {}".format(self.name))
return self
if name and name != 'config':
if name and self.name != 'config':
logger.debug("Looking for stack_group {} in group {}".format(name, self.name))
if recursive: