Fix path matching bugs by moving to pathlib

CHANGES.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.7.7
+- Add support for CLOUDBENDER_PROJECT_ROOT env variable to specify your root project 
+- Switch most os.path operations to pathlib to fix various corner cases caused by string matching
+
 ## 0.7.6
 - Added warning if rendered templates exceed max. inline size of 51200 bytes
 - Added optional removal of comments during include_raw processing to reduce user-data size

cloudbender/core.py

@@ -1,4 +1,4 @@
-import os
+import pathlib
 import logging
 
 from .utils import ensure_dir
@@ -12,24 +12,24 @@ logger = logging.getLogger(__name__)
 class CloudBender(object):
     """ Config Class to handle recursive conf/* config tree """
     def __init__(self, root_path):
-        self.root = root_path
+        self.root = pathlib.Path(root_path)
         self.sg = None
         self.all_stacks = []
         self.ctx = {
-            "config_path": os.path.join(self.root, "config"),
+            "config_path": self.root.joinpath("config"),
-            "template_path": os.path.join(self.root, "cloudformation"),
+            "template_path": self.root.joinpath("cloudformation"),
-            "parameter_path": os.path.join(self.root, "parameters"),
+            "parameter_path": self.root.joinpath("parameters"),
-            "artifact_paths": [os.path.join(self.root, "artifacts")]
+            "artifact_paths": [self.root.joinpath("artifacts")]
         }
 
-        if not os.path.isdir(self.ctx['config_path']):
+        if not self.ctx['config_path'].is_dir():
-            raise InvalidProjectDir("Check '{0}' exists and is a valid CloudBender project folder.".format(root_path))
+            raise InvalidProjectDir("Check '{0}' exists and is a valid CloudBender project folder.".format(self.ctx['config_path']))
 
     def read_config(self):
         """Load the <path>/config.yaml, <path>/*.yaml as stacks, sub-folders are sub-groups """
 
         # Read top level config.yaml and extract CloudBender CTX
-        _config = read_config_file(os.path.join(self.ctx['config_path'], 'config.yaml'))
+        _config = read_config_file(self.ctx['config_path'].joinpath('config.yaml'))
         if _config and _config.get('CloudBender'):
             self.ctx.update(_config.get('CloudBender'))
 
@@ -38,16 +38,17 @@ class CloudBender(object):
             if k in ['config_path', 'template_path', 'parameter_path', 'artifact_paths']:
                 if isinstance(v, list):
                     new_list = []
-                    for path in v:
+                    for p in v:
-                        if not os.path.isabs(path):
+                        path = pathlib.Path(p)
-                            new_list.append(os.path.normpath(os.path.join(self.root, path)))
+                        if not path.is_absolute():
+                            new_list.append(self.root.joinpath(path))
                         else:
                             new_list.append(path)
                     self.ctx[k] = new_list
 
                 elif isinstance(v, str):
-                    if not os.path.isabs(v):
+                    if not v.is_absolute():
-                        self.ctx[k] = os.path.normpath(os.path.join(self.root, v))
+                        self.ctx[k] = self.root.joinpath(v)
 
             if k in ['template_path', 'parameter_path']:
                 ensure_dir(self.ctx[k])

cloudbender/exceptions.py

@@ -6,5 +6,5 @@ class ParameterIllegalValue(Exception):
     """My documentation"""
 
 
-class InvalidProjectDir(BaseException):
+class InvalidProjectDir(Exception):
     """My documentation"""

cloudbender/jinja.py

@@ -179,7 +179,7 @@ def JinjaEnv(template_locations=[]):
 
     jinja_loaders = []
     for _dir in template_locations:
-        jinja_loaders.append(jinja2.FileSystemLoader(_dir))
+        jinja_loaders.append(jinja2.FileSystemLoader(str(_dir)))
     jenv.loader = jinja2.ChoiceLoader(jinja_loaders)
 
     jenv.globals['include_raw'] = include_raw_gz
@@ -206,14 +206,14 @@ def read_config_file(path, variables={}):
     jinja_variables = copy.deepcopy(variables)
     jinja_variables['ENV'] = os.environ
 
-    if os.path.exists(path):
+    if path.exists():
         logger.debug("Reading config file: {}".format(path))
         try:
             jenv = jinja2.Environment(
-                loader=jinja2.FileSystemLoader(os.path.dirname(path)),
+                loader=jinja2.FileSystemLoader(str(path.parent)),
                 undefined=jinja2.StrictUndefined,
                 extensions=['jinja2.ext.loopcontrols'])
-            template = jenv.get_template(os.path.basename(path))
+            template = jenv.get_template(path.name)
             rendered_template = template.render(jinja_variables)
             data = yaml.safe_load(rendered_template)
             if data:

cloudbender/stack.py

@@ -4,6 +4,7 @@ import hashlib
 import oyaml as yaml
 import json
 import time
+import pathlib
 import pprint
 
 from datetime import datetime, timedelta
@@ -37,7 +38,7 @@ class Stack(object):
     def __init__(self, name, template, path, rel_path, ctx):
         self.stackname = name
         self.template = template
-        self.path = path
+        self.path = pathlib.Path(path)
         self.rel_path = rel_path
         self.ctx = ctx
 

cloudbender/stackgroup.py

@@ -1,5 +1,3 @@
-import os
-import glob
 import logging
 import pprint
 
@@ -15,7 +13,7 @@ class StackGroup(object):
         self.name = None
         self.ctx = ctx
         self.path = path
-        self.rel_path = os.path.relpath(path, ctx['config_path'])
+        self.rel_path = path.relative_to(ctx['config_path'])
         self.config = {}
         self.sgs = []
         self.stacks = []
@@ -33,17 +31,17 @@ class StackGroup(object):
             s.dump_config()
 
     def read_config(self, parent_config={}):
-        if not os.path.isdir(self.path):
+        if not self.path.is_dir():
             return None
 
         # First read config.yaml if present
-        _config = read_config_file(os.path.join(self.path, 'config.yaml'), parent_config.get('variables', {}))
+        _config = read_config_file(self.path.joinpath('config.yaml'), parent_config.get('variables', {}))
 
         # Stack Group name if not explicit via config is derived from subfolder, or in case of root object the parent folder
         if "stackgroupname" in _config:
             self.name = _config["stackgroupname"]
         elif not self.name:
-            self.name = os.path.split(self.path)[1]
+            self.name = self.path.stem
 
         # Merge config with parent config
         self.config = dict_merge(parent_config, _config)
@@ -52,9 +50,9 @@ class StackGroup(object):
         logger.debug("StackGroup {} added.".format(self.name))
 
         # Add stacks
-        stacks = [s for s in glob.glob(os.path.join(self.path, '*.yaml')) if not s.endswith("config.yaml")]
+        stacks = [s for s in self.path.glob('*.yaml') if not s.name == "config.yaml"]
         for stack_path in stacks:
-            stackname = os.path.basename(stack_path).split('.')[0]
+            stackname = stack_path.name.split('.')[0]
             template = stackname
             if stackname_prefix:
                 stackname = stackname_prefix + stackname
@@ -64,7 +62,7 @@ class StackGroup(object):
             self.stacks.append(new_stack)
 
         # Create StackGroups recursively
-        for sub_group in [f.path for f in os.scandir(self.path) if f.is_dir()]:
+        for sub_group in [s for s in self.path.iterdir() if s.is_dir()]:
             sg = StackGroup(sub_group, self.ctx)
             sg.read_config(self.config)
 
@@ -77,12 +75,18 @@ class StackGroup(object):
             logger.debug("Looking for stack {} in group {}".format(name, self.name))
 
         for s in self.stacks:
-            if not name or (s.stackname == name and match_by == 'name') or (s.path.endswith(name) and match_by == 'path'):
+            if name:
-                if self.rel_path:
+                if match_by == 'name' and s.stackname != name:
-                    logger.debug("Found stack {} in group {}".format(s.stackname, self.rel_path))
+                    continue
-                else:
+
-                    logger.debug("Found stack {}".format(s.stackname))
+                if match_by == 'path' and not s.path.match(name):
-                stacks.append(s)
+                    continue
+
+            if self.rel_path:
+                logger.debug("Found stack {} in group {}".format(s.stackname, self.rel_path))
+            else:
+                logger.debug("Found stack {}".format(s.stackname))
+            stacks.append(s)
 
         if recursive:
             for sg in self.sgs:
@@ -94,11 +98,11 @@ class StackGroup(object):
 
     def get_stackgroup(self, name=None, recursive=True, match_by='name'):
         """ Returns stack group matching stackgroup_name or all if None """
-        if not name or (self.name == name and match_by == 'name') or (self.path.endswith(name) and match_by == 'path'):
+        if not name or (self.name == name and match_by == 'name') or (self.path.match(name) and match_by == 'path'):
             logger.debug("Found stack_group {}".format(self.name))
             return self
 
-        if name and name != 'config':
+        if name and self.name != 'config':
             logger.debug("Looking for stack_group {} in group {}".format(name, self.name))
 
         if recursive: