Add sops support, improved secret handling

CHANGES.md

@@ -1,10 +1,16 @@
 # Changelog
 
+## 0.8.0
+- Added support for sops encrypted config files, see: https://github.com/mozilla/sops
+- hide stack parameter output in terminal if `NoEcho` is set
+- *CloudBender no longer writes stack parameter files to prevent leaking secret values !*  
+  These files were never actually used anyways and there sole purpose was to track changes via git.
+
 ## 0.7.8
 - Add new function `outputs`, to query already deployed stack for their outputs
 
 ## 0.7.7
-- Add support for CLOUDBENDER_PROJECT_ROOT env variable to specify your root project 
+- Add support for CLOUDBENDER_PROJECT_ROOT env variable to specify your root project
 - Switch most os.path operations to pathlib to fix various corner cases caused by string matching
 
 ## 0.7.6

README.md

@@ -32,3 +32,11 @@ Commands:
   sync               Renders template and provisions it right away
   validate           Validates already rendered templates using cfn-lint
 ```
+
+# Secrets
+
+CloudBender supports Mozilla's [SOPS](https://github.com/mozilla/sops) to encrypt values in any config yaml file since version 0.8.  
+
+
+If a sops encrypted config file is detected CloudBender will automatically try to decrypt the file during execution.  
+All required information to decrypt has to be present in the embedded sops config or set ahead of time via sops supported ENVIRONMENT variables.

cloudbender/__init__.py

@@ -2,7 +2,7 @@ import logging
 
 __author__ = "Stefan Reimer"
 __email__ = "stefan@zero-downtimet.net"
-__version__ = "0.7.8"
+__version__ = "0.8.0"
 
 
 # Set up logging to ``/dev/null`` like a library is supposed to.

cloudbender/jinja.py

@@ -5,6 +5,8 @@ import re
 import base64
 import yaml
 import copy
+import subprocess
+import sys
 
 import jinja2
 from jinja2.utils import missing, object_type_repr
@@ -208,12 +210,16 @@ def read_config_file(path, variables={}):
 
     if path.exists():
         logger.debug("Reading config file: {}".format(path))
+
+        # First check for sops being present
         try:
             jenv = jinja2.Environment(
-                loader=jinja2.FileSystemLoader(str(path.parent)),
+                enable_async=True,
+                auto_reload=False,
+                loader=jinja2.FunctionLoader(_sops_loader),
                 undefined=jinja2.StrictUndefined,
                 extensions=['jinja2.ext.loopcontrols'])
-            template = jenv.get_template(path.name)
+            template = jenv.get_template(str(path))
             rendered_template = template.render(jinja_variables)
             data = yaml.safe_load(rendered_template)
             if data:
@@ -221,5 +227,31 @@ def read_config_file(path, variables={}):
 
         except Exception as e:
             logger.exception("Error reading config file: {} ({})".format(path, e))
+            sys.exit(1)
 
     return {}
+
+
+def _sops_loader(path):
+    """ Tries to loads yaml file
+        If "sops" key is detected the file is piped through sops before returned
+    """
+    with open(path, 'r') as f:
+        config_raw = f.read()
+        data = yaml.safe_load(config_raw)
+
+        if 'sops' in data:
+            try:
+                result = subprocess.run([
+                    'sops',
+                    '--input-type', 'yaml',
+                    '--output-type', 'yaml',
+                    '--decrypt', '/dev/stdin'
+                ], stdout=subprocess.PIPE, input=config_raw.encode('utf-8'))
+            except FileNotFoundError:
+                logger.exception("SOPS encrypted config {},  but unable to find sops binary! Try eg: https://github.com/mozilla/sops/releases/download/v3.5.0/sops-v3.5.0.linux".format(path))
+                sys.exit(1)
+
+            return result.stdout.decode('utf-8')
+        else:
+            return config_raw

cloudbender/stack.py

@@ -349,6 +349,11 @@ class Stack(object):
                 if p in self.parameters:
                     value = str(self.parameters[p])
                     self.cfn_parameters.append({'ParameterKey': p, 'ParameterValue': value})
+
+                    # Hide NoEcho parameters in shell output
+                    if 'NoEcho' in self.cfn_data['Parameters'][p] and self.cfn_data['Parameters'][p]['NoEcho']:
+                        value = '****'
+
                     logger.info('{} {} Parameter {}={}'.format(self.region, self.stackname, p, value))
                 else:
                     # If we have a Default defined in the CFN skip, as AWS will use it
@@ -385,7 +390,7 @@ class Stack(object):
 
         # Prepare parameters
         self.resolve_parameters()
-        self.write_parameter_file()
+        # self.write_parameter_file()
         self.read_template_file()
 
         logger.info('Creating {0} {1}'.format(self.region, self.stackname))
@@ -407,7 +412,7 @@ class Stack(object):
 
         # Prepare parameters
         self.resolve_parameters()
-        self.write_parameter_file()
+        # self.write_parameter_file()
         self.read_template_file()
 
         logger.info('Updating {0} {1}'.format(self.region, self.stackname))
@@ -446,7 +451,7 @@ class Stack(object):
 
         # Prepare parameters
         self.resolve_parameters()
-        self.write_parameter_file()
+        # self.write_parameter_file()
         self.read_template_file()
 
         logger.info('Creating change set {0} for stack {1}'.format(change_set_name, self.stackname))