Add retries for throttling events, add jinja loop extensions, minor bugfixes

This commit is contained in:
Stefan Reimer 2019-01-21 10:43:36 +00:00
parent 03cf3c943f
commit 25c375728a
5 changed files with 25 additions and 8 deletions

View File

@ -1,4 +1,5 @@
import os import os
import time
import boto3 import boto3
import botocore.session import botocore.session
@ -51,5 +52,16 @@ class BotoConnection():
def call(self, service, command, kwargs={}, profile=None, region=None): def call(self, service, command, kwargs={}, profile=None, region=None):
client = self._get_client(service, profile, region) while True:
return getattr(client, command)(**kwargs) try:
client = self._get_client(service, profile, region)
return getattr(client, command)(**kwargs)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'Throttling':
logger.warning("Throttling exception occured during {} - retry after 3s".format(command))
time.sleep(3)
pass
else:
raise e

View File

@ -21,6 +21,9 @@ class CloudBender(object):
"parameter_path": os.path.join(self.root, "parameters"), "parameter_path": os.path.join(self.root, "parameters"),
"artifact_paths": [os.path.join(self.root, "artifacts")] "artifact_paths": [os.path.join(self.root, "artifacts")]
} }
self.default_settings = {
'vars': { 'Mode': 'FortyTwo' }
}
if not os.path.isdir(self.root): if not os.path.isdir(self.root):
raise "Check '{0}' exists and is a valid project folder.".format(root_path) raise "Check '{0}' exists and is a valid project folder.".format(root_path)
@ -54,7 +57,7 @@ class CloudBender(object):
ensure_dir(self.ctx[k]) ensure_dir(self.ctx[k])
self.sg = StackGroup(self.ctx['config_path'], self.ctx) self.sg = StackGroup(self.ctx['config_path'], self.ctx)
self.sg.read_config() self.sg.read_config(self.default_settings)
self.all_stacks = self.sg.get_stacks() self.all_stacks = self.sg.get_stacks()

View File

@ -149,15 +149,17 @@ def pyminify(source, obfuscate=False, minify=True):
#source = pyminifier.obfuscate.apply_obfuscation(source) #source = pyminifier.obfuscate.apply_obfuscation(source)
source = pyminifier.token_utils.untokenize(tokens) source = pyminifier.token_utils.untokenize(tokens)
# logger.debug(source) #logger.info(source)
minified_source = pyminifier.compression.gz_pack(source) minified_source = pyminifier.compression.gz_pack(source)
logger.info("Compressed python code to {}".format(len(minified_source))) logger.info("Compressed python code to {}".format(len(minified_source)))
return minified_source return minified_source
def JinjaEnv(template_locations=[]): def JinjaEnv(template_locations=[]):
jenv = jinja2.Environment(trim_blocks=True, lstrip_blocks=True, undefined=jinja2.Undefined) jenv = jinja2.Environment(trim_blocks=True,
lstrip_blocks=True,
undefined=jinja2.Undefined,
extensions=['jinja2.ext.loopcontrols'])
jinja_loaders = [] jinja_loaders = []
for _dir in template_locations: for _dir in template_locations:

View File

@ -62,7 +62,7 @@ class Stack(object):
def read_config(self): def read_config(self):
_config = read_yaml_file(self.path) _config = read_yaml_file(self.path)
for p in ["stackname", "template", "dependencies", "default_lock", "multi_delete", "provides"]: for p in ["region", "stackname", "template", "dependencies", "default_lock", "multi_delete", "provides"]:
if p in _config: if p in _config:
setattr(self, p, _config[p]) setattr(self, p, _config[p])

View File

@ -56,7 +56,7 @@ class StackGroup(object):
profile = _config.get('profile', '') profile = _config.get('profile', '')
stackname_prefix = _config.get('stacknameprefix', '') stackname_prefix = _config.get('stacknameprefix', '')
logger.info("StackGroup {} added.".format(self.name)) logger.debug("StackGroup {} added.".format(self.name))
# Add stacks # Add stacks
stacks = [s for s in glob.glob(os.path.join(self.path, '*.yaml')) if not s.endswith("config.yaml")] stacks = [s for s in glob.glob(os.path.join(self.path, '*.yaml')) if not s.endswith("config.yaml")]