ci: remove aws cli pypi req

CHANGES.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## 0.9.10
+- Updates aws.lambda parser for Init Duration, incl. tests
+
+## 0.9.9
+- Improved error handling / descreased timouts calling AWS APIs to fail fast in case AWS throttles API calls
+  which causes each Lambda call to timout and run for 300s
+- Skip over more invalid vpc flowlog entries
+
+## 0.9.8
+- Fix for ALB AccessLog parser to handle spaces in request_url
+- Improved VPC FlowLog metadata augmentation
+- better error handling for VPC FlowLog parsing
+
+## 0.9.6
+- Augment VPC FlowLogs with ENI metadata incl. global cache
+
+## 0.9.5
+- Added support for VPC FlowLogs
+
 ## 0.9.4
 - Improved S3 file type detection, also handles one line access logs
 - default fluentd upstream url scheme

Makefile

@@ -22,7 +22,7 @@ build: $(PACKAGE_FILE)
 $(PACKAGE_FILE):
 	rm -rf dist && mkdir dist
 	cp -r index.py dist/
-	pip install --target dist --no-compile msgpack requests
+	pip install --isolated --target dist --no-compile msgpack requests
 	cd dist && zip -q -r $(PACKAGE) *
 
 upload: $(PACKAGE_FILE)

dev-requirements.txt

@@ -4,6 +4,6 @@ requests
 flake8
 pytest
 cfnlambda
-awscli
+#awscli
 #pytest-profiling
 #tuna

index.py

@@ -14,14 +14,21 @@ import io
 import urllib
 import datetime
 import boto3
+import botocore
 
 __author__ = "Stefan Reimer"
 __author_email__ = "stefan@zero-downtime.net"
-__version__ = "0.9.4"
+__version__ = "0.9.10"
 
-# Global alias lookup cache
+# IAM Alias lookup cache
 account_aliases = {}
 
+# ENI lookup cache
+enis = {}
+
+# IP lookup cache
+ips = {}
+
 logger = logging.getLogger(__name__)
 logging.getLogger("urllib3").setLevel(logging.WARNING)
 logging.getLogger('boto3').setLevel(logging.WARNING)
@@ -47,6 +54,7 @@ CHUNK_SIZE = 128
 DEBUG = boolean(os.getenv('DEBUG', default=False))
 TEST = boolean(os.getenv('TEST', default=False))
 RESOLVE_ACCOUNT = boolean(os.getenv('RESOLVE_ACCOUNT', default=True))
+ENHANCE_FLOWLOG = boolean(os.getenv('ENHANCE_FLOWLOG', default=True))
 
 if DEBUG:
     logging.getLogger().setLevel(logging.DEBUG)
@@ -77,22 +85,93 @@ def get_source(region, account_id):
     """ returns a new base source object
         resolves aws account_id to account alias and caches for lifetime of lambda function
     """
+    global RESOLVE_ACCOUNT
     source = {'account': account_id, 'region': region}
     if RESOLVE_ACCOUNT and not TEST:
         try:
             if account_id not in account_aliases:
-                iam = boto3.client('iam')
+                boto3_config = botocore.config.Config(retries=dict(max_attempts=2), connect_timeout=3, read_timeout=5)
+                iam = boto3.client('iam', config=boto3_config)
                 account_aliases[account_id] = iam.list_account_aliases()['AccountAliases'][0]
 
             source['account_alias'] = account_aliases[account_id]
 
-        except(KeyError, IndexError):
+        except(botocore.exceptions.ConnectTimeoutError, KeyError, IndexError):
-            logger.warning("Could not resolve IAM account alias")
+            logger.warning("Could not resolve IAM account alias, disabled for this session")
+            RESOLVE_ACCOUNT = False
             pass
 
     return source
 
 
+def add_flow_metadata(flow):
+    """ adds metadata to VPC flow: ENI, direction, type
+        caches the ENI and IP lookup tables for Lambda lifetime
+    """
+    global ENHANCE_FLOWLOG
+    if ENHANCE_FLOWLOG and not TEST:
+        try:
+            # Check cache and update if missed with all ENIs in one go
+            if flow['interface-id'] not in enis:
+                boto3_config = botocore.config.Config(retries=dict(max_attempts=2), connect_timeout=3, read_timeout=5)
+                ec2 = boto3.client('ec2', config=boto3_config)
+                interface_iter = ec2.get_paginator('describe_network_interfaces').paginate()
+                for response in interface_iter:
+                    for interface in response['NetworkInterfaces']:
+                        # Lookup table by ENI ID
+                        enis[interface['NetworkInterfaceId']] = interface
+
+                        # Lookup table by IP to classify traffic
+                        ips[interface['PrivateIpAddress']] = interface
+        except(botocore.exceptions.ConnectTimeoutError, KeyError, IndexError):
+            logger.warning("Error trying to get metadata for ENIs, disabling ENHANCE_FLOWLOG")
+            ENHANCE_FLOWLOG = False
+            return flow
+
+        try:
+            eni = enis[flow['interface-id']]
+            metadata = {'eni.az': eni['AvailabilityZone'],
+                        'eni.subnet': eni['SubnetId']}
+            remote_ip = None
+            if len(eni['Groups']):
+                metadata['eni.sg'] = eni['Groups'][0]['GroupName']
+
+            # Add PublicIP if attached
+            if 'Association' in eni and 'PublicIp' in eni['Association']:
+                metadata['eni.public_ip'] = eni['Association']['PublicIp']
+
+            # Determine traffic direction
+            if eni['PrivateIpAddress'] == flow['srcaddr']:
+                metadata['direction'] = 'Out'
+                remote_ip = flow['dstaddr']
+            elif eni['PrivateIpAddress'] == flow['dstaddr']:
+                metadata['direction'] = 'In'
+                remote_ip = flow['srcaddr']
+
+            # Try to classify traffic:
+            # Free,Regional,Out
+            if remote_ip:
+                if remote_ip in ips:
+                    if ips[remote_ip]['AvailabilityZone'] == eni['AvailabilityZone'] and ips[remote_ip]['VpcId'] == eni['VpcId']:
+                        metadata['traffic_class'] = 'Free'
+                    else:
+                        metadata['traffic_class'] = 'Regional'
+                else:
+                    # Incoming traffic is free 90% of times
+                    if metadata['direction'] == 'In':
+                        metadata['traffic_class'] = 'Free'
+                    else:
+                        metadata['traffic_class'] = 'Out'
+
+            flow.update(metadata)
+
+        except(KeyError, IndexError) as e:
+            logger.warning("Could not get additional data for ENI {} ({})".format(flow['interface-id'], e))
+            pass
+
+    return flow
+
+
 class Queue:
     url = urllib.parse.urlsplit(os.getenv('FLUENTD_URL', default=''), scheme='https')
     passwd = os.getenv('FLUENT_SHARED_KEY', default=None)
@@ -134,7 +213,7 @@ class Queue:
             _url = '{}/{}'.format(self.url.geturl(), self.tag)
             while True:
                 try:
-                    r = self.request.post(url=_url, data=msgpack.packb(self._queue), verify=self.verify_certs)
+                    r = self.request.post(url=_url, data=msgpack.packb(self._queue), verify=self.verify_certs, timeout=(6, 30))
                     if r:
                         break
                     else:
@@ -144,12 +223,11 @@ class Queue:
                     logger.warning("RequestException: {}".format(e))
                     pass
 
-                if retries >= 8:
+                if retries >= 2:
                     raise Exception("Error sending {} events to {}. Giving up.".format(events, _url))
 
                 retries = retries + 1
-                logger.warning("Error sending {} events to {}. Retrying in {} seconds.".format(events, _url, retries**2))
+                time.sleep(1)
-                time.sleep(retries**2)
         else:
             logger.debug("Test mode, dump only: {}".format(msgpack.packb(self._queue)))
 
@@ -186,6 +264,8 @@ def handler(event, context):
             logs = Queue("aws.cloudtrail")
         elif re.match("RDSOSMetrics", awsLogsData['logGroup']):
             logs = Queue("aws.rdsosmetrics")
+        elif re.match("vpcflowlog", awsLogsData['logGroup'], flags=re.IGNORECASE):
+            logs = Queue("aws.vpcflowlog")
         else:
             logs = Queue("aws.cloudwatch_logs")
 
@@ -212,7 +292,7 @@ def handler(event, context):
                 if mg:
                     parsed['RequestId'] = mg.group('request')
                     if mg.group('type') == 'REPORT':
-                        pattern = r'.*(?:\tDuration: (?P<duration>[\d\.\d]+) ms\s*)(?:\tBilled Duration: (?P<billed_duration>[\d\.\d]+) ms\s*)(?:\tMemory Size: (?P<memory_size>[\d\.\d]+) MB\s*)(?:\tMax Memory Used: (?P<max_memory_used>[\d\.\d]+) MB)'
+                        pattern = r'.*(?:\tDuration: (?P<duration>[\d\.\d]+) ms\s*)(?:\tBilled Duration: (?P<billed_duration>[\d\.\d]+) ms\s*)(?:\tMemory Size: (?P<memory_size>[\d\.\d]+) MB\s*)(?:\tMax Memory Used: (?P<max_memory_used>[\d\.\d]+) MB)(?:\tInit Duration: (?P<init_duration>[\d\.\d]+) ms\s*)?'
 
                     elif mg.group('type') == 'START':
                         pattern = r'.*(?:Version: (?P<version>.*))'
@@ -222,6 +302,7 @@ def handler(event, context):
 
                     data = re.match(pattern, e['message'])
                     for key in data.groupdict().keys():
+                        if data.group(key):
                             parsed[key] = data.group(key)
 
                     # All other info parsed, so just set type itself
@@ -275,6 +356,18 @@ def handler(event, context):
                 except (ValueError, TypeError, KeyError):
                     event['message'] = e['message']
 
+            # VPC FlowLog ?
+            # <version> <account-id> <interface-id> <srcaddr> <dstaddr> <srcport> <dstport> <protocol> <packets> <bytes> <start> <end> <action> <log-status>
+            elif logs.tag == 'aws.vpcflowlog':
+                row = e['message'].split(" ")
+
+                # Skip over NODATA,SKIPDATA entries, what would be the point having these in ES ?
+                if row[13] != 'OK':
+                    continue
+
+                parsed = add_flow_metadata({'interface-id': row[2], 'srcaddr': row[3], 'dstaddr': row[4], 'srcport': row[5], 'dstport': row[6], 'protocol': row[7],
+                                            'packets': row[8], 'bytes': row[9], 'start': row[10], 'end': row[11], 'action': row[12], 'log-status': row[13]})
+
             # Fallback add raw message
             else:
                 event['message'] = e['message']
@@ -313,7 +406,7 @@ def handler(event, context):
         source = get_source(region, account_id)
         source['s3_url'] = '{}/{}'.format(bucket, key)
 
-        alb_regex = re.compile(r"(?P<type>[^ ]*) (?P<timestamp>[^ ]*) (?P<elb>[^ ]*) (?P<client_ip>[^ ]*):(?P<client_port>[0-9]*) (?P<target_ip>[^ ]*)[:-](?P<target_port>[0-9]*) (?P<request_processing_time>[-.0-9]*) (?P<target_processing_time>[-.0-9]*) (?P<response_processing_time>[-.0-9]*) (?P<elb_status_code>|[-0-9]*) (?P<target_status_code>-|[-0-9]*) (?P<received_bytes>[-0-9]*) (?P<sent_bytes>[-0-9]*) \"(?P<request_verb>[^ ]*) (?P<request_url>[^ ]*) (?P<request_proto>- |[^ ]*)\" \"(?P<user_agent>[^\"]*)\" (?P<ssl_cipher>[A-Z0-9-]+) (?P<ssl_protocol>[A-Za-z0-9.-]*) (?P<target_group_arn>[^ ]*) \"(?P<trace_id>[^\"]*)\" \"(?P<domain_name>[^\"]*)\" \"(?P<chosen_cert_arn>[^\"]*)\" (?P<matched_rule_priority>[-.0-9]*) (?P<request_creation_time>[^ ]*) \"(?P<actions_executed>[^\"]*)\" \"(?P<redirect_url>[^ ]*)\" \"(?P<error_reason>[^ ]*)\"")
+        alb_regex = re.compile(r"(?P<type>[^ ]*) (?P<timestamp>[^ ]*) (?P<elb>[^ ]*) (?P<client_ip>[^ ]*):(?P<client_port>[0-9]*) (?P<target_ip>[^ ]*)[:-](?P<target_port>[0-9]*) (?P<request_processing_time>[-.0-9]*) (?P<target_processing_time>[-.0-9]*) (?P<response_processing_time>[-.0-9]*) (?P<elb_status_code>|[-0-9]*) (?P<target_status_code>-|[-0-9]*) (?P<received_bytes>[-0-9]*) (?P<sent_bytes>[-0-9]*) \"(?P<request_verb>[^ ]*) (?P<request_url>[^\"]*) (?P<request_proto>- |[^ ]*)\" \"(?P<user_agent>[^\"]*)\" (?P<ssl_cipher>[A-Z0-9-]+) (?P<ssl_protocol>[A-Za-z0-9.-]*) (?P<target_group_arn>[^ ]*) \"(?P<trace_id>[^\"]*)\" \"(?P<domain_name>[^\"]*)\" \"(?P<chosen_cert_arn>[^\"]*)\" (?P<matched_rule_priority>[-.0-9]*) (?P<request_creation_time>[^ ]*) \"(?P<actions_executed>[^\"]*)\" \"(?P<redirect_url>[^ ]*)\" \"(?P<error_reason>[^ ]*)\"")
 
         # try to identify file type by looking at first lines
         with gzip.open(file_path, mode='rt', newline='\n') as data:

test

@@ -0,0 +1,16 @@
+{
+  "messageType": "DATA_MESSAGE",
+  "owner": "123456789012",
+  "logGroup": "Cloudtrail/DefaultTrail",
+  "logStream": "123456789012_CloudTrail_eu-central-1",
+  "subscriptionFilters": [
+    "CloudBender_Mgmt"
+  ],
+  "logEvents": [
+    {
+      "id": "36010944938174877173576838392419674140970254593468989442",
+      "timestamp": 1614786618904,
+      "message": "{\"eventVersion\":\"1.08\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIVUV7DYO7JM46FRLW:AutoScaling\",\"arn\":\"arn:aws:sts::123456789012:assumed-role/AWSServiceRoleForAutoScaling/AutoScaling\",\"accountId\":\"123456789012\",\"sessionContext\":{\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIVUV7DYO7JM46FRLW\",\"arn\":\"arn:aws:iam::123456789012:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling\",\"accountId\":\"123456789012\",\"userName\":\"AWSServiceRoleForAutoScaling\"},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2021-03-03T15:18:15Z\"}},\"invokedBy\":\"autoscaling.amazonaws.com\"},\"eventTime\":\"2021-03-03T15:38:23Z\",\"eventSource\":\"ec2.amazonaws.com\",\"eventName\":\"DescribeInstanceStatus\",\"awsRegion\":\"eu-central-1\",\"sourceIPAddress\":\"autoscaling.amazonaws.com\",\"userAgent\":\"autoscaling.amazonaws.com\",\"requestParameters\":{\"instancesSet\":{\"items\":[{\"instanceId\":\"i-0fc26d9df63c21276\"},{\"instanceId\":\"i-01f635f7dd4af7d03\"},{\"instanceId\":\"i-0f5e6610de3ceb673\"}]},\"filterSet\":{},\"includeAllInstances\":true},\"responseElements\":null,\"requestID\":\"d89cde42-9a72-421a-af6b-e642ca862055\",\"eventID\":\"1dc68ed9-6879-4874-a286-179604a26747\",\"readOnly\":true,\"eventType\":\"AwsApiCall\",\"managementEvent\":true,\"eventCategory\":\"Management\",\"recipientAccountId\":\"123456789012\"}"
+    }
+  ]
+}

tests/test_parse.py

@@ -26,7 +26,11 @@ logging.getLogger("index").setLevel(logging.DEBUG)
 
 def test_parse():
     # Test Cloudwatch Logs
-    event = json.loads('{"awslogs": {"data": "H4sICILSAl0AA3Rlc3QArVNdb9tGEHzvr7gSfWgB09z7vuObHDOuCpsqKAZoawnGkTy5RChSJY9N0iD/vSslLew2BlykLwSxu9idmZt5H+39NLl7X747+CiNLhfl4u4mW68XV1l0Fg1vej9imTIupNLGAmVY7ob7q3GYD9hJ3Jsp6dy+alwS/BTi3dzXoR36j2PrMHq3xzkG1CbAE8qT22+uF2W2LresrirHgVPhjagtOLPbWa00B23BUIkrprma6rE9HDe+bLvgxylKb6MX3TA3F75v/Hh3c78P0fZ0Lfvd9+E48D5qGzzKhWIMJAjJubVCAWgtlJLGaMmUEUpaC0IwARKPWsus5oIKgYdDi8IEt0eOVEomQQmOy8zZX4Lh+kVdD3Mf2iYlDwXa9NGHsy9DIJ+JoHiVkx9WF2SVk/L7IltcEiqAa4VfvCE4I7fhV3yBJj4hTYne/g/o1DPR3S7zl6vtJhzfPgYeU15SmwqdUnXOmP4FO05UNegq9sL6WNQUYsu9iiuEpAWvtdF6E9zhcN4Nrrn76DOyG4c9mXiaJBOv5vq1D8lD/R/7MKmHftfef5a2lpodoQOj1OJBJqnmilrKjZW4EJSmqAhVlINU/AnauMM+op0Vxar4m7c68dYpQCr5OQdA3sY606D1YwM7HgvXNLFzsIsVBTBm58EJ5F1k19linZHTvpRs+vnQuODJ6DvvJj8ln35Ii0F4m276cnS1r1z9mny7H6aAgzUGgtSu60jnpvDdvzWQQA1XqJwSRmlJgTF6RMHQEJYpqpg1EsCiKFZrgCc0MALgUTSuj278maBB82V+RRY5WebrcpG/yL4m2U/LEoufexIJDP3LjeRSCkXx+Q1DhdHOGAoNGijnWgqBdmQY1KdygnAePUmWX5LC/zbj4BLT+izbfTm6p3LyT3RF9uOqKP8zwHA5j+5ocZynhp5LRfbTJly0Xecb8qDJALBDNuHG74fxHVm3f3gs4/zNBRbdW/Kp8WryeNzqU/0owPbDV38Ce70icCIGAAA=" } }')
+    event = json.loads('{"awslogs": {"data": "H4sIAGPPdF4AA+VUTW/bRhC991dshR5awBRnv3cJ9CDHjKvClgOJAdpahrAkVwoRilTJZRI3yH/v0HYCu7ULF82tF4KYGcx78+bNfpzsfd+7nc+uD36STE5m2Wxznq5Ws9N0cjRp3ze+wzBlXEiljQXKMFy3u9OuHQ6Yid37Pq7dPi9dHHwfou3QFKFqm9uyVei822MdA2pj4DHl8eV3Z7MsXWVXrMhzx4FT4Y0oLDiz3VqtNAdtwVCJLfoh74uuOowdX1Z18F0/SS4nL+p2KI99U/puc77bh8nVDVr6zjdhLPg4qUoE5UIxBhKE5NxaoQC0FkpJY7RkygglrQUhmACJoNYyq7mgQiBwqFCY4PY4I5WSSVCCYzNz9FkwbD8rinZoQlUm5L5A62by6ei/MZDPZLB8vSA/XxyTiwXJflqmsxNCBXCt8IsYgjNyGd7gBsrohmlC9NVXYKeeye5yvnh5cbUO4+4j4BHlGbWJ0AlVU8b0b5hxIi9A55EX1keioBBZ7lWUIyUteKGN1uvgDodp3bpyc+szsu3aPel5Esc9z4firQ/xff0f+jAu2mZb7R4dW0vNRurAKLUIyCTVXFFLubESG4LSFBWhinKQij8xNvawD8ZOl8uL5Ze51c3cOgFIJJ9yAJzbWGdKtH5kYMsj4coycg62kaIAxmw9OIFzL9OzdLZKyU2/hKyb4VC64Enna+9638d3P6TCQ/iQrJusc4XPXfGWfL9v+4CFBR4EKVxdk9r14Ye/ayCBGq5QOSWM0pICY3RkwdAQlimqmDUSwKIoVmuAJzQwAuDBaZyNbvyVoEEX88UpmS3IfLHKZosX6bck/WWeYfCxlUhg6F9uJJdSKGq0MgwVRjvjUWjQQDnXUgi0I8NDfepOkI593IkMRicyyChNuEnATjlXuBEnvIdtbqKSexmJ3KMTDeURyBHQCMkKvg69D5vO97575zd7v2+7axJawoSawvkxKYauQ73razL0VbMjKOPUYnw31m7euK4kPxKYwudI327DXeQZSuivokS6OCFL//uAhXN8t551gM/Z0z+ze+rF+Cu7ZfrqYpn9a4LhZOjceOxYTw2dSkX2/TocV3XtS3IvyQAwQ9bh/HZ5q+oPj2GsPz/GoPtA7hKve4/gVt/E//cCzJsq3OtCrUKIsdGozNWnb/4EFS89L0YIAAA=" } }')
+    index.handler(event, context)
+
+    # {"messageType":"DATA_MESSAGE","owner":"123456789012","logGroup":"vpcflowlog","logStream":"eni-123","subscriptionFilters":["CloudBender_Mgmt"],"logEvents":[{"id":"34622050453399460077466588752684659904424057309929734144","timestamp":1552506436228,"message":"2 747836459185 eni-033856bc201b3773b 10.10.0.227 10.3.9.111 443 52866 6 2 135 1563806853 1563806911 ACCEPT OK"},{"id":"34622050453399460077466588752684659904424057309929734144","timestamp":1552506436228,"message":"2 747836459185 eni-033856bc201b3773b 10.10.9.48 10.10.0.227 24224 17234 6 10 1947 1563827256 1563827316 ACCEPT OK"}]}
+    event = json.loads('{"awslogs": {"data": "H4sIAHN1N10AA82R3UoDMRCF732KkOu6JPOXpHe1rr2QotDeiUh/Ylno7pbdrSLFd3eqFPQNhFwczkzOzMecbJ37frXLy49DtmN7O1lOXublYjGZlXZk2/cmd2p7QGIJMTkPau/b3axrjwetvB02r/v2XZ0ffzF0eVVrITfVtX5Ttz+u+01XHYaqbe6q/ZC73o6f7HTfHrc3udnm7mW+qwf7/B1QvuVmODecbLXVHCQBcOyIEVMicS4EEuEYA4NEEk7JEQE5DuhSghSQPJEOHiqFG1a17umZgZ0QalgcXaA1HkygEFGIk49szls7xMiy3oDzawwB18a7Qp8rAMJZY5EK770hQsMQRYwYMB7ZeBaMTiLjRSbtm0yn5ePSPNzbz9G/o0oFxT+AQABkfNCTK5d3xicKPzgQgOUi0ctvsufPqy8JYO/9TQIAAA==" } }')
     index.handler(event, context)
 
     # Cloudfront Access Logs via S3
@@ -37,6 +41,10 @@ def test_parse():
     event = json.loads('{ "Records": [ { "eventVersion": "2.0", "eventTime": "1970-01-01T00:00:00.000Z", "requestParameters": { "sourceIPAddress": "127.0.0.1" }, "s3": { "configurationId": "testConfigRule", "object": { "eTag": "0123456789abcdef0123456789abcdef", "sequencer": "0A1B2C3D4E5F678901", "key": "tests/test_alb_accesslogs.gz", "size": 1024 }, "bucket": { "arn": "arn:aws:s3:::mybucket", "name": "file://", "ownerIdentity": { "principalId": "EXAMPLE" } }, "s3SchemaVersion": "1.0" }, "responseElements": { "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH", "x-amz-request-id": "EXAMPLE123456789" }, "awsRegion": "us-east-1", "eventName": "ObjectCreated:Put", "userIdentity": { "principalId": "EXAMPLE" }, "eventSource": "aws:s3" } ] }')
     index.handler(event, context)
 
+    # cloudtrail incl. _keyed
+    event = json.loads('{"awslogs": {"data": "H4sIAAAAAAAAA5VU227jNhB972foOUpEXajLmzZOFi6aZhG5WXTXi4AmRy5RSnJJKt40yL93KCpZw82mW8CQIc2ZOXMOh/MYdGAM28LqYQdBFSzqVX13ddE09fuL4CQY9j1o/EziJM1oXpQRifGzGrbv9TDuMHKuhlFYzaQ6W0DLRmVX7sWDGquBdUf5d1PKhLqDMeTQY7oKCaaYcWO4ljsrh/5SKgvaBNVnz/EOegH67mrb2eDLVP3iHlMd4DGQAkkSGpGoTNMyKUieFnlO8iTLaZEUSRmnpKR5StKozKM4S7MySWlRFgh3gqxEGyzrUBGhJM0LSgk2m54824PlH9cBOMZb7Ar7WwfVOiCnUbEOTtbBaEAvBUalfcAIYi0aOmFqY8YOxM2gYILutOy53DG1FD5+c10vb3+7zRe/X+c/X6X08uaXj1U92qHhTMl+O2Ux7Rnxv2J7UxlrqurQ1op5nlAj0Vn9sWlA30sOjvdy0Af1zv5Vm/Nh7O3cz2HRKWzQAhR8PvQWvlqvbv62RE7QR4L/n9JX1UnWHambVGEoNF6Wl8lQifFKTlnH/h56hJzyoXvTgB8R7Q70V9bNR/hmsSeE72GzFJeAE8rc8C6YZc4VF2LWarkZLRjvU9cyTP/DzQpnFjx/y5TxpnG8MnMJTx5HMQmjBH8rklWkqEj2CUldadnfD3+CePfgzfueGb7FaXhXsnutalJUcfJp4p9gzTBq7oHA4+Nqz6gXfxbgru0Glj1eop5DY5kdjbd5b25g+3xfDu+7n62JaPmhFkLjSP2Xjvlg6i0W+QGshr9GvNcfmMZO3TbxJyDnNk0D8zhLC50Lfj6IzpMhw6jlMRWlaGnCYxLn1Pn5KpAgJGtzIVKGzyj5LrDNADdMJCDhsKG5A35xZ9ROS29uy58wV6OAWqlnb12bVo/wNOkzu6E3cKGgc7sQQ/2o1Dfly8XEKIqSC0jjsGR5HKYxYSFr6SYEmsacFTSOsuzbsc5JRHBagChDWuRliPs0DVlc0JDkJY1SFuM6zWeXmbju1cPc2MuovSzAval38pwpNcE71uNCdf1OC/wo6xzHfjtoP9FXL9CZCLeJxLf6jbv75Kz86R9rVP9k2AYAAA==" } }')
+    index.handler(event, context)
+
     # unknown file
     event = json.loads('{ "Records": [ { "eventVersion": "2.0", "eventTime": "1970-01-01T00:00:00.000Z", "requestParameters": { "sourceIPAddress": "127.0.0.1" }, "s3": { "configurationId": "testConfigRule", "object": { "eTag": "0123456789abcdef0123456789abcdef", "sequencer": "0A1B2C3D4E5F678901", "key": "tests/test_s3_unknown.gz", "size": 1024 }, "bucket": { "arn": "arn:aws:s3:::mybucket", "name": "file://", "ownerIdentity": { "principalId": "EXAMPLE" } }, "s3SchemaVersion": "1.0" }, "responseElements": { "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH", "x-amz-request-id": "EXAMPLE123456789" }, "awsRegion": "us-east-1", "eventName": "ObjectCreated:Put", "userIdentity": { "principalId": "EXAMPLE" }, "eventSource": "aws:s3" } ] }')
     index.handler(event, context)