Compare commits
No commits in common. "c04e5e87569ec2c237b58a1ab4d627343b8a562b" and "1e12bd7807d15c662eb1e33f26f294d1941e266d" have entirely different histories.
c04e5e8756
...
1e12bd7807
19
CHANGES.md
19
CHANGES.md
@ -1,24 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## 0.9.10
|
||||
- Updates aws.lambda parser for Init Duration, incl. tests
|
||||
|
||||
## 0.9.9
|
||||
- Improved error handling / descreased timouts calling AWS APIs to fail fast in case AWS throttles API calls
|
||||
which causes each Lambda call to timout and run for 300s
|
||||
- Skip over more invalid vpc flowlog entries
|
||||
|
||||
## 0.9.8
|
||||
- Fix for ALB AccessLog parser to handle spaces in request_url
|
||||
- Improved VPC FlowLog metadata augmentation
|
||||
- better error handling for VPC FlowLog parsing
|
||||
|
||||
## 0.9.6
|
||||
- Augment VPC FlowLogs with ENI metadata incl. global cache
|
||||
|
||||
## 0.9.5
|
||||
- Added support for VPC FlowLogs
|
||||
|
||||
## 0.9.4
|
||||
- Improved S3 file type detection, also handles one line access logs
|
||||
- default fluentd upstream url scheme
|
||||
|
||||
2
Makefile
2
Makefile
@ -22,7 +22,7 @@ build: $(PACKAGE_FILE)
|
||||
$(PACKAGE_FILE):
|
||||
rm -rf dist && mkdir dist
|
||||
cp -r index.py dist/
|
||||
pip install --isolated --target dist --no-compile msgpack requests
|
||||
pip install --target dist --no-compile msgpack requests
|
||||
cd dist && zip -q -r $(PACKAGE) *
|
||||
|
||||
upload: $(PACKAGE_FILE)
|
||||
|
||||
@ -4,6 +4,6 @@ requests
|
||||
flake8
|
||||
pytest
|
||||
cfnlambda
|
||||
#awscli
|
||||
awscli
|
||||
#pytest-profiling
|
||||
#tuna
|
||||
|
||||
117
index.py
117
index.py
@ -14,21 +14,14 @@ import io
|
||||
import urllib
|
||||
import datetime
|
||||
import boto3
|
||||
import botocore
|
||||
|
||||
__author__ = "Stefan Reimer"
|
||||
__author_email__ = "stefan@zero-downtime.net"
|
||||
__version__ = "0.9.10"
|
||||
__version__ = "0.9.4"
|
||||
|
||||
# IAM Alias lookup cache
|
||||
# Global alias lookup cache
|
||||
account_aliases = {}
|
||||
|
||||
# ENI lookup cache
|
||||
enis = {}
|
||||
|
||||
# IP lookup cache
|
||||
ips = {}
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||||
logging.getLogger('boto3').setLevel(logging.WARNING)
|
||||
@ -54,7 +47,6 @@ CHUNK_SIZE = 128
|
||||
DEBUG = boolean(os.getenv('DEBUG', default=False))
|
||||
TEST = boolean(os.getenv('TEST', default=False))
|
||||
RESOLVE_ACCOUNT = boolean(os.getenv('RESOLVE_ACCOUNT', default=True))
|
||||
ENHANCE_FLOWLOG = boolean(os.getenv('ENHANCE_FLOWLOG', default=True))
|
||||
|
||||
if DEBUG:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
@ -85,93 +77,22 @@ def get_source(region, account_id):
|
||||
""" returns a new base source object
|
||||
resolves aws account_id to account alias and caches for lifetime of lambda function
|
||||
"""
|
||||
global RESOLVE_ACCOUNT
|
||||
source = {'account': account_id, 'region': region}
|
||||
if RESOLVE_ACCOUNT and not TEST:
|
||||
try:
|
||||
if account_id not in account_aliases:
|
||||
boto3_config = botocore.config.Config(retries=dict(max_attempts=2), connect_timeout=3, read_timeout=5)
|
||||
iam = boto3.client('iam', config=boto3_config)
|
||||
iam = boto3.client('iam')
|
||||
account_aliases[account_id] = iam.list_account_aliases()['AccountAliases'][0]
|
||||
|
||||
source['account_alias'] = account_aliases[account_id]
|
||||
|
||||
except(botocore.exceptions.ConnectTimeoutError, KeyError, IndexError):
|
||||
logger.warning("Could not resolve IAM account alias, disabled for this session")
|
||||
RESOLVE_ACCOUNT = False
|
||||
except(KeyError, IndexError):
|
||||
logger.warning("Could not resolve IAM account alias")
|
||||
pass
|
||||
|
||||
return source
|
||||
|
||||
|
||||
def add_flow_metadata(flow):
|
||||
""" adds metadata to VPC flow: ENI, direction, type
|
||||
caches the ENI and IP lookup tables for Lambda lifetime
|
||||
"""
|
||||
global ENHANCE_FLOWLOG
|
||||
if ENHANCE_FLOWLOG and not TEST:
|
||||
try:
|
||||
# Check cache and update if missed with all ENIs in one go
|
||||
if flow['interface-id'] not in enis:
|
||||
boto3_config = botocore.config.Config(retries=dict(max_attempts=2), connect_timeout=3, read_timeout=5)
|
||||
ec2 = boto3.client('ec2', config=boto3_config)
|
||||
interface_iter = ec2.get_paginator('describe_network_interfaces').paginate()
|
||||
for response in interface_iter:
|
||||
for interface in response['NetworkInterfaces']:
|
||||
# Lookup table by ENI ID
|
||||
enis[interface['NetworkInterfaceId']] = interface
|
||||
|
||||
# Lookup table by IP to classify traffic
|
||||
ips[interface['PrivateIpAddress']] = interface
|
||||
except(botocore.exceptions.ConnectTimeoutError, KeyError, IndexError):
|
||||
logger.warning("Error trying to get metadata for ENIs, disabling ENHANCE_FLOWLOG")
|
||||
ENHANCE_FLOWLOG = False
|
||||
return flow
|
||||
|
||||
try:
|
||||
eni = enis[flow['interface-id']]
|
||||
metadata = {'eni.az': eni['AvailabilityZone'],
|
||||
'eni.subnet': eni['SubnetId']}
|
||||
remote_ip = None
|
||||
if len(eni['Groups']):
|
||||
metadata['eni.sg'] = eni['Groups'][0]['GroupName']
|
||||
|
||||
# Add PublicIP if attached
|
||||
if 'Association' in eni and 'PublicIp' in eni['Association']:
|
||||
metadata['eni.public_ip'] = eni['Association']['PublicIp']
|
||||
|
||||
# Determine traffic direction
|
||||
if eni['PrivateIpAddress'] == flow['srcaddr']:
|
||||
metadata['direction'] = 'Out'
|
||||
remote_ip = flow['dstaddr']
|
||||
elif eni['PrivateIpAddress'] == flow['dstaddr']:
|
||||
metadata['direction'] = 'In'
|
||||
remote_ip = flow['srcaddr']
|
||||
|
||||
# Try to classify traffic:
|
||||
# Free,Regional,Out
|
||||
if remote_ip:
|
||||
if remote_ip in ips:
|
||||
if ips[remote_ip]['AvailabilityZone'] == eni['AvailabilityZone'] and ips[remote_ip]['VpcId'] == eni['VpcId']:
|
||||
metadata['traffic_class'] = 'Free'
|
||||
else:
|
||||
metadata['traffic_class'] = 'Regional'
|
||||
else:
|
||||
# Incoming traffic is free 90% of times
|
||||
if metadata['direction'] == 'In':
|
||||
metadata['traffic_class'] = 'Free'
|
||||
else:
|
||||
metadata['traffic_class'] = 'Out'
|
||||
|
||||
flow.update(metadata)
|
||||
|
||||
except(KeyError, IndexError) as e:
|
||||
logger.warning("Could not get additional data for ENI {} ({})".format(flow['interface-id'], e))
|
||||
pass
|
||||
|
||||
return flow
|
||||
|
||||
|
||||
class Queue:
|
||||
url = urllib.parse.urlsplit(os.getenv('FLUENTD_URL', default=''), scheme='https')
|
||||
passwd = os.getenv('FLUENT_SHARED_KEY', default=None)
|
||||
@ -213,7 +134,7 @@ class Queue:
|
||||
_url = '{}/{}'.format(self.url.geturl(), self.tag)
|
||||
while True:
|
||||
try:
|
||||
r = self.request.post(url=_url, data=msgpack.packb(self._queue), verify=self.verify_certs, timeout=(6, 30))
|
||||
r = self.request.post(url=_url, data=msgpack.packb(self._queue), verify=self.verify_certs)
|
||||
if r:
|
||||
break
|
||||
else:
|
||||
@ -223,11 +144,12 @@ class Queue:
|
||||
logger.warning("RequestException: {}".format(e))
|
||||
pass
|
||||
|
||||
if retries >= 2:
|
||||
if retries >= 8:
|
||||
raise Exception("Error sending {} events to {}. Giving up.".format(events, _url))
|
||||
|
||||
retries = retries + 1
|
||||
time.sleep(1)
|
||||
logger.warning("Error sending {} events to {}. Retrying in {} seconds.".format(events, _url, retries**2))
|
||||
time.sleep(retries**2)
|
||||
else:
|
||||
logger.debug("Test mode, dump only: {}".format(msgpack.packb(self._queue)))
|
||||
|
||||
@ -264,8 +186,6 @@ def handler(event, context):
|
||||
logs = Queue("aws.cloudtrail")
|
||||
elif re.match("RDSOSMetrics", awsLogsData['logGroup']):
|
||||
logs = Queue("aws.rdsosmetrics")
|
||||
elif re.match("vpcflowlog", awsLogsData['logGroup'], flags=re.IGNORECASE):
|
||||
logs = Queue("aws.vpcflowlog")
|
||||
else:
|
||||
logs = Queue("aws.cloudwatch_logs")
|
||||
|
||||
@ -292,7 +212,7 @@ def handler(event, context):
|
||||
if mg:
|
||||
parsed['RequestId'] = mg.group('request')
|
||||
if mg.group('type') == 'REPORT':
|
||||
pattern = r'.*(?:\tDuration: (?P<duration>[\d\.\d]+) ms\s*)(?:\tBilled Duration: (?P<billed_duration>[\d\.\d]+) ms\s*)(?:\tMemory Size: (?P<memory_size>[\d\.\d]+) MB\s*)(?:\tMax Memory Used: (?P<max_memory_used>[\d\.\d]+) MB)(?:\tInit Duration: (?P<init_duration>[\d\.\d]+) ms\s*)?'
|
||||
pattern = r'.*(?:\tDuration: (?P<duration>[\d\.\d]+) ms\s*)(?:\tBilled Duration: (?P<billed_duration>[\d\.\d]+) ms\s*)(?:\tMemory Size: (?P<memory_size>[\d\.\d]+) MB\s*)(?:\tMax Memory Used: (?P<max_memory_used>[\d\.\d]+) MB)'
|
||||
|
||||
elif mg.group('type') == 'START':
|
||||
pattern = r'.*(?:Version: (?P<version>.*))'
|
||||
@ -302,8 +222,7 @@ def handler(event, context):
|
||||
|
||||
data = re.match(pattern, e['message'])
|
||||
for key in data.groupdict().keys():
|
||||
if data.group(key):
|
||||
parsed[key] = data.group(key)
|
||||
parsed[key] = data.group(key)
|
||||
|
||||
# All other info parsed, so just set type itself
|
||||
event['message'] = mg.group('type')
|
||||
@ -356,18 +275,6 @@ def handler(event, context):
|
||||
except (ValueError, TypeError, KeyError):
|
||||
event['message'] = e['message']
|
||||
|
||||
# VPC FlowLog ?
|
||||
# <version> <account-id> <interface-id> <srcaddr> <dstaddr> <srcport> <dstport> <protocol> <packets> <bytes> <start> <end> <action> <log-status>
|
||||
elif logs.tag == 'aws.vpcflowlog':
|
||||
row = e['message'].split(" ")
|
||||
|
||||
# Skip over NODATA,SKIPDATA entries, what would be the point having these in ES ?
|
||||
if row[13] != 'OK':
|
||||
continue
|
||||
|
||||
parsed = add_flow_metadata({'interface-id': row[2], 'srcaddr': row[3], 'dstaddr': row[4], 'srcport': row[5], 'dstport': row[6], 'protocol': row[7],
|
||||
'packets': row[8], 'bytes': row[9], 'start': row[10], 'end': row[11], 'action': row[12], 'log-status': row[13]})
|
||||
|
||||
# Fallback add raw message
|
||||
else:
|
||||
event['message'] = e['message']
|
||||
@ -406,7 +313,7 @@ def handler(event, context):
|
||||
source = get_source(region, account_id)
|
||||
source['s3_url'] = '{}/{}'.format(bucket, key)
|
||||
|
||||
alb_regex = re.compile(r"(?P<type>[^ ]*) (?P<timestamp>[^ ]*) (?P<elb>[^ ]*) (?P<client_ip>[^ ]*):(?P<client_port>[0-9]*) (?P<target_ip>[^ ]*)[:-](?P<target_port>[0-9]*) (?P<request_processing_time>[-.0-9]*) (?P<target_processing_time>[-.0-9]*) (?P<response_processing_time>[-.0-9]*) (?P<elb_status_code>|[-0-9]*) (?P<target_status_code>-|[-0-9]*) (?P<received_bytes>[-0-9]*) (?P<sent_bytes>[-0-9]*) \"(?P<request_verb>[^ ]*) (?P<request_url>[^\"]*) (?P<request_proto>- |[^ ]*)\" \"(?P<user_agent>[^\"]*)\" (?P<ssl_cipher>[A-Z0-9-]+) (?P<ssl_protocol>[A-Za-z0-9.-]*) (?P<target_group_arn>[^ ]*) \"(?P<trace_id>[^\"]*)\" \"(?P<domain_name>[^\"]*)\" \"(?P<chosen_cert_arn>[^\"]*)\" (?P<matched_rule_priority>[-.0-9]*) (?P<request_creation_time>[^ ]*) \"(?P<actions_executed>[^\"]*)\" \"(?P<redirect_url>[^ ]*)\" \"(?P<error_reason>[^ ]*)\"")
|
||||
alb_regex = re.compile(r"(?P<type>[^ ]*) (?P<timestamp>[^ ]*) (?P<elb>[^ ]*) (?P<client_ip>[^ ]*):(?P<client_port>[0-9]*) (?P<target_ip>[^ ]*)[:-](?P<target_port>[0-9]*) (?P<request_processing_time>[-.0-9]*) (?P<target_processing_time>[-.0-9]*) (?P<response_processing_time>[-.0-9]*) (?P<elb_status_code>|[-0-9]*) (?P<target_status_code>-|[-0-9]*) (?P<received_bytes>[-0-9]*) (?P<sent_bytes>[-0-9]*) \"(?P<request_verb>[^ ]*) (?P<request_url>[^ ]*) (?P<request_proto>- |[^ ]*)\" \"(?P<user_agent>[^\"]*)\" (?P<ssl_cipher>[A-Z0-9-]+) (?P<ssl_protocol>[A-Za-z0-9.-]*) (?P<target_group_arn>[^ ]*) \"(?P<trace_id>[^\"]*)\" \"(?P<domain_name>[^\"]*)\" \"(?P<chosen_cert_arn>[^\"]*)\" (?P<matched_rule_priority>[-.0-9]*) (?P<request_creation_time>[^ ]*) \"(?P<actions_executed>[^\"]*)\" \"(?P<redirect_url>[^ ]*)\" \"(?P<error_reason>[^ ]*)\"")
|
||||
|
||||
# try to identify file type by looking at first lines
|
||||
with gzip.open(file_path, mode='rt', newline='\n') as data:
|
||||
|
||||
16
test
16
test
@ -1,16 +0,0 @@
|
||||
{
|
||||
"messageType": "DATA_MESSAGE",
|
||||
"owner": "123456789012",
|
||||
"logGroup": "Cloudtrail/DefaultTrail",
|
||||
"logStream": "123456789012_CloudTrail_eu-central-1",
|
||||
"subscriptionFilters": [
|
||||
"CloudBender_Mgmt"
|
||||
],
|
||||
"logEvents": [
|
||||
{
|
||||
"id": "36010944938174877173576838392419674140970254593468989442",
|
||||
"timestamp": 1614786618904,
|
||||
"message": "{\"eventVersion\":\"1.08\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIVUV7DYO7JM46FRLW:AutoScaling\",\"arn\":\"arn:aws:sts::123456789012:assumed-role/AWSServiceRoleForAutoScaling/AutoScaling\",\"accountId\":\"123456789012\",\"sessionContext\":{\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIVUV7DYO7JM46FRLW\",\"arn\":\"arn:aws:iam::123456789012:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling\",\"accountId\":\"123456789012\",\"userName\":\"AWSServiceRoleForAutoScaling\"},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2021-03-03T15:18:15Z\"}},\"invokedBy\":\"autoscaling.amazonaws.com\"},\"eventTime\":\"2021-03-03T15:38:23Z\",\"eventSource\":\"ec2.amazonaws.com\",\"eventName\":\"DescribeInstanceStatus\",\"awsRegion\":\"eu-central-1\",\"sourceIPAddress\":\"autoscaling.amazonaws.com\",\"userAgent\":\"autoscaling.amazonaws.com\",\"requestParameters\":{\"instancesSet\":{\"items\":[{\"instanceId\":\"i-0fc26d9df63c21276\"},{\"instanceId\":\"i-01f635f7dd4af7d03\"},{\"instanceId\":\"i-0f5e6610de3ceb673\"}]},\"filterSet\":{},\"includeAllInstances\":true},\"responseElements\":null,\"requestID\":\"d89cde42-9a72-421a-af6b-e642ca862055\",\"eventID\":\"1dc68ed9-6879-4874-a286-179604a26747\",\"readOnly\":true,\"eventType\":\"AwsApiCall\",\"managementEvent\":true,\"eventCategory\":\"Management\",\"recipientAccountId\":\"123456789012\"}"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -26,11 +26,7 @@ logging.getLogger("index").setLevel(logging.DEBUG)
|
||||
|
||||
def test_parse():
|
||||
# Test Cloudwatch Logs
|
||||
event = json.loads('{"awslogs": {"data": "H4sIAGPPdF4AA+VUTW/bRhC991dshR5awBRnv3cJ9CDHjKvClgOJAdpahrAkVwoRilTJZRI3yH/v0HYCu7ULF82tF4KYGcx78+bNfpzsfd+7nc+uD36STE5m2Wxznq5Ws9N0cjRp3ze+wzBlXEiljQXKMFy3u9OuHQ6Yid37Pq7dPi9dHHwfou3QFKFqm9uyVei822MdA2pj4DHl8eV3Z7MsXWVXrMhzx4FT4Y0oLDiz3VqtNAdtwVCJLfoh74uuOowdX1Z18F0/SS4nL+p2KI99U/puc77bh8nVDVr6zjdhLPg4qUoE5UIxBhKE5NxaoQC0FkpJY7RkygglrQUhmACJoNYyq7mgQiBwqFCY4PY4I5WSSVCCYzNz9FkwbD8rinZoQlUm5L5A62by6ei/MZDPZLB8vSA/XxyTiwXJflqmsxNCBXCt8IsYgjNyGd7gBsrohmlC9NVXYKeeye5yvnh5cbUO4+4j4BHlGbWJ0AlVU8b0b5hxIi9A55EX1keioBBZ7lWUIyUteKGN1uvgDodp3bpyc+szsu3aPel5Esc9z4firQ/xff0f+jAu2mZb7R4dW0vNRurAKLUIyCTVXFFLubESG4LSFBWhinKQij8xNvawD8ZOl8uL5Ze51c3cOgFIJJ9yAJzbWGdKtH5kYMsj4coycg62kaIAxmw9OIFzL9OzdLZKyU2/hKyb4VC64Enna+9638d3P6TCQ/iQrJusc4XPXfGWfL9v+4CFBR4EKVxdk9r14Ye/ayCBGq5QOSWM0pICY3RkwdAQlimqmDUSwKIoVmuAJzQwAuDBaZyNbvyVoEEX88UpmS3IfLHKZosX6bck/WWeYfCxlUhg6F9uJJdSKGq0MgwVRjvjUWjQQDnXUgi0I8NDfepOkI593IkMRicyyChNuEnATjlXuBEnvIdtbqKSexmJ3KMTDeURyBHQCMkKvg69D5vO97575zd7v2+7axJawoSawvkxKYauQ73razL0VbMjKOPUYnw31m7euK4kPxKYwudI327DXeQZSuivokS6OCFL//uAhXN8t551gM/Z0z+ze+rF+Cu7ZfrqYpn9a4LhZOjceOxYTw2dSkX2/TocV3XtS3IvyQAwQ9bh/HZ5q+oPj2GsPz/GoPtA7hKve4/gVt/E//cCzJsq3OtCrUKIsdGozNWnb/4EFS89L0YIAAA=" } }')
|
||||
index.handler(event, context)
|
||||
|
||||
# {"messageType":"DATA_MESSAGE","owner":"123456789012","logGroup":"vpcflowlog","logStream":"eni-123","subscriptionFilters":["CloudBender_Mgmt"],"logEvents":[{"id":"34622050453399460077466588752684659904424057309929734144","timestamp":1552506436228,"message":"2 747836459185 eni-033856bc201b3773b 10.10.0.227 10.3.9.111 443 52866 6 2 135 1563806853 1563806911 ACCEPT OK"},{"id":"34622050453399460077466588752684659904424057309929734144","timestamp":1552506436228,"message":"2 747836459185 eni-033856bc201b3773b 10.10.9.48 10.10.0.227 24224 17234 6 10 1947 1563827256 1563827316 ACCEPT OK"}]}
|
||||
event = json.loads('{"awslogs": {"data": "H4sIAHN1N10AA82R3UoDMRCF732KkOu6JPOXpHe1rr2QotDeiUh/Ylno7pbdrSLFd3eqFPQNhFwczkzOzMecbJ37frXLy49DtmN7O1lOXublYjGZlXZk2/cmd2p7QGIJMTkPau/b3axrjwetvB02r/v2XZ0ffzF0eVVrITfVtX5Ttz+u+01XHYaqbe6q/ZC73o6f7HTfHrc3udnm7mW+qwf7/B1QvuVmODecbLXVHCQBcOyIEVMicS4EEuEYA4NEEk7JEQE5DuhSghSQPJEOHiqFG1a17umZgZ0QalgcXaA1HkygEFGIk49szls7xMiy3oDzawwB18a7Qp8rAMJZY5EK770hQsMQRYwYMB7ZeBaMTiLjRSbtm0yn5ePSPNzbz9G/o0oFxT+AQABkfNCTK5d3xicKPzgQgOUi0ctvsufPqy8JYO/9TQIAAA==" } }')
|
||||
event = json.loads('{"awslogs": {"data": "H4sICILSAl0AA3Rlc3QArVNdb9tGEHzvr7gSfWgB09z7vuObHDOuCpsqKAZoawnGkTy5RChSJY9N0iD/vSslLew2BlykLwSxu9idmZt5H+39NLl7X747+CiNLhfl4u4mW68XV1l0Fg1vej9imTIupNLGAmVY7ob7q3GYD9hJ3Jsp6dy+alwS/BTi3dzXoR36j2PrMHq3xzkG1CbAE8qT22+uF2W2LresrirHgVPhjagtOLPbWa00B23BUIkrprma6rE9HDe+bLvgxylKb6MX3TA3F75v/Hh3c78P0fZ0Lfvd9+E48D5qGzzKhWIMJAjJubVCAWgtlJLGaMmUEUpaC0IwARKPWsus5oIKgYdDi8IEt0eOVEomQQmOy8zZX4Lh+kVdD3Mf2iYlDwXa9NGHsy9DIJ+JoHiVkx9WF2SVk/L7IltcEiqAa4VfvCE4I7fhV3yBJj4hTYne/g/o1DPR3S7zl6vtJhzfPgYeU15SmwqdUnXOmP4FO05UNegq9sL6WNQUYsu9iiuEpAWvtdF6E9zhcN4Nrrn76DOyG4c9mXiaJBOv5vq1D8lD/R/7MKmHftfef5a2lpodoQOj1OJBJqnmilrKjZW4EJSmqAhVlINU/AnauMM+op0Vxar4m7c68dYpQCr5OQdA3sY606D1YwM7HgvXNLFzsIsVBTBm58EJ5F1k19linZHTvpRs+vnQuODJ6DvvJj8ln35Ii0F4m276cnS1r1z9mny7H6aAgzUGgtSu60jnpvDdvzWQQA1XqJwSRmlJgTF6RMHQEJYpqpg1EsCiKFZrgCc0MALgUTSuj278maBB82V+RRY5WebrcpG/yL4m2U/LEoufexIJDP3LjeRSCkXx+Q1DhdHOGAoNGijnWgqBdmQY1KdygnAePUmWX5LC/zbj4BLT+izbfTm6p3LyT3RF9uOqKP8zwHA5j+5ocZynhp5LRfbTJly0Xecb8qDJALBDNuHG74fxHVm3f3gs4/zNBRbdW/Kp8WryeNzqU/0owPbDV38Ce70icCIGAAA=" } }')
|
||||
index.handler(event, context)
|
||||
|
||||
# Cloudfront Access Logs via S3
|
||||
@ -41,10 +37,6 @@ def test_parse():
|
||||
event = json.loads('{ "Records": [ { "eventVersion": "2.0", "eventTime": "1970-01-01T00:00:00.000Z", "requestParameters": { "sourceIPAddress": "127.0.0.1" }, "s3": { "configurationId": "testConfigRule", "object": { "eTag": "0123456789abcdef0123456789abcdef", "sequencer": "0A1B2C3D4E5F678901", "key": "tests/test_alb_accesslogs.gz", "size": 1024 }, "bucket": { "arn": "arn:aws:s3:::mybucket", "name": "file://", "ownerIdentity": { "principalId": "EXAMPLE" } }, "s3SchemaVersion": "1.0" }, "responseElements": { "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH", "x-amz-request-id": "EXAMPLE123456789" }, "awsRegion": "us-east-1", "eventName": "ObjectCreated:Put", "userIdentity": { "principalId": "EXAMPLE" }, "eventSource": "aws:s3" } ] }')
|
||||
index.handler(event, context)
|
||||
|
||||
# cloudtrail incl. _keyed
|
||||
event = json.loads('{"awslogs": {"data": "H4sIAAAAAAAAA5VU227jNhB972foOUpEXajLmzZOFi6aZhG5WXTXi4AmRy5RSnJJKt40yL93KCpZw82mW8CQIc2ZOXMOh/MYdGAM28LqYQdBFSzqVX13ddE09fuL4CQY9j1o/EziJM1oXpQRifGzGrbv9TDuMHKuhlFYzaQ6W0DLRmVX7sWDGquBdUf5d1PKhLqDMeTQY7oKCaaYcWO4ljsrh/5SKgvaBNVnz/EOegH67mrb2eDLVP3iHlMd4DGQAkkSGpGoTNMyKUieFnlO8iTLaZEUSRmnpKR5StKozKM4S7MySWlRFgh3gqxEGyzrUBGhJM0LSgk2m54824PlH9cBOMZb7Ar7WwfVOiCnUbEOTtbBaEAvBUalfcAIYi0aOmFqY8YOxM2gYILutOy53DG1FD5+c10vb3+7zRe/X+c/X6X08uaXj1U92qHhTMl+O2Ux7Rnxv2J7UxlrqurQ1op5nlAj0Vn9sWlA30sOjvdy0Af1zv5Vm/Nh7O3cz2HRKWzQAhR8PvQWvlqvbv62RE7QR4L/n9JX1UnWHambVGEoNF6Wl8lQifFKTlnH/h56hJzyoXvTgB8R7Q70V9bNR/hmsSeE72GzFJeAE8rc8C6YZc4VF2LWarkZLRjvU9cyTP/DzQpnFjx/y5TxpnG8MnMJTx5HMQmjBH8rklWkqEj2CUldadnfD3+CePfgzfueGb7FaXhXsnutalJUcfJp4p9gzTBq7oHA4+Nqz6gXfxbgru0Glj1eop5DY5kdjbd5b25g+3xfDu+7n62JaPmhFkLjSP2Xjvlg6i0W+QGshr9GvNcfmMZO3TbxJyDnNk0D8zhLC50Lfj6IzpMhw6jlMRWlaGnCYxLn1Pn5KpAgJGtzIVKGzyj5LrDNADdMJCDhsKG5A35xZ9ROS29uy58wV6OAWqlnb12bVo/wNOkzu6E3cKGgc7sQQ/2o1Dfly8XEKIqSC0jjsGR5HKYxYSFr6SYEmsacFTSOsuzbsc5JRHBagChDWuRliPs0DVlc0JDkJY1SFuM6zWeXmbju1cPc2MuovSzAval38pwpNcE71uNCdf1OC/wo6xzHfjtoP9FXL9CZCLeJxLf6jbv75Kz86R9rVP9k2AYAAA==" } }')
|
||||
index.handler(event, context)
|
||||
|
||||
# unknown file
|
||||
event = json.loads('{ "Records": [ { "eventVersion": "2.0", "eventTime": "1970-01-01T00:00:00.000Z", "requestParameters": { "sourceIPAddress": "127.0.0.1" }, "s3": { "configurationId": "testConfigRule", "object": { "eTag": "0123456789abcdef0123456789abcdef", "sequencer": "0A1B2C3D4E5F678901", "key": "tests/test_s3_unknown.gz", "size": 1024 }, "bucket": { "arn": "arn:aws:s3:::mybucket", "name": "file://", "ownerIdentity": { "principalId": "EXAMPLE" } }, "s3SchemaVersion": "1.0" }, "responseElements": { "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH", "x-amz-request-id": "EXAMPLE123456789" }, "awsRegion": "us-east-1", "eventName": "ObjectCreated:Put", "userIdentity": { "principalId": "EXAMPLE" }, "eventSource": "aws:s3" } ] }')
|
||||
index.handler(event, context)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user