sns-alert-hub/tests/test_aws-lambda-rie.py

119 lines
18 KiB
Python
Executable File

#!/usr/bin/env python
import json
import subprocess
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
s = requests.Session()
retries = Retry(
total=3,
backoff_factor=1,
status_forcelist=[
502,
503,
504],
allowed_methods="POST")
s.mount("http://", HTTPAdapter(max_retries=retries))
class Test:
@classmethod
def setup_class(cls):
cls.p = subprocess.Popen(
"./aws-lambda-rie python -m awslambdaric app.handler", shell=True
)
@classmethod
def teardown_class(cls):
cls.p.terminate()
def send_event(self, event):
self.p.poll()
assert self.p.returncode is None, "aws-lambda-rie not running"
s.post(
"http://localhost:8080/2015-03-31/functions/function/invocations",
json=event,
)
# Cloudwatch Alarm
def test_cloudwatch_alarm(self):
event = json.loads(
r' { "Records": [ { "EventSource": "aws:sns", "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:us-west-2:123456789012:AlertHub:626fe212-7605-47f5-85ed-d77e68f448b9", "Sns": { "Type": "Notification", "MessageId": "a45aa76b-1a4e-55fc-9c26-fdfa56068892", "TopicArn": "arn:aws:sns:us-west-2:123456789012:AlertHub", "Subject": "ALARM: \"example-bastion-CPUCreditsAlarm-18414G67CHF2I\" in US West (Oregon)", "Message": "{\"AlarmName\":\"example-bastion-CPUCreditsAlarm-18414G67CHF2I\",\"AlarmDescription\":\"At least one instance/member of example-bastion.Asg in us-west-2 starts to consume surplus CPUCredits. Additional charges might incur after 24h! <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/burstable-performance-instances-monitoring-cpu-credits.html|(AWS Docs)>\",\"AWSAccountId\":\"123456789012\",\"AlarmConfigurationUpdatedTimestamp\":\"2022-02-24T09:03:10.154+0000\",\"NewStateValue\":\"ALARM\",\"NewStateReason\":\"Threshold Crossed: 1 out of the last 1 datapoints [0.0 (24/02/22 08:59:00)] was less than the threshold (5.0) (minimum 1 datapoint for OK -> ALARM transition).\",\"StateChangeTime\":\"2022-02-24T09:04:08.879+0000\",\"Region\":\"US West (Oregon)\",\"AlarmArn\":\"arn:aws:cloudwatch:us-west-2:123456789012:alarm:example-bastion-CPUCreditsAlarm-18414G67CHF2I\",\"OldStateValue\":\"OK\",\"Trigger\":{\"MetricName\":\"CPUSurplusCreditBalance\",\"Namespace\":\"AWS/EC2\",\"StatisticType\":\"Statistic\",\"Statistic\":\"SUM\",\"Unit\":null,\"Dimensions\":[{\"value\":\"example-bastion-BastionAsg-1K4RJ9QCQF2P1\",\"name\":\"AutoScalingGroupName\"}],\"Period\":300,\"EvaluationPeriods\":1,\"DatapointsToAlarm\":1,\"ComparisonOperator\":\"LessThanThreshold\",\"Threshold\":5.0,\"TreatMissingData\":\"notBreaching\",\"EvaluateLowSampleCountPercentile\":\"\"}}", "Timestamp": "2022-02-24T09:04:08.941Z", "SignatureVersion": "1", "Signature": "eESZxAoYVNuaNd6yxFqrIRTi7vRR9+JFUiTkifp6GmWPr+TwwS85y3kBZqsQXsN9935cIC+S0POdrF2s0raVcBcZ+Ggzi6SE3PAcd+uZfV5zgCOHZGA5AABCsa5fHmmJvqNgDoJQFXTcDriBBoNCbtHhbPPN4O9W1uXFmg+jYNox3kWycgGDvsbsrS4kA7sPwqxJAtQSkY72jvCAnCa9M66hIgX3qR/RcUSSv9BQrELOpbuVAQ40l0LSwB8n2rPeNDNc95YbRLXSRCAAKM2Gd3+l0G4mQkoTozoPA752mIef5tz3bp8MdM4lKG4pjUuPRDKj6N+yuwmTiyq0mYDvSw==", "SigningCertUrl": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-7ff5318490ec183fbaddaa2a969abfda.pem", "UnsubscribeUrl": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-west-2:123456789012:AlertHub:626fe212-7605-47f5-85ed-d77e68f448b9", "MessageAttributes": {} } } ] }'
)
self.send_event(event)
# CloudBender launch event
def test_cloudbender(self):
event = json.loads(
r' { "Records": [ { "EventSource": "aws:sns", "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub:0c04d2e7-32ec-4933-b913-84c7172e6d90", "Sns": { "Type": "Notification", "MessageId": "25bb2fd0-3221-5c07-aea1-76acf75017c3", "TopicArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub", "Subject": null, "Message": "{\n \"Source\": \"CloudBender\",\n \"AWSAccountId\": \"123456789012\",\n \"Region\": \"eu-central-1\",\n \"Artifact\": \"bastion\",\n \"Hostname\": \"bastion\",\n \"Asg\": \"zdt-bastion-BastionAsg-UABA2PIZV4TI\",\n \"Instance\": \"i-0d9f08855cdfcd740\",\n \"Subject\": \"Test\",\n \"Level\": \"Info\",\n \"Message\": \"Hello\",\n \"Attachment\": \"\",\n \"Emoji\": \"\"\n}\n", "Timestamp": "2021-07-29T15:03:13.318Z", "SignatureVersion": "1", "Signature": "OhLoCz8RWazyZ+ZHK03QLby6M3jmdtZvLWoNFygAHIaljQ0ZHsd6mc4TskDTnqpUpCtd/iIl+TLIPN8hYyflbLk2/cN3LXXWcSQ0GWqQ/bWIEhej54oCmUgZjIzrVfRlgz7mlUkhRnjQoRWYpcXRycQczMWuF2DCeIDP6v3ON53BxR8NdCeQXiquwoFlHaAaZIviRoUMqwp2Cl1T0NaBLeL9zmsdPvJF6EaXRbu3rqC1hdrA6E+nV2lzYNKg09POxA9JVpURmMEq3AC4tXm1Gu73PWQgWgoDSOQx+SOjMrbMeCAqf5R6typBV1BRDsGPkNkt9n4oto0FR9iyDmuWog==", "SigningCertUrl": "https://sns.eu-central-1.amazonaws.com/SimpleNotificationService-010a507c1833636cd94bdb98bd93083a.pem", "UnsubscribeUrl": "https://sns.eu-central-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-central-1:123456789012:AlertHub:0c04d2e7-32ec-4933-b913-84c7172e6d90", "MessageAttributes": {} } } ] }'
)
self.send_event(event)
# Alertmanager
def test_alertmanager(self):
event = json.loads(
r' { "Records": [ { "EventSource": "aws:sns", "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "Sns": { "Type": "Notification", "MessageId": "10ae86eb-9ddc-5c2f-806c-df6ecb6bde42", "TopicArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub", "Subject": null, "Message": "{\"receiver\":\"alerthub-notifications\",\"status\":\"firing\",\"alerts\":[{\"status\":\"firing\",\"labels\":{\"alertname\":\"KubeVersionMismatch\",\"awsRegion\":\"eu-central-1\",\"clusterName\":\"test\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"severity\":\"warning\"},\"annotations\":{\"description\":\"There are 2 different semantic versions of Kubernetes components running.\",\"runbook_url\":\"https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch\",\"summary\":\"Different semantic versions of Kubernetes components running.\"},\"startsAt\":\"2021-08-04T13:17:40.31Z\",\"endsAt\":\"0001-01-01T00:00:00Z\",\"generatorURL\":\"https://prometheus/graph?g0.expr=count%28count+by%28git_version%29+%28label_replace%28kubernetes_build_info%7Bjob%21~%22kube-dns%7Ccoredns%22%7D%2C+%22git_version%22%2C+%22%241%22%2C+%22git_version%22%2C+%22%28v%5B0-9%5D%2A.%5B0-9%5D%2A%29.%2A%22%29%29%29+%3E+1\\u0026g0.tab=1\",\"fingerprint\":\"5f94d4a22730c666\"}],\"groupLabels\":{},\"commonLabels\":{\"alertname\":\"KubeVersionMismatch\",\"awsRegion\":\"eu-central-1\",\"clusterName\":\"test\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"severity\":\"warning\"},\"commonAnnotations\":{\"description\":\"There are 2 different semantic versions of Kubernetes components running.\",\"runbook_url\":\"https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch\",\"summary\":\"Different semantic versions of Kubernetes components running.\"},\"externalURL\":\"https://alertmanager\",\"version\":\"4\",\"groupKey\":\"{}:{}\",\"truncatedAlerts\":0}\n", "Timestamp": "2021-08-05T03:01:11.233Z", "SignatureVersion": "1", "Signature": "pSUYO7LDIfzCbBrp/S2HXV3/yzls3vfYy+2di6HsKG8Mf+CV97RLnen15ieAo3eKA8IfviZIzyREasbF0cwfUeruHPbW1B8kO572fDyV206zmUxvR63r6oM6OyLv9XKBmvyYHKawkOgHZHEMP3v1wMIIHK2W5KbJtXoUcks5DVamooVb9iFF58uqTf+Ccy31bOL4tFyMR9nr8NU55vEIlGEVno8A9Q21TujdZTg0V0BmRgPafcic96udWungjmfhZ005378N32u2hlLj6BRneTpHHSXHBw4wKZreKpX+INZwiZ4P8hzVfgRvAIh/4gXN9+0UJSHgnsaqUcLDNoLZTQ==", "SigningCertUrl": "https://sns.eu-central-1.amazonaws.com/SimpleNotificationService-010a507c1833636cd94bdb98bd93083a.pem", "UnsubscribeUrl": "https://sns.eu-central-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "MessageAttributes": {} } } ] }'
)
self.send_event(event)
# Alertmanager Multiple
def test_alertmanager_multi(self):
event = json.loads(
r' { "Records": [ { "EventSource": "aws:sns", "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "Sns": { "Type": "Notification", "MessageId": "10ae86eb-9ddc-5c2f-806c-df6ecb6bde42", "TopicArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub", "Subject": null, "Message": "{\"receiver\":\"alerthub-notifications\",\"status\":\"resolved\",\"alerts\":[{\"status\":\"resolved\",\"labels\":{\"alertname\":\"KubeDeploymentReplicasMismatch\",\"awsAccount\":\"123456789012\",\"awsRegion\":\"us-west-2\",\"clusterName\":\"test-cluster\",\"container\":\"kube-state-metrics\",\"deployment\":\"example-job\",\"endpoint\":\"http\",\"instance\":\"10.244.202.71:8080\",\"job\":\"kube-state-metrics\",\"namespace\":\"default\",\"pod\":\"metrics-kube-state-metrics-56546f44c7-h57jx\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"service\":\"metrics-kube-state-metrics\",\"severity\":\"warning\"},\"annotations\":{\"description\":\"Deployment default/example-job has not matched the expected number of replicas for longer than 15 minutes.\",\"runbook_url\":\"https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch\",\"summary\":\"Deployment has not matched the expected number of replicas.\"},\"startsAt\":\"2021-09-29T12:36:11.394Z\",\"endsAt\":\"2021-09-29T14:51:11.394Z\",\"generatorURL\":\"https://prometheus.dev.example.com/graph?g0.expr=%28kube_deployment_spec_replicas%7Bjob%3D%22kube-state-metrics%22%7D+%3E+kube_deployment_status_replicas_available%7Bjob%3D%22kube-state-metrics%22%7D%29+and+%28changes%28kube_deployment_status_replicas_updated%7Bjob%3D%22kube-state-metrics%22%7D%5B10m%5D%29+%3D%3D+0%29\\u0026g0.tab=1\",\"fingerprint\":\"59ad2f1a4567b43b\"},{\"status\":\"firing\",\"labels\":{\"alertname\":\"KubeVersionMismatch\",\"awsRegion\":\"eu-central-1\",\"clusterName\":\"test\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"severity\":\"warning\"},\"annotations\":{\"description\":\"There are 2 different semantic versions of Kubernetes components running.\",\"runbook_url\":\"https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch\",\"summary\":\"Different semantic versions of Kubernetes components running.\"},\"startsAt\":\"2021-08-04T13:17:40.31Z\",\"endsAt\":\"0001-01-01T00:00:00Z\",\"generatorURL\":\"https://prometheus/graph?g0.expr=count%28count+by%28git_version%29+%28label_replace%28kubernetes_build_info%7Bjob%21~%22kube-dns%7Ccoredns%22%7D%2C+%22git_version%22%2C+%22%241%22%2C+%22git_version%22%2C+%22%28v%5B0-9%5D%2A.%5B0-9%5D%2A%29.%2A%22%29%29%29+%3E+1\\u0026g0.tab=1\",\"fingerprint\":\"5f94d4a22730c666\"}],\"groupLabels\":{\"job\":\"kube-state-metrics\"},\"commonLabels\":{\"alertname\":\"KubeDeploymentReplicasMismatch\",\"awsAccount\":\"123456789012\",\"awsRegion\":\"us-west-2\",\"clusterName\":\"test-cluster\",\"container\":\"kube-state-metrics\",\"deployment\":\"example-job\",\"endpoint\":\"http\",\"instance\":\"10.244.202.71:8080\",\"job\":\"kube-state-metrics\",\"namespace\":\"default\",\"pod\":\"metrics-kube-state-metrics-56546f44c7-h57jx\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"service\":\"metrics-kube-state-metrics\",\"severity\":\"warning\"},\"commonAnnotations\":{\"description\":\"Deployment default/example-job has not matched the expected number of replicas for longer than 15 minutes.\",\"runbook_url\":\"https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch\",\"summary\":\"Deployment has not matched the expected number of replicas.\"},\"externalURL\":\"https://alertmanager.dev.example.com\",\"version\":\"4\",\"groupKey\":\"{}:{job=\\\"kube-state-metrics\\\"}\",\"truncatedAlerts\":0}\n", "Timestamp": "2021-08-05T03:01:11.233Z", "SignatureVersion": "1", "Signature": "pSUYO7LDIfzCbBrp/S2HXV3/yzls3vfYy+2di6HsKG8Mf+CV97RLnen15ieAo3eKA8IfviZIzyREasbF0cwfUeruHPbW1B8kO572fDyV206zmUxvR63r6oM6OyLv9XKBmvyYHKawkOgHZHEMP3v1wMIIHK2W5KbJtXoUcks5DVamooVb9iFF58uqTf+Ccy31bOL4tFyMR9nr8NU55vEIlGEVno8A9Q21TujdZTg0V0BmRgPafcic96udWungjmfhZ005378N32u2hlLj6BRneTpHHSXHBw4wKZreKpX+INZwiZ4P8hzVfgRvAIh/4gXN9+0UJSHgnsaqUcLDNoLZTQ==", "SigningCertUrl": "https://sns.eu-central-1.amazonaws.com/SimpleNotificationService-010a507c1833636cd94bdb98bd93083a.pem", "UnsubscribeUrl": "https://sns.eu-central-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "MessageAttributes": {} } } ] }'
)
self.send_event(event)
# ElastiCache snaphshot
def test_elasticache_snapshot(self):
event = json.loads(
r' {"Records": [{"EventSource": "aws:sns", "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "Sns": {"Type": "Notification", "MessageId": "10ae86eb-9ddc-5c2f-806c-df6ecb6bde42", "TopicArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub", "Subject": null, "Message": "{\"ElastiCache:SnapshotComplete\":\"redis-prod-0002-001\"}" }}]}'
)
self.send_event(event)
# RDS
def test_rds_event(self):
event = json.loads(
r''' {
"Records": [
{
"EventSource": "aws:sns",
"EventVersion": "1.0",
"EventSubscriptionArn": "arn:aws:sns:us-west-2:123456789012:AlertHub:63470449-620d-44ce-971f-ad9582804b13",
"Sns": {
"Type": "Notification",
"MessageId": "ef1f821c-a04f-5c5c-9dff-df498532069b",
"TopicArn": "arn:aws:sns:us-west-2:123456789012:AlertHub",
"Subject": "RDS Notification Message",
"Message": "{\"Event Source\":\"db-cluster-snapshot\",\"Event Time\":\"2023-08-15 07:03:24.491\",\"Identifier Link\":\"https://console.aws.amazon.com/rds/home?region=us-west-2#snapshot:engine=aurora;id=rds:projectdb-cluster-2023-08-15-07-03\",\"Source ID\":\"rds:projectdb-cluster-2023-08-15-07-03\",\"Source ARN\":\"arn:aws:rds:us-west-2:123456789012:cluster-snapshot:rds:projectdb-cluster-2023-08-15-07-03\",\"Event ID\":\"http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_Events.Messages.html#RDS-EVENT-0168\",\"Event Message\":\"Creating automated cluster snapshot\",\"Tags\":{}}",
"Timestamp": "2023-08-15T07:03:25.289Z",
"SignatureVersion": "1",
"Signature": "mRtx+ddS1uzF3alGDWnDtUkAz+Gno8iuv0wPwkeBJPe1LAcKTXVteYhQdP2BB5ZunPlWXPSDsNtFl8Eh6v4/fcdukxH/czc6itqgGiciQ3DCICLvOJrvrVVgsVvHgOA/Euh8wryzxeQ3HJ/nmF9sg/PtuKyxvGxyO7NSFJrRKkqwkuG1Wr/8gcN3nrenqNTzKiC16kzVuKISWgXM1jqbsleQ4MyBcjq61LRwODKB8tc8vJ6PLGOs4Lrc3qeruCqF3Tzpl43680RsaRBBn1SLycwFVdB1kpHSXuk+YJQ6BS7s6rbMoyhPOpSCFHMZXC/eEb09wTzgpop0KDE/koiUsg==",
"SigningCertUrl": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-01d088a6f77103d0fe307c0069e40ed6.pem",
"UnsubscribeUrl": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-west-2:123456789012:AlertHub:63470449-620d-44ce-971f-ad9582804b13",
"MessageAttributes": {
"Resource": {
"Type": "String",
"Value": "arn:aws:rds:us-west-2:123456789012:cluster-snapshot:rds:projectdb-cluster-2023-08-15-07-03"
},
"EventID": {
"Type": "String",
"Value": "RDS-EVENT-0168"
}
}
}
}
]
}
'''
)
self.send_event(event)
def test_asg(self):
event = json.loads(
r' {"Records": [{"EventSource": "aws:sns", "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "Sns": {"Type": "Notification", "MessageId": "10ae86eb-9ddc-5c2f-806c-df6ecb6bde42", "TopicArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub", "Subject": null, "Message": "{\"Origin\":\"AutoScalingGroup\",\"Destination\":\"EC2\",\"Progress\":50,\"AccountId\":\"123456789012\",\"Description\":\"Terminating EC2 instance: i-023ca42b188ffd91d\",\"RequestId\":\"1764cac3-224b-46bf-8bed-407a5b868e63\",\"EndTime\":\"2023-05-15T08:51:16.195Z\",\"AutoScalingGroupARN\":\"arn:aws:autoscaling:us-west-2:123456789012:autoScalingGroup:4a4fb6e3-22b4-487b-8335-3904f02ff9fd:autoScalingGroupName/powerbi\",\"ActivityId\":\"1764cac3-224b-46bf-8bed-407a5b868e63\",\"StartTime\":\"2023-05-15T08:50:14.145Z\",\"Service\":\"AWS Auto Scaling\",\"Time\":\"2023-05-15T08:51:16.195Z\",\"EC2InstanceId\":\"i-023ca42b188ffd91d\",\"StatusCode\":\"InProgress\",\"StatusMessage\":\"\",\"Details\":{\"Subnet ID\":\"subnet-fe2d6189\",\"Availability Zone\":\"us-west-2a\"},\"AutoScalingGroupName\":\"powerbi\",\"Cause\":\"At 2023-05-15T08:50:03Z the scheduled action end executed. Setting min size from 1 to 0. Setting desired capacity from 1 to 0. At 2023-05-15T08:50:03Z a scheduled action update of AutoScalingGroup constraints to min: 0, max: 1, desired: 0 changing the desired capacity from 1 to 0. At 2023-05-15T08:50:13Z an instance was taken out of service in response to a difference between desired and actual capacity, shrinking the capacity from 1 to 0. At 2023-05-15T08:50:14Z instance i-023ca42b188ffd91d was selected for termination.\",\"Event\":\"autoscaling:EC2_INSTANCE_TERMINATE\"}" }}]}'
)
self.send_event(event)