feat: Initial commit v0.7.2

Dockerfile

@@ -0,0 +1,55 @@
+# https://aws.amazon.com/blogs/aws/new-for-aws-lambda-container-image-support/
+ARG FUNCTION_DIR="/app"
+ARG RUNTIME_VERSION="3.9"
+ARG DISTRO_VERSION="3.15"
+
+# Stage 1 - bundle base image + runtime
+# Grab a fresh copy of the image and install GCC
+FROM python:${RUNTIME_VERSION}-alpine${DISTRO_VERSION} AS python-alpine
+# Install GCC (Alpine uses musl but we compile and link dependencies with GCC)
+RUN apk add --no-cache \
+    libstdc++
+
+# Stage 2 - build function and dependencies
+FROM python-alpine AS build-image
+# Install aws-lambda-cpp build dependencies
+RUN apk add --no-cache \
+    build-base \
+    libtool \
+    autoconf \
+    automake \
+    libexecinfo-dev \
+    make \
+    cmake \
+    libcurl \
+    libffi-dev \
+    openssl-dev
+    # cargo
+
+# Include global args in this stage of the build
+ARG FUNCTION_DIR
+ARG RUNTIME_VERSION
+# Create function directory
+RUN mkdir -p ${FUNCTION_DIR}
+# Copy handler function
+COPY app/* ${FUNCTION_DIR}
+
+# Install requirements
+COPY requirements.txt requirements.txt
+RUN python${RUNTIME_VERSION} -m pip install -r requirements.txt --target ${FUNCTION_DIR}
+
+# Install Lambda Runtime Interface Client for Python
+RUN python${RUNTIME_VERSION} -m pip install awslambdaric --target ${FUNCTION_DIR}
+
+# Stage 3 - final runtime image
+# Grab a fresh copy of the Python image
+FROM python-alpine
+# Include global arg in this stage of the build
+ARG FUNCTION_DIR
+# Set working directory to function root directory
+WORKDIR ${FUNCTION_DIR}
+# Copy in the built dependencies
+COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR}
+
+ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ]
+CMD [ "app.handler" ]

Jenkinsfile

@@ -0,0 +1,56 @@
+pipeline {
+  agent { node { label 'podman-aws-trivy' } }
+
+  stages {
+    stage('Prepare'){
+        // get tags
+        steps {
+            sh 'git fetch -q --tags ${GIT_URL} +refs/heads/${BRANCH_NAME}:refs/remotes/origin/${BRANCH_NAME}'
+        }
+    }
+
+    // Build using rootless podman
+    stage('Build'){
+        steps {
+            sh 'make build'
+        }
+    }
+
+    stage('Test'){
+        steps {
+            sh 'make test'
+        }
+    }
+
+    // Scan via trivy
+    stage('Scan'){
+        environment { 
+            TRIVY_FORMAT = "template"
+            TRIVY_OUTPUT = "reports/trivy.html"
+        }
+        steps {
+            sh 'mkdir -p reports'
+            sh 'make scan'
+            publishHTML target : [
+                allowMissing: true,
+                alwaysLinkToLastBuild: true,
+                keepAll: true,
+                reportDir: 'reports',
+                reportFiles: 'trivy.html',
+                reportName: 'TrivyScan',
+                reportTitles: 'TrivyScan'
+            ]
+
+            // Scan again and fail on CRITICAL vulns
+            sh 'TRIVY_EXIT_CODE=1 TRIVY_SEVERITY=CRITICAL make scan'
+        }
+    }
+
+    // Push to ECR
+    stage('Push'){
+        steps {
+            sh 'make push'
+        }
+    }
+  }
+}

Makefile

@@ -0,0 +1,44 @@
+REGISTRY := public.ecr.aws/zero-downtime    
+REPOSITORY := sns-alert-hub
+REGION := us-east-1    
+    
+# Parse version from latest git semver tag    
+GTAG=$(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)    
+TAG ?= $(shell echo $(GTAG) | awk -F '-' '{ print $$1 "-" $$2 }' | sed -e 's/-$$//')    
+    
+ifeq ($(TRIVY_REMOTE),)    
+  TRIVY_OPTS := image    
+else    
+  TRIVY_OPTS := client --remote ${TRIVY_REMOTE}    
+endif    
+    
+.PHONY: build push scan test    
+    
+all: build    
+    
+build:    
+	podman build --rm --squash-all -t $(REPOSITORY):$(TAG) -t $(REPOSITORY):latest .    
+    
+push:    
+	aws ecr-public get-login-password --region $(REGION) | podman login --username AWS --password-stdin $(REGISTRY)    
+	podman tag $(REPOSITORY):latest $(REGISTRY)/$(REPOSITORY):$(TAG) $(REGISTRY)/$(REPOSITORY):latest    
+	podman push $(REGISTRY)/$(REPOSITORY):$(TAG)    
+	podman push $(REGISTRY)/$(REPOSITORY):latest    
+	# Delete all untagged images    
+	# aws ecr-public batch-delete-image --repository-name $(REPOSITORY) --region $(REGION) --image-ids $$(for image in $$(aws ecr-public describe-images --repository-name $(REPOSITORY) --region $(REGION) --output json | jq -r '.imageDetails[] | select(.imageTags | not ).imageDigest'); do echo -n "imageDigest=$$image "; done)
+    
+scan:    
+	trivy $(TRIVY_OPTS) $(REPOSITORY):$(TAG)    
+    
+test:    
+	flake8 --ignore=E501 app/app.py
+	./test.py
+
+aws_lambda:
+	@[ -z $(AWS_LAMBDA) ] && { echo "Missing aws-lambda-rie in PATH"; exit 1;} || true
+
+run: aws_lambda
+	podman run --rm -v $(AWS_LAMBDA):/aws-lambda -p 8080:8080 --entrypoint /aws-lambda/aws-lambda-rie $(TAG) /usr/local/bin/python -m awslambdaric app.handler
+
+dev: aws_lambda
+	cd app && aws-lambda-rie /usr/bin/python -m awslambdaric app.handler

SNSAlertHub.yaml

@@ -0,0 +1,139 @@
+AWSTemplateFormatVersion: "2010-09-09"
+
+Description: "SNS Topic and tools to fan out alerts to email and or Slack"
+
+Conditions:
+
+  IsSetEmail:
+    Fn::Not:
+      - Fn::Equals:
+        - Ref: AlertEmail
+        - ""
+  IsSetSlack:
+    Fn::Not:
+      - Fn::Equals:
+        - Ref: AlertSlackWebHook
+        - ""
+
+Resources:
+
+  AlertHubTopic:
+    Type: AWS::SNS::Topic
+    Properties:
+      TopicName: AlertHub
+
+  # Email
+  EmailAlertsSubscription:
+    Type: AWS::SNS::Subscription
+    Condition: IsSetEmail
+    Properties:
+      Endpoint: { Ref: AlertEmail }
+      Protocol: email
+      TopicArn: { Ref: AlertHubTopic }
+
+  # Slack
+  SlackAlertsSubscription:
+    Type: AWS::SNS::Subscription
+    Condition: IsSetSlack
+    Properties:
+      Endpoint: {"Fn::GetAtt": ["SNSAlertHubFunction", "Arn"] }
+      Protocol: lambda
+      TopicArn: { Ref: AlertHubTopic }
+
+  IamRole:
+    Type: AWS::IAM::Role
+    Condition: IsSetSlack
+    Properties:
+      Policies:
+        - PolicyName: ResolveAccountAlias
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+                - iam:ListAccountAliases
+              Resource:
+                - "*"
+
+        - PolicyName: LogtoCloudwatchGroup
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+              - logs:CreateLogStream
+              - logs:PutLogEvents
+              Resource:
+                - Fn::Sub: "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/SNSAlertHub:log-stream:*"
+            - Effect: Allow
+              Action:
+              - logs:CreateLogGroup
+              Resource:
+                - Fn::Sub: "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/SNSAlertHub:*"
+
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Action:
+            - sts:AssumeRole
+            Effect: Allow
+            Principal:
+              Service: [ lambda.amazonaws.com ]
+
+  SNSAlertHubAllowed2Lambda:
+    Type: AWS::Lambda::Permission
+    Condition: IsSetSlack
+    Properties:
+      Action: lambda:InvokeFunction
+      Principal: sns.amazonaws.com
+      FunctionName: { Ref: SNSAlertHubFunction }
+      SourceArn: { Ref: AlertHubTopic }
+
+  SNSAlertHubFunction:
+    Type: AWS::Lambda::Function
+    Condition: IsSetSlack
+    Properties:
+      PackageType: Image
+      Code:
+        ImageUri: { "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ImageTag}" }
+      Description: "Lambda function to forward alerts from SNS to Slack"
+      FunctionName: SNSAlertHub
+      MemorySize: 128
+      Role: { "Fn::GetAtt": ["IamRole", "Arn"] }
+      Timeout: 10
+      Environment:
+        Variables:
+          WEBHOOK_URL: { Ref: AlertSlackWebHook }
+          # DEBUG: "1"
+
+Metadata:
+  Template:
+    Name: sns-alert-hub
+    Hash: 98fcf521f053f7412a90ce360ab62807
+    AwsCfnLib: v0.2.1
+  CloudBender:
+    Version: 0.9.9
+
+Parameters:
+
+  ImageTag:
+    Type: String
+    Description: "(Optional) Overwrite default ImageTag"
+    Default: "sns-alert-hub:v0.5.8"
+
+  AlertEmail:
+    Type: String
+    Description: "(Optional) Email address to receive alerts via SMTP"
+    Default: ""
+
+  AlertSlackWebHook:
+    Type: String
+    Description: "(Optional) Encrypted (KMS Default key) Slack webhook to post alerts; deploys Slack Lambda function"
+    Default: ""
+    NoEcho: True
+
+Outputs:
+
+  AlertHubTopic:
+    Value: { Ref: AlertHubTopic }
+    Description: ARN of the SNS AlertHub Topic

app/app.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python
+import os
+import json
+import logging
+import boto3
+import dateutil.parser
+import humanize
+import urllib
+
+import apprise
+
+__author__ = "Stefan Reimer"
+__author_email__ = "stefan@zero-downtime.net"
+__version__ = "0.7.2"
+
+# Global alias lookup cache
+account_aliases = {}
+
+logger = logging.getLogger(__name__)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+logging.getLogger("boto3").setLevel(logging.WARNING)
+logging.getLogger("botocore").setLevel(logging.WARNING)
+
+
+def boolean(value):
+    if value in ("t", "T", "true", "True", "TRUE", "1", 1, True):
+        return True
+    return False
+
+
+DEBUG = boolean(os.getenv("DEBUG", default=False))
+RESOLVE_ACCOUNT = boolean(os.getenv("RESOLVE_ACCOUNT", default=True))
+WEBHOOK_URL = os.environ.get("WEBHOOK_URL", "dbus://")
+
+if DEBUG:
+    logging.getLogger().setLevel(logging.DEBUG)
+else:
+    logging.getLogger().setLevel(logging.INFO)
+
+# Ensure slack URLs use ?blocks=yes
+if "slack.com" in WEBHOOK_URL:
+    scheme, netloc, path, query_string, fragment = urllib.parse.urlsplit(WEBHOOK_URL)
+    query_params = urllib.parse.parse_qs(query_string)
+    query_params["blocks"] = ["yes"]
+    new_query_string = urllib.parse.urlencode(query_params, doseq=True)
+    WEBHOOK_URL = urllib.parse.urlunsplit((scheme, netloc, path, new_query_string, fragment))
+
+# Setup apprise
+asset = apprise.AppriseAsset()
+
+# Set our app_id which is also used as User-Agent
+asset.app_desc = "SNSAlertHub part of ZeroDownTime CloudBender"
+asset.app_url = "https://zero-downtime.net"
+asset.image_url_mask = (
+    "https://cdn.zero-downtime.net/assets/zdt/apprise/{TYPE}-{XY}{EXTENSION}"
+)
+asset.app_id = "{} / {} {}".format("cloudbender", __version__, "zero-downtime.net")
+
+apobj = apprise.Apprise(asset=asset)
+apobj.add(WEBHOOK_URL)
+
+
+def get_alias(account_id):
+    """resolves aws account_id to account alias and caches for lifetime of lambda function"""
+    if RESOLVE_ACCOUNT:
+        try:
+            if account_id not in account_aliases:
+                iam = boto3.client("iam")
+                account_aliases[account_id] = iam.list_account_aliases()[
+                    "AccountAliases"
+                ][0]
+
+            return account_aliases[account_id]
+
+        except (KeyError, IndexError):
+            logger.warning("Could not resolve IAM account alias")
+            pass
+
+    return account_id
+
+
+def handler(event, context):
+    logger.debug(json.dumps({"aws.event": event}))  # sort_keys=True, indent=4
+
+    (region, account_id) = context.invoked_function_arn.split(":")[3:5]
+
+    sns = event["Records"][0]["Sns"]
+
+    # Guess what we have, try to parse as json first
+    try:
+        msg = json.loads(sns["Message"])
+
+    except json.decoder.JSONDecodeError:
+        msg = {}
+        pass
+
+    # CloudWatch Alarm ?
+    if "AlarmName" in msg:
+        title = "AWS Cloudwatch Alarm"
+
+        # Discard NewStateValue == OK && OldStateValue == INSUFFICIENT_DATA as these are triggered by installing new Alarms and only cause confusion
+        if msg["NewStateValue"] == "OK" and msg["OldStateValue"] == "INSUFFICIENT_DATA":
+            logger.info(
+                "Discarding Cloudwatch Metrics Alarm as state is OK and previous state was insufficient data, most likely new alarm being installed"
+            )
+            return 0
+
+        body = msg["AlarmDescription"]
+
+        msg_context = "{account} - {region} -> <https://{region}.console.aws.amazon.com/cloudwatch/home?region={region}#alarmsV2:alarm/{alarm}|Alarm Details>".format(
+            region=region,
+            alarm=msg["AlarmName"],
+            account=get_alias(msg["AWSAccountId"]),
+        )
+
+        try:
+            notify_map = {
+                "ok": apprise.NotifyType.SUCCESS,
+                "alarm": apprise.NotifyType.FAILURE,
+                "insuffcient_data": apprise.NotifyType.INFO,
+            }
+            msg_type = notify_map[msg["NewStateValue"].lower()]
+        except KeyError:
+            msg_type = apprise.NotifyType.WARNING
+
+        body = body + "\n\n_{}_".format(msg_context)
+        apobj.notify(body=body, title=title, notify_type=msg_type)
+
+    elif "Source" in msg and msg["Source"] == "CloudBender":
+        title = "AWS EC2 - CloudBender"
+
+        try:
+            msg_context = "{account} - {region} - {host} ({instance}) <https://{region}.console.aws.amazon.com/ec2/autoscaling/home?region={region}#AutoScalingGroups:id={asg};view=history|{artifact} ASG>".format(
+                account=get_alias(msg["AWSAccountId"]),
+                region=msg["Region"],
+                asg=msg["Asg"],
+                instance=msg["Instance"],
+                host=msg["Hostname"],
+                artifact=msg["Artifact"],
+            )
+        except KeyError:
+            msg_context = "{account} - {region}".format(
+                account=get_alias(msg["AWSAccountId"]), region=msg["Region"]
+            )
+
+        try:
+            notify_map = {
+                "warning": apprise.NotifyType.WARNING,
+                "error": apprise.NotifyType.FAILURE,
+                "info": apprise.NotifyType.INFO,
+                "success": apprise.NotifyType.SUCCESS,
+            }
+            msg_type = notify_map[msg["Level"].lower()]
+        except KeyError:
+            msg_type = apprise.NotifyType.WARNING
+
+        if "Subject" in msg and msg["Subject"]:
+            title = msg["Subject"]
+
+        body = ""
+        if "Message" in msg and msg["Message"]:
+            body = msg["Message"]
+
+        if "Attachment" in msg and msg["Attachment"]:
+            body = body + "\n```{}```".format(msg["Attachment"])
+
+        body = body + "\n\n_{}_".format(msg_context)
+        apobj.notify(body=body, title=title, notify_type=msg_type)
+
+    elif "receiver" in msg and msg["receiver"] == "alerthub-notifications":
+
+        for alert in msg["alerts"]:
+
+            # First msg_type
+            msg_type = apprise.NotifyType.WARNING
+            try:
+                if alert["status"] == "resolved":
+                    msg_type = apprise.NotifyType.SUCCESS
+                else:
+                    if alert["labels"]["severity"] == "critical":
+                        msg_type = apprise.NotifyType.FAILURE
+            except KeyError:
+                pass
+
+            # set title to Alertname
+            try:
+                title = alert["labels"]["alertname"]
+            except KeyError:
+                title = "Alertmanager"
+
+            # assemble message body
+            try:
+                body = "{}\n{}".format(
+                    alert["annotations"]["summary"], alert["annotations"]["description"]
+                )
+
+                if alert["status"] == "resolved":
+                    body = body + "\nDuration: {}".format(
+                        humanize.time.precisedelta(
+                            dateutil.parser.parse(alert["startsAt"]) - dateutil.parser.parse(alert["endsAt"])
+                        )
+                    )
+                else:
+                    if "runbook_url" in alert["annotations"]:
+                        body = body + " <{}|Runbook>".format(
+                            alert["annotations"]["runbook_url"]
+                        )
+                    if "generatorURL" in alert["annotations"]:
+                        body = body + " <{}|Source>".format(
+                            alert["annotations"]["generatorURL"]
+                        )
+            except KeyError:
+                body = "Unknown Alert:\n{}".format(alert)
+
+            try:
+                msg_context = "{account} - {region} - <{alert_manager_link}/#/alerts?receiver=alerthub-notifications|{cluster}>".format(
+                    cluster=alert["labels"]["clusterName"],
+                    region=alert["labels"]["awsRegion"],
+                    account=get_alias(alert["labels"]["awsAccount"]),
+                    alert_manager_link=msg["externalURL"],
+                )
+                body = body + "\n\n_{}_".format(msg_context)
+            except KeyError:
+                pass
+
+            # Finally send each parsed alert
+            apobj.notify(body=body, title=title, notify_type=msg_type)
+
+    else:
+        body = sns["Message"]
+        apobj.notify(body=body, title="Unknow message type", notify_type=apprise.NotifyType.WARNING)

requirements-dev.txt

@@ -0,0 +1,5 @@
+boto3
+apprise
+humanize
+awslambdaric
+flake8

requirements.txt

@@ -0,0 +1,3 @@
+boto3
+apprise
+humanize

test.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+import json
+import requests
+import subprocess
+import time
+
+
+def send_event(event):
+    requests.post('http://localhost:8080/2015-03-31/functions/function/invocations', json=event)
+
+
+p = subprocess.Popen('cd app && aws-lambda-rie /usr/bin/python -m awslambdaric app.handler', shell=True)
+time.sleep(3)
+
+# Cloudwatch Alarm
+event = json.loads(r' { "Records": [ { "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:EXAMPLE", "EventSource": "aws:sns", "Sns": { "SignatureVersion": "1", "Timestamp": "1970-01-01T00:00:00.000Z", "Signature": "EXAMPLE", "SigningCertUrl": "EXAMPLE", "MessageId": "95df01b4-ee98-5cb9-9903-4c221d41eb5e", "Message": "{\"AlarmName\":\"sns-slack-test-from-cloudwatch-total-cpu\",\"AlarmDescription\":\"Hello from AWS\",\"AWSAccountId\":\"123456789012\",\"NewStateValue\":\"OK\",\"NewStateReason\":\"Threshold Crossed: 1 datapoint (7.9053535353535365) was not greater than or equal to the threshold (8.0).\",\"StateChangeTime\":\"2015-11-09T21:19:43.454+0000\",\"Region\":\"US - N. Virginia\",\"OldStateValue\":\"ALARM\",\"Trigger\":{\"MetricName\":\"CPUUtilization\",\"Namespace\":\"AWS/EC2\",\"Statistic\":\"AVERAGE\",\"Unit\":null,\"Dimensions\":[],\"Period\":300,\"EvaluationPeriods\":1,\"ComparisonOperator\":\"GreaterThanOrEqualToThreshold\",\"Threshold\":8.0}}", "MessageAttributes": { "Test": { "Type": "String", "Value": "TestString" }, "TestBinary": { "Type": "Binary", "Value": "TestBinary" } }, "Type": "Notification", "UnsubscribeUrl": "EXAMPLE", "TopicArn": "arn:aws:sns:us-east-1:123456789012:production-notices", "Subject": "OK: sns-slack-test-from-cloudwatch-total-cpu" } } ] }')
+send_event(event)
+
+
+# CloudBender launch event
+event = json.loads(r' { "Records": [ { "EventSource": "aws:sns", "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub:0c04d2e7-32ec-4933-b913-84c7172e6d90", "Sns": { "Type": "Notification", "MessageId": "25bb2fd0-3221-5c07-aea1-76acf75017c3", "TopicArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub", "Subject": null, "Message": "{\n  \"Source\": \"CloudBender\",\n  \"AWSAccountId\": \"123456789012\",\n  \"Region\": \"eu-central-1\",\n  \"Artifact\": \"bastion\",\n  \"Hostname\": \"bastion\",\n  \"Asg\": \"zdt-bastion-BastionAsg-UABA2PIZV4TI\",\n  \"Instance\": \"i-0d9f08855cdfcd740\",\n  \"Subject\": \"Test\",\n  \"Level\": \"Info\",\n  \"Message\": \"Hello\",\n  \"Attachment\": \"\",\n  \"Emoji\": \"\"\n}\n", "Timestamp": "2021-07-29T15:03:13.318Z", "SignatureVersion": "1", "Signature": "OhLoCz8RWazyZ+ZHK03QLby6M3jmdtZvLWoNFygAHIaljQ0ZHsd6mc4TskDTnqpUpCtd/iIl+TLIPN8hYyflbLk2/cN3LXXWcSQ0GWqQ/bWIEhej54oCmUgZjIzrVfRlgz7mlUkhRnjQoRWYpcXRycQczMWuF2DCeIDP6v3ON53BxR8NdCeQXiquwoFlHaAaZIviRoUMqwp2Cl1T0NaBLeL9zmsdPvJF6EaXRbu3rqC1hdrA6E+nV2lzYNKg09POxA9JVpURmMEq3AC4tXm1Gu73PWQgWgoDSOQx+SOjMrbMeCAqf5R6typBV1BRDsGPkNkt9n4oto0FR9iyDmuWog==", "SigningCertUrl": "https://sns.eu-central-1.amazonaws.com/SimpleNotificationService-010a507c1833636cd94bdb98bd93083a.pem", "UnsubscribeUrl": "https://sns.eu-central-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-central-1:123456789012:AlertHub:0c04d2e7-32ec-4933-b913-84c7172e6d90", "MessageAttributes": {} } } ] }')
+send_event(event)
+
+
+# Alertmanager
+event = json.loads(r' { "Records": [ { "EventSource": "aws:sns", "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "Sns": { "Type": "Notification", "MessageId": "10ae86eb-9ddc-5c2f-806c-df6ecb6bde42", "TopicArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub", "Subject": null, "Message": "{\"receiver\":\"alerthub-notifications\",\"status\":\"firing\",\"alerts\":[{\"status\":\"firing\",\"labels\":{\"alertname\":\"KubeVersionMismatch\",\"awsRegion\":\"eu-central-1\",\"clusterName\":\"test\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"severity\":\"warning\"},\"annotations\":{\"description\":\"There are 2 different semantic versions of Kubernetes components running.\",\"runbook_url\":\"https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch\",\"summary\":\"Different semantic versions of Kubernetes components running.\"},\"startsAt\":\"2021-08-04T13:17:40.31Z\",\"endsAt\":\"0001-01-01T00:00:00Z\",\"generatorURL\":\"https://prometheus/graph?g0.expr=count%28count+by%28git_version%29+%28label_replace%28kubernetes_build_info%7Bjob%21~%22kube-dns%7Ccoredns%22%7D%2C+%22git_version%22%2C+%22%241%22%2C+%22git_version%22%2C+%22%28v%5B0-9%5D%2A.%5B0-9%5D%2A%29.%2A%22%29%29%29+%3E+1\\u0026g0.tab=1\",\"fingerprint\":\"5f94d4a22730c666\"}],\"groupLabels\":{},\"commonLabels\":{\"alertname\":\"KubeVersionMismatch\",\"awsRegion\":\"eu-central-1\",\"clusterName\":\"test\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"severity\":\"warning\"},\"commonAnnotations\":{\"description\":\"There are 2 different semantic versions of Kubernetes components running.\",\"runbook_url\":\"https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch\",\"summary\":\"Different semantic versions of Kubernetes components running.\"},\"externalURL\":\"https://alertmanager\",\"version\":\"4\",\"groupKey\":\"{}:{}\",\"truncatedAlerts\":0}\n", "Timestamp": "2021-08-05T03:01:11.233Z", "SignatureVersion": "1", "Signature": "pSUYO7LDIfzCbBrp/S2HXV3/yzls3vfYy+2di6HsKG8Mf+CV97RLnen15ieAo3eKA8IfviZIzyREasbF0cwfUeruHPbW1B8kO572fDyV206zmUxvR63r6oM6OyLv9XKBmvyYHKawkOgHZHEMP3v1wMIIHK2W5KbJtXoUcks5DVamooVb9iFF58uqTf+Ccy31bOL4tFyMR9nr8NU55vEIlGEVno8A9Q21TujdZTg0V0BmRgPafcic96udWungjmfhZ005378N32u2hlLj6BRneTpHHSXHBw4wKZreKpX+INZwiZ4P8hzVfgRvAIh/4gXN9+0UJSHgnsaqUcLDNoLZTQ==", "SigningCertUrl": "https://sns.eu-central-1.amazonaws.com/SimpleNotificationService-010a507c1833636cd94bdb98bd93083a.pem", "UnsubscribeUrl": "https://sns.eu-central-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "MessageAttributes": {} } } ] }')
+send_event(event)
+
+event = json.loads(r' { "Records": [ { "EventSource": "aws:sns", "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "Sns": { "Type": "Notification", "MessageId": "10ae86eb-9ddc-5c2f-806c-df6ecb6bde42", "TopicArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub", "Subject": null, "Message": "{\"receiver\":\"alerthub-notifications\",\"status\":\"resolved\",\"alerts\":[{\"status\":\"resolved\",\"labels\":{\"alertname\":\"KubeDeploymentReplicasMismatch\",\"awsAccount\":\"668666974128\",\"awsRegion\":\"us-west-2\",\"clusterName\":\"mayne-aws-dev\",\"container\":\"kube-state-metrics\",\"deployment\":\"extrude-job\",\"endpoint\":\"http\",\"instance\":\"10.244.202.71:8080\",\"job\":\"kube-state-metrics\",\"namespace\":\"default\",\"pod\":\"metrics-kube-state-metrics-56546f44c7-h57jx\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"service\":\"metrics-kube-state-metrics\",\"severity\":\"warning\"},\"annotations\":{\"description\":\"Deployment default/extrude-job has not matched the expected number of replicas for longer than 15 minutes.\",\"runbook_url\":\"https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch\",\"summary\":\"Deployment has not matched the expected number of replicas.\"},\"startsAt\":\"2021-09-29T12:36:11.394Z\",\"endsAt\":\"2021-09-29T14:51:11.394Z\",\"generatorURL\":\"https://prometheus.dev.mayneinc.com/graph?g0.expr=%28kube_deployment_spec_replicas%7Bjob%3D%22kube-state-metrics%22%7D+%3E+kube_deployment_status_replicas_available%7Bjob%3D%22kube-state-metrics%22%7D%29+and+%28changes%28kube_deployment_status_replicas_updated%7Bjob%3D%22kube-state-metrics%22%7D%5B10m%5D%29+%3D%3D+0%29\\u0026g0.tab=1\",\"fingerprint\":\"59ad2f1a4567b43b\"},{\"status\":\"firing\",\"labels\":{\"alertname\":\"KubeVersionMismatch\",\"awsRegion\":\"eu-central-1\",\"clusterName\":\"test\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"severity\":\"warning\"},\"annotations\":{\"description\":\"There are 2 different semantic versions of Kubernetes components running.\",\"runbook_url\":\"https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch\",\"summary\":\"Different semantic versions of Kubernetes components running.\"},\"startsAt\":\"2021-08-04T13:17:40.31Z\",\"endsAt\":\"0001-01-01T00:00:00Z\",\"generatorURL\":\"https://prometheus/graph?g0.expr=count%28count+by%28git_version%29+%28label_replace%28kubernetes_build_info%7Bjob%21~%22kube-dns%7Ccoredns%22%7D%2C+%22git_version%22%2C+%22%241%22%2C+%22git_version%22%2C+%22%28v%5B0-9%5D%2A.%5B0-9%5D%2A%29.%2A%22%29%29%29+%3E+1\\u0026g0.tab=1\",\"fingerprint\":\"5f94d4a22730c666\"}],\"groupLabels\":{\"job\":\"kube-state-metrics\"},\"commonLabels\":{\"alertname\":\"KubeDeploymentReplicasMismatch\",\"awsAccount\":\"668666974128\",\"awsRegion\":\"us-west-2\",\"clusterName\":\"mayne-aws-dev\",\"container\":\"kube-state-metrics\",\"deployment\":\"extrude-job\",\"endpoint\":\"http\",\"instance\":\"10.244.202.71:8080\",\"job\":\"kube-state-metrics\",\"namespace\":\"default\",\"pod\":\"metrics-kube-state-metrics-56546f44c7-h57jx\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"service\":\"metrics-kube-state-metrics\",\"severity\":\"warning\"},\"commonAnnotations\":{\"description\":\"Deployment default/extrude-job has not matched the expected number of replicas for longer than 15 minutes.\",\"runbook_url\":\"https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch\",\"summary\":\"Deployment has not matched the expected number of replicas.\"},\"externalURL\":\"https://alertmanager.dev.mayneinc.com\",\"version\":\"4\",\"groupKey\":\"{}:{job=\\\"kube-state-metrics\\\"}\",\"truncatedAlerts\":0}\n", "Timestamp": "2021-08-05T03:01:11.233Z", "SignatureVersion": "1", "Signature": "pSUYO7LDIfzCbBrp/S2HXV3/yzls3vfYy+2di6HsKG8Mf+CV97RLnen15ieAo3eKA8IfviZIzyREasbF0cwfUeruHPbW1B8kO572fDyV206zmUxvR63r6oM6OyLv9XKBmvyYHKawkOgHZHEMP3v1wMIIHK2W5KbJtXoUcks5DVamooVb9iFF58uqTf+Ccy31bOL4tFyMR9nr8NU55vEIlGEVno8A9Q21TujdZTg0V0BmRgPafcic96udWungjmfhZ005378N32u2hlLj6BRneTpHHSXHBw4wKZreKpX+INZwiZ4P8hzVfgRvAIh/4gXN9+0UJSHgnsaqUcLDNoLZTQ==", "SigningCertUrl": "https://sns.eu-central-1.amazonaws.com/SimpleNotificationService-010a507c1833636cd94bdb98bd93083a.pem", "UnsubscribeUrl": "https://sns.eu-central-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "MessageAttributes": {} } } ] }')
+send_event(event)
+
+p.terminate()