feat: Initial commit v0.7.2
ZeroDownTime/sns-alert-hub/pipeline/head There was a failure building this commit
Details
ZeroDownTime/sns-alert-hub/pipeline/head There was a failure building this commit
Details
This commit is contained in:
commit
618cb50feb
|
|
@ -0,0 +1,55 @@
|
|||
# https://aws.amazon.com/blogs/aws/new-for-aws-lambda-container-image-support/
|
||||
ARG FUNCTION_DIR="/app"
|
||||
ARG RUNTIME_VERSION="3.9"
|
||||
ARG DISTRO_VERSION="3.15"
|
||||
|
||||
# Stage 1 - bundle base image + runtime
|
||||
# Grab a fresh copy of the image and install GCC
|
||||
FROM python:${RUNTIME_VERSION}-alpine${DISTRO_VERSION} AS python-alpine
|
||||
# Install GCC (Alpine uses musl but we compile and link dependencies with GCC)
|
||||
RUN apk add --no-cache \
|
||||
libstdc++
|
||||
|
||||
# Stage 2 - build function and dependencies
|
||||
FROM python-alpine AS build-image
|
||||
# Install aws-lambda-cpp build dependencies
|
||||
RUN apk add --no-cache \
|
||||
build-base \
|
||||
libtool \
|
||||
autoconf \
|
||||
automake \
|
||||
libexecinfo-dev \
|
||||
make \
|
||||
cmake \
|
||||
libcurl \
|
||||
libffi-dev \
|
||||
openssl-dev
|
||||
# cargo
|
||||
|
||||
# Include global args in this stage of the build
|
||||
ARG FUNCTION_DIR
|
||||
ARG RUNTIME_VERSION
|
||||
# Create function directory
|
||||
RUN mkdir -p ${FUNCTION_DIR}
|
||||
# Copy handler function
|
||||
COPY app/* ${FUNCTION_DIR}
|
||||
|
||||
# Install requirements
|
||||
COPY requirements.txt requirements.txt
|
||||
RUN python${RUNTIME_VERSION} -m pip install -r requirements.txt --target ${FUNCTION_DIR}
|
||||
|
||||
# Install Lambda Runtime Interface Client for Python
|
||||
RUN python${RUNTIME_VERSION} -m pip install awslambdaric --target ${FUNCTION_DIR}
|
||||
|
||||
# Stage 3 - final runtime image
|
||||
# Grab a fresh copy of the Python image
|
||||
FROM python-alpine
|
||||
# Include global arg in this stage of the build
|
||||
ARG FUNCTION_DIR
|
||||
# Set working directory to function root directory
|
||||
WORKDIR ${FUNCTION_DIR}
|
||||
# Copy in the built dependencies
|
||||
COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR}
|
||||
|
||||
ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ]
|
||||
CMD [ "app.handler" ]
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
pipeline {
|
||||
agent { node { label 'podman-aws-trivy' } }
|
||||
|
||||
stages {
|
||||
stage('Prepare'){
|
||||
// get tags
|
||||
steps {
|
||||
sh 'git fetch -q --tags ${GIT_URL} +refs/heads/${BRANCH_NAME}:refs/remotes/origin/${BRANCH_NAME}'
|
||||
}
|
||||
}
|
||||
|
||||
// Build using rootless podman
|
||||
stage('Build'){
|
||||
steps {
|
||||
sh 'make build'
|
||||
}
|
||||
}
|
||||
|
||||
stage('Test'){
|
||||
steps {
|
||||
sh 'make test'
|
||||
}
|
||||
}
|
||||
|
||||
// Scan via trivy
|
||||
stage('Scan'){
|
||||
environment {
|
||||
TRIVY_FORMAT = "template"
|
||||
TRIVY_OUTPUT = "reports/trivy.html"
|
||||
}
|
||||
steps {
|
||||
sh 'mkdir -p reports'
|
||||
sh 'make scan'
|
||||
publishHTML target : [
|
||||
allowMissing: true,
|
||||
alwaysLinkToLastBuild: true,
|
||||
keepAll: true,
|
||||
reportDir: 'reports',
|
||||
reportFiles: 'trivy.html',
|
||||
reportName: 'TrivyScan',
|
||||
reportTitles: 'TrivyScan'
|
||||
]
|
||||
|
||||
// Scan again and fail on CRITICAL vulns
|
||||
sh 'TRIVY_EXIT_CODE=1 TRIVY_SEVERITY=CRITICAL make scan'
|
||||
}
|
||||
}
|
||||
|
||||
// Push to ECR
|
||||
stage('Push'){
|
||||
steps {
|
||||
sh 'make push'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
REGISTRY := public.ecr.aws/zero-downtime
|
||||
REPOSITORY := sns-alert-hub
|
||||
REGION := us-east-1
|
||||
|
||||
# Parse version from latest git semver tag
|
||||
GTAG=$(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
|
||||
TAG ?= $(shell echo $(GTAG) | awk -F '-' '{ print $$1 "-" $$2 }' | sed -e 's/-$$//')
|
||||
|
||||
ifeq ($(TRIVY_REMOTE),)
|
||||
TRIVY_OPTS := image
|
||||
else
|
||||
TRIVY_OPTS := client --remote ${TRIVY_REMOTE}
|
||||
endif
|
||||
|
||||
.PHONY: build push scan test
|
||||
|
||||
all: build
|
||||
|
||||
build:
|
||||
podman build --rm --squash-all -t $(REPOSITORY):$(TAG) -t $(REPOSITORY):latest .
|
||||
|
||||
push:
|
||||
aws ecr-public get-login-password --region $(REGION) | podman login --username AWS --password-stdin $(REGISTRY)
|
||||
podman tag $(REPOSITORY):latest $(REGISTRY)/$(REPOSITORY):$(TAG) $(REGISTRY)/$(REPOSITORY):latest
|
||||
podman push $(REGISTRY)/$(REPOSITORY):$(TAG)
|
||||
podman push $(REGISTRY)/$(REPOSITORY):latest
|
||||
# Delete all untagged images
|
||||
# aws ecr-public batch-delete-image --repository-name $(REPOSITORY) --region $(REGION) --image-ids $$(for image in $$(aws ecr-public describe-images --repository-name $(REPOSITORY) --region $(REGION) --output json | jq -r '.imageDetails[] | select(.imageTags | not ).imageDigest'); do echo -n "imageDigest=$$image "; done)
|
||||
|
||||
scan:
|
||||
trivy $(TRIVY_OPTS) $(REPOSITORY):$(TAG)
|
||||
|
||||
test:
|
||||
flake8 --ignore=E501 app/app.py
|
||||
./test.py
|
||||
|
||||
aws_lambda:
|
||||
@[ -z $(AWS_LAMBDA) ] && { echo "Missing aws-lambda-rie in PATH"; exit 1;} || true
|
||||
|
||||
run: aws_lambda
|
||||
podman run --rm -v $(AWS_LAMBDA):/aws-lambda -p 8080:8080 --entrypoint /aws-lambda/aws-lambda-rie $(TAG) /usr/local/bin/python -m awslambdaric app.handler
|
||||
|
||||
dev: aws_lambda
|
||||
cd app && aws-lambda-rie /usr/bin/python -m awslambdaric app.handler
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
AWSTemplateFormatVersion: "2010-09-09"
|
||||
|
||||
Description: "SNS Topic and tools to fan out alerts to email and or Slack"
|
||||
|
||||
Conditions:
|
||||
|
||||
IsSetEmail:
|
||||
Fn::Not:
|
||||
- Fn::Equals:
|
||||
- Ref: AlertEmail
|
||||
- ""
|
||||
IsSetSlack:
|
||||
Fn::Not:
|
||||
- Fn::Equals:
|
||||
- Ref: AlertSlackWebHook
|
||||
- ""
|
||||
|
||||
Resources:
|
||||
|
||||
AlertHubTopic:
|
||||
Type: AWS::SNS::Topic
|
||||
Properties:
|
||||
TopicName: AlertHub
|
||||
|
||||
# Email
|
||||
EmailAlertsSubscription:
|
||||
Type: AWS::SNS::Subscription
|
||||
Condition: IsSetEmail
|
||||
Properties:
|
||||
Endpoint: { Ref: AlertEmail }
|
||||
Protocol: email
|
||||
TopicArn: { Ref: AlertHubTopic }
|
||||
|
||||
# Slack
|
||||
SlackAlertsSubscription:
|
||||
Type: AWS::SNS::Subscription
|
||||
Condition: IsSetSlack
|
||||
Properties:
|
||||
Endpoint: {"Fn::GetAtt": ["SNSAlertHubFunction", "Arn"] }
|
||||
Protocol: lambda
|
||||
TopicArn: { Ref: AlertHubTopic }
|
||||
|
||||
IamRole:
|
||||
Type: AWS::IAM::Role
|
||||
Condition: IsSetSlack
|
||||
Properties:
|
||||
Policies:
|
||||
- PolicyName: ResolveAccountAlias
|
||||
PolicyDocument:
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- iam:ListAccountAliases
|
||||
Resource:
|
||||
- "*"
|
||||
|
||||
- PolicyName: LogtoCloudwatchGroup
|
||||
PolicyDocument:
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- logs:CreateLogStream
|
||||
- logs:PutLogEvents
|
||||
Resource:
|
||||
- Fn::Sub: "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/SNSAlertHub:log-stream:*"
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- logs:CreateLogGroup
|
||||
Resource:
|
||||
- Fn::Sub: "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/SNSAlertHub:*"
|
||||
|
||||
AssumeRolePolicyDocument:
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Action:
|
||||
- sts:AssumeRole
|
||||
Effect: Allow
|
||||
Principal:
|
||||
Service: [ lambda.amazonaws.com ]
|
||||
|
||||
SNSAlertHubAllowed2Lambda:
|
||||
Type: AWS::Lambda::Permission
|
||||
Condition: IsSetSlack
|
||||
Properties:
|
||||
Action: lambda:InvokeFunction
|
||||
Principal: sns.amazonaws.com
|
||||
FunctionName: { Ref: SNSAlertHubFunction }
|
||||
SourceArn: { Ref: AlertHubTopic }
|
||||
|
||||
SNSAlertHubFunction:
|
||||
Type: AWS::Lambda::Function
|
||||
Condition: IsSetSlack
|
||||
Properties:
|
||||
PackageType: Image
|
||||
Code:
|
||||
ImageUri: { "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ImageTag}" }
|
||||
Description: "Lambda function to forward alerts from SNS to Slack"
|
||||
FunctionName: SNSAlertHub
|
||||
MemorySize: 128
|
||||
Role: { "Fn::GetAtt": ["IamRole", "Arn"] }
|
||||
Timeout: 10
|
||||
Environment:
|
||||
Variables:
|
||||
WEBHOOK_URL: { Ref: AlertSlackWebHook }
|
||||
# DEBUG: "1"
|
||||
|
||||
Metadata:
|
||||
Template:
|
||||
Name: sns-alert-hub
|
||||
Hash: 98fcf521f053f7412a90ce360ab62807
|
||||
AwsCfnLib: v0.2.1
|
||||
CloudBender:
|
||||
Version: 0.9.9
|
||||
|
||||
Parameters:
|
||||
|
||||
ImageTag:
|
||||
Type: String
|
||||
Description: "(Optional) Overwrite default ImageTag"
|
||||
Default: "sns-alert-hub:v0.5.8"
|
||||
|
||||
AlertEmail:
|
||||
Type: String
|
||||
Description: "(Optional) Email address to receive alerts via SMTP"
|
||||
Default: ""
|
||||
|
||||
AlertSlackWebHook:
|
||||
Type: String
|
||||
Description: "(Optional) Encrypted (KMS Default key) Slack webhook to post alerts; deploys Slack Lambda function"
|
||||
Default: ""
|
||||
NoEcho: True
|
||||
|
||||
Outputs:
|
||||
|
||||
AlertHubTopic:
|
||||
Value: { Ref: AlertHubTopic }
|
||||
Description: ARN of the SNS AlertHub Topic
|
||||
|
|
@ -0,0 +1,231 @@
|
|||
#!/usr/bin/env python
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import boto3
|
||||
import dateutil.parser
|
||||
import humanize
|
||||
import urllib
|
||||
|
||||
import apprise
|
||||
|
||||
__author__ = "Stefan Reimer"
|
||||
__author_email__ = "stefan@zero-downtime.net"
|
||||
__version__ = "0.7.2"
|
||||
|
||||
# Global alias lookup cache
|
||||
account_aliases = {}
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||||
logging.getLogger("boto3").setLevel(logging.WARNING)
|
||||
logging.getLogger("botocore").setLevel(logging.WARNING)
|
||||
|
||||
|
||||
def boolean(value):
|
||||
if value in ("t", "T", "true", "True", "TRUE", "1", 1, True):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
DEBUG = boolean(os.getenv("DEBUG", default=False))
|
||||
RESOLVE_ACCOUNT = boolean(os.getenv("RESOLVE_ACCOUNT", default=True))
|
||||
WEBHOOK_URL = os.environ.get("WEBHOOK_URL", "dbus://")
|
||||
|
||||
if DEBUG:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
else:
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
# Ensure slack URLs use ?blocks=yes
|
||||
if "slack.com" in WEBHOOK_URL:
|
||||
scheme, netloc, path, query_string, fragment = urllib.parse.urlsplit(WEBHOOK_URL)
|
||||
query_params = urllib.parse.parse_qs(query_string)
|
||||
query_params["blocks"] = ["yes"]
|
||||
new_query_string = urllib.parse.urlencode(query_params, doseq=True)
|
||||
WEBHOOK_URL = urllib.parse.urlunsplit((scheme, netloc, path, new_query_string, fragment))
|
||||
|
||||
# Setup apprise
|
||||
asset = apprise.AppriseAsset()
|
||||
|
||||
# Set our app_id which is also used as User-Agent
|
||||
asset.app_desc = "SNSAlertHub part of ZeroDownTime CloudBender"
|
||||
asset.app_url = "https://zero-downtime.net"
|
||||
asset.image_url_mask = (
|
||||
"https://cdn.zero-downtime.net/assets/zdt/apprise/{TYPE}-{XY}{EXTENSION}"
|
||||
)
|
||||
asset.app_id = "{} / {} {}".format("cloudbender", __version__, "zero-downtime.net")
|
||||
|
||||
apobj = apprise.Apprise(asset=asset)
|
||||
apobj.add(WEBHOOK_URL)
|
||||
|
||||
|
||||
def get_alias(account_id):
|
||||
"""resolves aws account_id to account alias and caches for lifetime of lambda function"""
|
||||
if RESOLVE_ACCOUNT:
|
||||
try:
|
||||
if account_id not in account_aliases:
|
||||
iam = boto3.client("iam")
|
||||
account_aliases[account_id] = iam.list_account_aliases()[
|
||||
"AccountAliases"
|
||||
][0]
|
||||
|
||||
return account_aliases[account_id]
|
||||
|
||||
except (KeyError, IndexError):
|
||||
logger.warning("Could not resolve IAM account alias")
|
||||
pass
|
||||
|
||||
return account_id
|
||||
|
||||
|
||||
def handler(event, context):
|
||||
logger.debug(json.dumps({"aws.event": event})) # sort_keys=True, indent=4
|
||||
|
||||
(region, account_id) = context.invoked_function_arn.split(":")[3:5]
|
||||
|
||||
sns = event["Records"][0]["Sns"]
|
||||
|
||||
# Guess what we have, try to parse as json first
|
||||
try:
|
||||
msg = json.loads(sns["Message"])
|
||||
|
||||
except json.decoder.JSONDecodeError:
|
||||
msg = {}
|
||||
pass
|
||||
|
||||
# CloudWatch Alarm ?
|
||||
if "AlarmName" in msg:
|
||||
title = "AWS Cloudwatch Alarm"
|
||||
|
||||
# Discard NewStateValue == OK && OldStateValue == INSUFFICIENT_DATA as these are triggered by installing new Alarms and only cause confusion
|
||||
if msg["NewStateValue"] == "OK" and msg["OldStateValue"] == "INSUFFICIENT_DATA":
|
||||
logger.info(
|
||||
"Discarding Cloudwatch Metrics Alarm as state is OK and previous state was insufficient data, most likely new alarm being installed"
|
||||
)
|
||||
return 0
|
||||
|
||||
body = msg["AlarmDescription"]
|
||||
|
||||
msg_context = "{account} - {region} -> <https://{region}.console.aws.amazon.com/cloudwatch/home?region={region}#alarmsV2:alarm/{alarm}|Alarm Details>".format(
|
||||
region=region,
|
||||
alarm=msg["AlarmName"],
|
||||
account=get_alias(msg["AWSAccountId"]),
|
||||
)
|
||||
|
||||
try:
|
||||
notify_map = {
|
||||
"ok": apprise.NotifyType.SUCCESS,
|
||||
"alarm": apprise.NotifyType.FAILURE,
|
||||
"insuffcient_data": apprise.NotifyType.INFO,
|
||||
}
|
||||
msg_type = notify_map[msg["NewStateValue"].lower()]
|
||||
except KeyError:
|
||||
msg_type = apprise.NotifyType.WARNING
|
||||
|
||||
body = body + "\n\n_{}_".format(msg_context)
|
||||
apobj.notify(body=body, title=title, notify_type=msg_type)
|
||||
|
||||
elif "Source" in msg and msg["Source"] == "CloudBender":
|
||||
title = "AWS EC2 - CloudBender"
|
||||
|
||||
try:
|
||||
msg_context = "{account} - {region} - {host} ({instance}) <https://{region}.console.aws.amazon.com/ec2/autoscaling/home?region={region}#AutoScalingGroups:id={asg};view=history|{artifact} ASG>".format(
|
||||
account=get_alias(msg["AWSAccountId"]),
|
||||
region=msg["Region"],
|
||||
asg=msg["Asg"],
|
||||
instance=msg["Instance"],
|
||||
host=msg["Hostname"],
|
||||
artifact=msg["Artifact"],
|
||||
)
|
||||
except KeyError:
|
||||
msg_context = "{account} - {region}".format(
|
||||
account=get_alias(msg["AWSAccountId"]), region=msg["Region"]
|
||||
)
|
||||
|
||||
try:
|
||||
notify_map = {
|
||||
"warning": apprise.NotifyType.WARNING,
|
||||
"error": apprise.NotifyType.FAILURE,
|
||||
"info": apprise.NotifyType.INFO,
|
||||
"success": apprise.NotifyType.SUCCESS,
|
||||
}
|
||||
msg_type = notify_map[msg["Level"].lower()]
|
||||
except KeyError:
|
||||
msg_type = apprise.NotifyType.WARNING
|
||||
|
||||
if "Subject" in msg and msg["Subject"]:
|
||||
title = msg["Subject"]
|
||||
|
||||
body = ""
|
||||
if "Message" in msg and msg["Message"]:
|
||||
body = msg["Message"]
|
||||
|
||||
if "Attachment" in msg and msg["Attachment"]:
|
||||
body = body + "\n```{}```".format(msg["Attachment"])
|
||||
|
||||
body = body + "\n\n_{}_".format(msg_context)
|
||||
apobj.notify(body=body, title=title, notify_type=msg_type)
|
||||
|
||||
elif "receiver" in msg and msg["receiver"] == "alerthub-notifications":
|
||||
|
||||
for alert in msg["alerts"]:
|
||||
|
||||
# First msg_type
|
||||
msg_type = apprise.NotifyType.WARNING
|
||||
try:
|
||||
if alert["status"] == "resolved":
|
||||
msg_type = apprise.NotifyType.SUCCESS
|
||||
else:
|
||||
if alert["labels"]["severity"] == "critical":
|
||||
msg_type = apprise.NotifyType.FAILURE
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# set title to Alertname
|
||||
try:
|
||||
title = alert["labels"]["alertname"]
|
||||
except KeyError:
|
||||
title = "Alertmanager"
|
||||
|
||||
# assemble message body
|
||||
try:
|
||||
body = "{}\n{}".format(
|
||||
alert["annotations"]["summary"], alert["annotations"]["description"]
|
||||
)
|
||||
|
||||
if alert["status"] == "resolved":
|
||||
body = body + "\nDuration: {}".format(
|
||||
humanize.time.precisedelta(
|
||||
dateutil.parser.parse(alert["startsAt"]) - dateutil.parser.parse(alert["endsAt"])
|
||||
)
|
||||
)
|
||||
else:
|
||||
if "runbook_url" in alert["annotations"]:
|
||||
body = body + " <{}|Runbook>".format(
|
||||
alert["annotations"]["runbook_url"]
|
||||
)
|
||||
if "generatorURL" in alert["annotations"]:
|
||||
body = body + " <{}|Source>".format(
|
||||
alert["annotations"]["generatorURL"]
|
||||
)
|
||||
except KeyError:
|
||||
body = "Unknown Alert:\n{}".format(alert)
|
||||
|
||||
try:
|
||||
msg_context = "{account} - {region} - <{alert_manager_link}/#/alerts?receiver=alerthub-notifications|{cluster}>".format(
|
||||
cluster=alert["labels"]["clusterName"],
|
||||
region=alert["labels"]["awsRegion"],
|
||||
account=get_alias(alert["labels"]["awsAccount"]),
|
||||
alert_manager_link=msg["externalURL"],
|
||||
)
|
||||
body = body + "\n\n_{}_".format(msg_context)
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# Finally send each parsed alert
|
||||
apobj.notify(body=body, title=title, notify_type=msg_type)
|
||||
|
||||
else:
|
||||
body = sns["Message"]
|
||||
apobj.notify(body=body, title="Unknow message type", notify_type=apprise.NotifyType.WARNING)
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
boto3
|
||||
apprise
|
||||
humanize
|
||||
awslambdaric
|
||||
flake8
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
boto3
|
||||
apprise
|
||||
humanize
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
#!/usr/bin/env python
|
||||
import json
|
||||
import requests
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
def send_event(event):
|
||||
requests.post('http://localhost:8080/2015-03-31/functions/function/invocations', json=event)
|
||||
|
||||
|
||||
p = subprocess.Popen('cd app && aws-lambda-rie /usr/bin/python -m awslambdaric app.handler', shell=True)
|
||||
time.sleep(3)
|
||||
|
||||
# Cloudwatch Alarm
|
||||
event = json.loads(r' { "Records": [ { "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:EXAMPLE", "EventSource": "aws:sns", "Sns": { "SignatureVersion": "1", "Timestamp": "1970-01-01T00:00:00.000Z", "Signature": "EXAMPLE", "SigningCertUrl": "EXAMPLE", "MessageId": "95df01b4-ee98-5cb9-9903-4c221d41eb5e", "Message": "{\"AlarmName\":\"sns-slack-test-from-cloudwatch-total-cpu\",\"AlarmDescription\":\"Hello from AWS\",\"AWSAccountId\":\"123456789012\",\"NewStateValue\":\"OK\",\"NewStateReason\":\"Threshold Crossed: 1 datapoint (7.9053535353535365) was not greater than or equal to the threshold (8.0).\",\"StateChangeTime\":\"2015-11-09T21:19:43.454+0000\",\"Region\":\"US - N. Virginia\",\"OldStateValue\":\"ALARM\",\"Trigger\":{\"MetricName\":\"CPUUtilization\",\"Namespace\":\"AWS/EC2\",\"Statistic\":\"AVERAGE\",\"Unit\":null,\"Dimensions\":[],\"Period\":300,\"EvaluationPeriods\":1,\"ComparisonOperator\":\"GreaterThanOrEqualToThreshold\",\"Threshold\":8.0}}", "MessageAttributes": { "Test": { "Type": "String", "Value": "TestString" }, "TestBinary": { "Type": "Binary", "Value": "TestBinary" } }, "Type": "Notification", "UnsubscribeUrl": "EXAMPLE", "TopicArn": "arn:aws:sns:us-east-1:123456789012:production-notices", "Subject": "OK: sns-slack-test-from-cloudwatch-total-cpu" } } ] }')
|
||||
send_event(event)
|
||||
|
||||
|
||||
# CloudBender launch event
|
||||
event = json.loads(r' { "Records": [ { "EventSource": "aws:sns", "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub:0c04d2e7-32ec-4933-b913-84c7172e6d90", "Sns": { "Type": "Notification", "MessageId": "25bb2fd0-3221-5c07-aea1-76acf75017c3", "TopicArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub", "Subject": null, "Message": "{\n \"Source\": \"CloudBender\",\n \"AWSAccountId\": \"123456789012\",\n \"Region\": \"eu-central-1\",\n \"Artifact\": \"bastion\",\n \"Hostname\": \"bastion\",\n \"Asg\": \"zdt-bastion-BastionAsg-UABA2PIZV4TI\",\n \"Instance\": \"i-0d9f08855cdfcd740\",\n \"Subject\": \"Test\",\n \"Level\": \"Info\",\n \"Message\": \"Hello\",\n \"Attachment\": \"\",\n \"Emoji\": \"\"\n}\n", "Timestamp": "2021-07-29T15:03:13.318Z", "SignatureVersion": "1", "Signature": "OhLoCz8RWazyZ+ZHK03QLby6M3jmdtZvLWoNFygAHIaljQ0ZHsd6mc4TskDTnqpUpCtd/iIl+TLIPN8hYyflbLk2/cN3LXXWcSQ0GWqQ/bWIEhej54oCmUgZjIzrVfRlgz7mlUkhRnjQoRWYpcXRycQczMWuF2DCeIDP6v3ON53BxR8NdCeQXiquwoFlHaAaZIviRoUMqwp2Cl1T0NaBLeL9zmsdPvJF6EaXRbu3rqC1hdrA6E+nV2lzYNKg09POxA9JVpURmMEq3AC4tXm1Gu73PWQgWgoDSOQx+SOjMrbMeCAqf5R6typBV1BRDsGPkNkt9n4oto0FR9iyDmuWog==", "SigningCertUrl": "https://sns.eu-central-1.amazonaws.com/SimpleNotificationService-010a507c1833636cd94bdb98bd93083a.pem", "UnsubscribeUrl": "https://sns.eu-central-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-central-1:123456789012:AlertHub:0c04d2e7-32ec-4933-b913-84c7172e6d90", "MessageAttributes": {} } } ] }')
|
||||
send_event(event)
|
||||
|
||||
|
||||
# Alertmanager
|
||||
event = json.loads(r' { "Records": [ { "EventSource": "aws:sns", "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "Sns": { "Type": "Notification", "MessageId": "10ae86eb-9ddc-5c2f-806c-df6ecb6bde42", "TopicArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub", "Subject": null, "Message": "{\"receiver\":\"alerthub-notifications\",\"status\":\"firing\",\"alerts\":[{\"status\":\"firing\",\"labels\":{\"alertname\":\"KubeVersionMismatch\",\"awsRegion\":\"eu-central-1\",\"clusterName\":\"test\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"severity\":\"warning\"},\"annotations\":{\"description\":\"There are 2 different semantic versions of Kubernetes components running.\",\"runbook_url\":\"https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch\",\"summary\":\"Different semantic versions of Kubernetes components running.\"},\"startsAt\":\"2021-08-04T13:17:40.31Z\",\"endsAt\":\"0001-01-01T00:00:00Z\",\"generatorURL\":\"https://prometheus/graph?g0.expr=count%28count+by%28git_version%29+%28label_replace%28kubernetes_build_info%7Bjob%21~%22kube-dns%7Ccoredns%22%7D%2C+%22git_version%22%2C+%22%241%22%2C+%22git_version%22%2C+%22%28v%5B0-9%5D%2A.%5B0-9%5D%2A%29.%2A%22%29%29%29+%3E+1\\u0026g0.tab=1\",\"fingerprint\":\"5f94d4a22730c666\"}],\"groupLabels\":{},\"commonLabels\":{\"alertname\":\"KubeVersionMismatch\",\"awsRegion\":\"eu-central-1\",\"clusterName\":\"test\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"severity\":\"warning\"},\"commonAnnotations\":{\"description\":\"There are 2 different semantic versions of Kubernetes components running.\",\"runbook_url\":\"https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch\",\"summary\":\"Different semantic versions of Kubernetes components running.\"},\"externalURL\":\"https://alertmanager\",\"version\":\"4\",\"groupKey\":\"{}:{}\",\"truncatedAlerts\":0}\n", "Timestamp": "2021-08-05T03:01:11.233Z", "SignatureVersion": "1", "Signature": "pSUYO7LDIfzCbBrp/S2HXV3/yzls3vfYy+2di6HsKG8Mf+CV97RLnen15ieAo3eKA8IfviZIzyREasbF0cwfUeruHPbW1B8kO572fDyV206zmUxvR63r6oM6OyLv9XKBmvyYHKawkOgHZHEMP3v1wMIIHK2W5KbJtXoUcks5DVamooVb9iFF58uqTf+Ccy31bOL4tFyMR9nr8NU55vEIlGEVno8A9Q21TujdZTg0V0BmRgPafcic96udWungjmfhZ005378N32u2hlLj6BRneTpHHSXHBw4wKZreKpX+INZwiZ4P8hzVfgRvAIh/4gXN9+0UJSHgnsaqUcLDNoLZTQ==", "SigningCertUrl": "https://sns.eu-central-1.amazonaws.com/SimpleNotificationService-010a507c1833636cd94bdb98bd93083a.pem", "UnsubscribeUrl": "https://sns.eu-central-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "MessageAttributes": {} } } ] }')
|
||||
send_event(event)
|
||||
|
||||
event = json.loads(r' { "Records": [ { "EventSource": "aws:sns", "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "Sns": { "Type": "Notification", "MessageId": "10ae86eb-9ddc-5c2f-806c-df6ecb6bde42", "TopicArn": "arn:aws:sns:eu-central-1:123456789012:AlertHub", "Subject": null, "Message": "{\"receiver\":\"alerthub-notifications\",\"status\":\"resolved\",\"alerts\":[{\"status\":\"resolved\",\"labels\":{\"alertname\":\"KubeDeploymentReplicasMismatch\",\"awsAccount\":\"668666974128\",\"awsRegion\":\"us-west-2\",\"clusterName\":\"mayne-aws-dev\",\"container\":\"kube-state-metrics\",\"deployment\":\"extrude-job\",\"endpoint\":\"http\",\"instance\":\"10.244.202.71:8080\",\"job\":\"kube-state-metrics\",\"namespace\":\"default\",\"pod\":\"metrics-kube-state-metrics-56546f44c7-h57jx\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"service\":\"metrics-kube-state-metrics\",\"severity\":\"warning\"},\"annotations\":{\"description\":\"Deployment default/extrude-job has not matched the expected number of replicas for longer than 15 minutes.\",\"runbook_url\":\"https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch\",\"summary\":\"Deployment has not matched the expected number of replicas.\"},\"startsAt\":\"2021-09-29T12:36:11.394Z\",\"endsAt\":\"2021-09-29T14:51:11.394Z\",\"generatorURL\":\"https://prometheus.dev.mayneinc.com/graph?g0.expr=%28kube_deployment_spec_replicas%7Bjob%3D%22kube-state-metrics%22%7D+%3E+kube_deployment_status_replicas_available%7Bjob%3D%22kube-state-metrics%22%7D%29+and+%28changes%28kube_deployment_status_replicas_updated%7Bjob%3D%22kube-state-metrics%22%7D%5B10m%5D%29+%3D%3D+0%29\\u0026g0.tab=1\",\"fingerprint\":\"59ad2f1a4567b43b\"},{\"status\":\"firing\",\"labels\":{\"alertname\":\"KubeVersionMismatch\",\"awsRegion\":\"eu-central-1\",\"clusterName\":\"test\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"severity\":\"warning\"},\"annotations\":{\"description\":\"There are 2 different semantic versions of Kubernetes components running.\",\"runbook_url\":\"https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch\",\"summary\":\"Different semantic versions of Kubernetes components running.\"},\"startsAt\":\"2021-08-04T13:17:40.31Z\",\"endsAt\":\"0001-01-01T00:00:00Z\",\"generatorURL\":\"https://prometheus/graph?g0.expr=count%28count+by%28git_version%29+%28label_replace%28kubernetes_build_info%7Bjob%21~%22kube-dns%7Ccoredns%22%7D%2C+%22git_version%22%2C+%22%241%22%2C+%22git_version%22%2C+%22%28v%5B0-9%5D%2A.%5B0-9%5D%2A%29.%2A%22%29%29%29+%3E+1\\u0026g0.tab=1\",\"fingerprint\":\"5f94d4a22730c666\"}],\"groupLabels\":{\"job\":\"kube-state-metrics\"},\"commonLabels\":{\"alertname\":\"KubeDeploymentReplicasMismatch\",\"awsAccount\":\"668666974128\",\"awsRegion\":\"us-west-2\",\"clusterName\":\"mayne-aws-dev\",\"container\":\"kube-state-metrics\",\"deployment\":\"extrude-job\",\"endpoint\":\"http\",\"instance\":\"10.244.202.71:8080\",\"job\":\"kube-state-metrics\",\"namespace\":\"default\",\"pod\":\"metrics-kube-state-metrics-56546f44c7-h57jx\",\"prometheus\":\"monitoring/metrics-kube-prometheus-st-prometheus\",\"service\":\"metrics-kube-state-metrics\",\"severity\":\"warning\"},\"commonAnnotations\":{\"description\":\"Deployment default/extrude-job has not matched the expected number of replicas for longer than 15 minutes.\",\"runbook_url\":\"https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch\",\"summary\":\"Deployment has not matched the expected number of replicas.\"},\"externalURL\":\"https://alertmanager.dev.mayneinc.com\",\"version\":\"4\",\"groupKey\":\"{}:{job=\\\"kube-state-metrics\\\"}\",\"truncatedAlerts\":0}\n", "Timestamp": "2021-08-05T03:01:11.233Z", "SignatureVersion": "1", "Signature": "pSUYO7LDIfzCbBrp/S2HXV3/yzls3vfYy+2di6HsKG8Mf+CV97RLnen15ieAo3eKA8IfviZIzyREasbF0cwfUeruHPbW1B8kO572fDyV206zmUxvR63r6oM6OyLv9XKBmvyYHKawkOgHZHEMP3v1wMIIHK2W5KbJtXoUcks5DVamooVb9iFF58uqTf+Ccy31bOL4tFyMR9nr8NU55vEIlGEVno8A9Q21TujdZTg0V0BmRgPafcic96udWungjmfhZ005378N32u2hlLj6BRneTpHHSXHBw4wKZreKpX+INZwiZ4P8hzVfgRvAIh/4gXN9+0UJSHgnsaqUcLDNoLZTQ==", "SigningCertUrl": "https://sns.eu-central-1.amazonaws.com/SimpleNotificationService-010a507c1833636cd94bdb98bd93083a.pem", "UnsubscribeUrl": "https://sns.eu-central-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-central-1:123456789012:AlertHub:0e7ce1ba-c3e4-4264-bae1-4eb71c91235a", "MessageAttributes": {} } } ] }')
|
||||
send_event(event)
|
||||
|
||||
p.terminate()
|
||||
Loading…
Reference in New Issue