diff --git a/overlay/zdt/configs/edition/common-packages.conf b/overlay/zdt/configs/edition/common-packages.conf
index 1f4d47e..4962353 100644
--- a/overlay/zdt/configs/edition/common-packages.conf
+++ b/overlay/zdt/configs/edition/common-packages.conf
@@ -13,11 +13,9 @@ ethtool = true
nvme-cli = true
xfsprogs = true
dhclient = true
-monit = true
busybox-extras = true
tcpdump = true
uuidgen = true
-neofetch = edge-community
tiny-cloud = edge-main
tiny-cloud-openrc = edge-main
tiny-cloud-network = edge-main
@@ -25,3 +23,5 @@ tiny-cloud-aws = edge-main
conmon = edge-community
prometheus-node-exporter = true
prometheus-wireguard-exporter = true
+zdt-base = kubezero
+fluent-bit = kubezero
diff --git a/overlay/zdt/configs/edition/common-services.conf b/overlay/zdt/configs/edition/common-services.conf
index 2e5bc23..5554c40 100644
--- a/overlay/zdt/configs/edition/common-services.conf
+++ b/overlay/zdt/configs/edition/common-services.conf
@@ -6,6 +6,7 @@ sysinit {
boot {
syslog = null
syslog-ng = true
+ zdt-mount = true
}
default {
diff --git a/overlay/zdt/configs/edition/common.conf b/overlay/zdt/configs/edition/common.conf
index 588bbdd..c6bf33f 100644
--- a/overlay/zdt/configs/edition/common.conf
+++ b/overlay/zdt/configs/edition/common.conf
@@ -1,6 +1,5 @@
# vim: ts=2 et:
-scripts = [ setup-common ]
packages { include required("common-packages.conf") }
services { include required("common-services.conf") }
@@ -9,11 +8,14 @@ repos {
"https://dl-cdn.alpinelinux.org/alpine/edge/community" = edge-community
}
+repos_keys = [ "https://cdn.zero-downtime.net/alpine/stefan@zero-downtime.net-61bb6bfb.rsa.pub" ]
+
WHEN {
aws {
packages {
aws-cli = true
py3-boto3 = true
+ zdt-base-aws = kubezero
}
}
diff --git a/overlay/zdt/configs/edition/kubezero-packages-x86_64.conf b/overlay/zdt/configs/edition/kubezero-packages-x86_64.conf
new file mode 100644
index 0000000..105018e
--- /dev/null
+++ b/overlay/zdt/configs/edition/kubezero-packages-x86_64.conf
@@ -0,0 +1,2 @@
+aws-neuron-driver = kubezero
+nvidia-open-gpu = kubezero
diff --git a/overlay/zdt/configs/edition/kubezero-packages.conf b/overlay/zdt/configs/edition/kubezero-packages.conf
index e69de29..966e464 100644
--- a/overlay/zdt/configs/edition/kubezero-packages.conf
+++ b/overlay/zdt/configs/edition/kubezero-packages.conf
@@ -0,0 +1,6 @@
+cri-tools = kubezero
+cri-o = "kubezero=~1.24"
+kubelet = "kubezero=~1.24"
+kubectl = "kubezero=~1.24"
+ecr-credential-provider = "kubezero=~1.24"
+aws-iam-authenticator = "kubezero=~0.5.9"
diff --git a/overlay/zdt/configs/edition/kubezero.conf b/overlay/zdt/configs/edition/kubezero.conf
index bff927f..1eaeb1b 100644
--- a/overlay/zdt/configs/edition/kubezero.conf
+++ b/overlay/zdt/configs/edition/kubezero.conf
@@ -9,7 +9,7 @@ name = [ kubezero-1.24.7 ]
# size = 2G
WHEN {
- kubezero {
- scripts = [ setup-kubernetes ]
+ x86_64 {
+ packages { include required("kubezero-packages-x86_64.conf") }
}
}
diff --git a/overlay/zdt/configs/machine/metal.conf b/overlay/zdt/configs/machine/metal.conf
index 637f211..d64441d 100644
--- a/overlay/zdt/configs/machine/metal.conf
+++ b/overlay/zdt/configs/machine/metal.conf
@@ -6,7 +6,8 @@ machine_name = "Bare Metal"
# Kernel and firmware are 900M !
size = 2G
-EXCLUDE = ["aws"]
+# TODO: for now use AWS as nocloud is not merged yet
+#EXCLUDE = ["aws"]
packages.linux-virt = null
packages.linux-lts = true
diff --git a/overlay/zdt/configs/zdt.conf b/overlay/zdt/configs/zdt.conf
index c8392a1..c27d2bb 100644
--- a/overlay/zdt/configs/zdt.conf
+++ b/overlay/zdt/configs/zdt.conf
@@ -5,7 +5,6 @@ project = zdt-alpine
# all build configs start with these
Default {
project = ${project}
- kubeversion = 1.24
# image name/description components
name = [ zdt-alpine ]
@@ -18,7 +17,7 @@ Default {
size = 1G
login = alpine
- local_format = qcow2
+ local_format = vhd
# image access
access.PUBLIC = false
@@ -28,27 +27,40 @@ Default {
}
# profile build matrix
+# atm we only support:
+# - tiny-cloud
+# - uefi boot
+# - latest stable Alpine 3.16
+
Dimensions {
version {
- "3.16" { include required("version/3.16.conf") }
+ "3.16" { include required("version/3.16.conf")
+ repos {
+ "https://cdn.zero-downtime.net/alpine/v3.16/kubezero" = kubezero
+ }
+ }
# edge { include required("version/edge.conf") }
}
arch {
x86_64 { include required("arch/x86_64.conf") }
aarch64 { include required("arch/aarch64.conf") }
}
+
firmware {
- #bios { include required("firmware/bios.conf") }
+ # bios { include required("firmware/bios.conf") }
uefi { include required("firmware/uefi.conf") }
}
+
bootstrap {
- #cloudinit { include required("bootstrap/cloudinit.conf") }
+ # cloudinit { include required("bootstrap/cloudinit.conf") }
tiny { include required("bootstrap/tiny.conf") }
}
+
machine {
vm { include required("machine/vm.conf") }
metal { include required("machine/metal.conf") }
}
+
cloud {
aws { include required("cloud/aws.conf") }
aws.regions {
@@ -61,6 +73,7 @@ Dimensions {
us-west-2 = true
}
}
+
edition {
minimal { include required("edition/minimal.conf") }
kubezero { include required("edition/kubezero.conf") }
diff --git a/overlay/zdt/scripts/setup-common b/overlay/zdt/scripts/setup-common
deleted file mode 100755
index 4389c18..0000000
--- a/overlay/zdt/scripts/setup-common
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/bin/sh -eu
-# vim: ts=4 et:
-
-[ -z "$DEBUG" ] || [ "$DEBUG" = 0 ] || set -x
-
-SETUP=/tmp/setup.d
-TARGET=/mnt
-
-# Enable ZDT repo
-echo "@kubezero https://cdn.zero-downtime.net/alpine/v${VERSION}/kubezero" >> "$TARGET/etc/apk/repositories"
-wget -q -O $TARGET/etc/apk/keys/stefan@zero-downtime.net-61bb6bfb.rsa.pub https://cdn.zero-downtime.net/alpine/stefan@zero-downtime.net-61bb6bfb.rsa.pub
-
-# Install ZDT packages here after repo is available
-apk -U --root "$TARGET" --no-cache add \
- fluent-bit@kubezero
-
-# Install custom sysctl settings
-cp $SETUP/zdt-sysctl.conf $TARGET/etc/sysctl.d/60-zdt.conf
-
-# Fix dhcp to set MTU properly
-install -o root -g root -Dm644 -t $TARGET/etc/dhcp $SETUP/dhclient.conf
-echo 'Setup dhclient'
-
-# Enable SSH keepalive
-sed -i -e "s/^[\s#]*TCPKeepAlive\s.*/TCPKeepAlive yes/" -e "s/^[\s#]*ClientAliveInterval\s.*/ClientAliveInterval 60/" $TARGET/etc/ssh/sshd_config
-echo 'Enabled SSH keep alives'
-
-# CgroupsV2
-sed -i -e "s/^[\s#]*rc_cgroup_mode=.*/rc_cgroup_mode=\"unified\"/" $TARGET/etc/rc.conf
-
-# Setup syslog-ng json logging
-cp $SETUP/syslog-ng.conf $TARGET/etc/syslog-ng/syslog-ng.conf
-cp $SETUP/syslog-ng.logrotate.conf $TARGET/etc/logrotate.d/syslog-ng
-mkdir -p $TARGET/var/log/containers
-echo 'syslog set to json logging'
-
-# Change logrotate to run hourly rather than daily
-mv $TARGET/etc/periodic/daily/logrotate $TARGET/etc/periodic/hourly/
-echo 'Switch logrotate to run hourly rather than daily'
-
-# Install cloudbender shutdown hook
-cp $SETUP/cloudbender.stop $TARGET/etc/local.d
-mkdir -p $TARGET/etc/cloudbender/shutdown.d
-echo 'Installed cloudbender shutdown hook'
-
-# Install tools
-cp $SETUP/route53.py $TARGET/usr/local/bin
-cp $SETUP/get_iam_sshkeys.py $TARGET/usr/sbin
-echo 'Installed route53.py and get_iam_sshkeys.py'
-
-# ps_mem
-#wget https://raw.githubusercontent.com/pixelb/ps_mem/master/ps_mem.py
-#sed -i -e 's,#!/usr/bin/env python,#!/usr/bin/env python3,' ps_mem.py
-#chmod +x ps_mem.py
-cp $SETUP/ps_mem.py $TARGET/usr/local/bin/ps_mem
-echo 'Installed ps_mem'
-
-# use init to spawn monit
-echo ":2345:respawn:/usr/bin/monit -Ic /etc/monitrc" >> $TARGET/etc/inittab
-mkdir -p $TARGET/etc/monit.d
-cp $SETUP/monitrc $TARGET/etc/monitrc && chmod 600 $TARGET/etc/monitrc
-cp $SETUP/monit_alert.sh $TARGET/usr/local/bin/monit_alert.sh
-echo 'Enable monit via init, hooked up cloudbender alerting'
-
-# QoL
-mv $TARGET/etc/profile.d/color_prompt.sh.disabled $TARGET/etc/profile.d/color_prompt.sh
-echo 'alias rs="doas bash"' > $TARGET/etc/profile.d/alias.sh
-
-# branding
-cp $SETUP/neofetch.conf $TARGET/etc/neofetch.conf
-cp $SETUP/zdt-ascii.txt $TARGET/etc/neofetch-logo.txt
-echo '[ -n "$SSH_TTY" -a "$SHLVL" -eq 1 ] && neofetch --config /etc/neofetch.conf' > $TARGET/etc/profile.d/motd.sh
-echo 'Installed ZDT branding via neofetch'
-
-printf '\n# Zero Down Time config applied'
diff --git a/overlay/zdt/scripts/setup-kubernetes b/overlay/zdt/scripts/setup-kubernetes
deleted file mode 100755
index 185ab6d..0000000
--- a/overlay/zdt/scripts/setup-kubernetes
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/sh -eu
-# vim: ts=4 et:
-
-[ -z "$DEBUG" ] || [ "$DEBUG" = 0 ] || set -x
-
-SETUP=/tmp/setup.d
-TARGET=/mnt
-
-KUBE_VERSION=1.24
-AWS_IAM_VERSION=0.5.9
-
-apk -U --root "$TARGET" --no-cache add \
- cri-tools@kubezero \
- cri-o@kubezero=~$KUBE_VERSION \
- kubelet@kubezero=~$KUBE_VERSION \
- kubectl@kubezero=~$KUBE_VERSION \
- ecr-credential-provider@kubezero=~$KUBE_VERSION \
- aws-iam-authenticator@kubezero=~$AWS_IAM_VERSION
-
-# Only install custom kernel modules for X86_64
-if [ "$ARCH" == "x86_64" ]; then
- apk -U --root "$TARGET" --no-cache add \
- aws-neuron-driver@kubezero \
- nvidia-open-gpu@kubezero
-fi
-
-# Pre-load container images
-# echo 'Pre-loaded Kubernetes control container images'
-
-printf '\n\n# Zero Down Time config applied'
diff --git a/overlay/zdt/scripts/setup.d/cloudbender.stop b/overlay/zdt/scripts/setup.d/cloudbender.stop
deleted file mode 100755
index d84fd44..0000000
--- a/overlay/zdt/scripts/setup.d/cloudbender.stop
+++ /dev/null
@@ -1,15 +0,0 @@
-# Include dynamic config setting create at boot
-[ -r /etc/cloudbender/rc.conf ] && . /etc/cloudbender/rc.conf
-
-rm -f /tmp/shutdown.log
-
-for cmd in $(ls /etc/cloudbender/shutdown.d/* | sort); do
- . $cmd 1>>/tmp/shutdown.log 2>&1
-done
-
-[ $DEBUG -eq 1 ] && SHUTDOWNLOG="$(cat /tmp/shutdown.log)"
-
-[ -n "$RC_REBOOT" ] && ACTION="rebooting" || ACTION="terminated"
-[ -z "$DISABLE_SCALING_EVENTS" ] && cloudbender_sns_alarm.sh "Instance $ACTION" "" Info "$SHUTDOWNLOG"
-
-sleep ${SHUTDOWN_PAUSE:-0}
diff --git a/overlay/zdt/scripts/setup.d/dhclient.conf b/overlay/zdt/scripts/setup.d/dhclient.conf
deleted file mode 100644
index 12b6b25..0000000
--- a/overlay/zdt/scripts/setup.d/dhclient.conf
+++ /dev/null
@@ -1,12 +0,0 @@
-# Borrowed from Ubuntu 20.04LTS minimal EC2 AMi
-
-option rfc3442-classless-static-routes code 121 = array of unsigned integer 8;
-
-send host-name = gethostname();
-request subnet-mask, broadcast-address, time-offset, routers,
- domain-name, domain-name-servers, domain-search, host-name,
- dhcp6.name-servers, dhcp6.domain-search, dhcp6.fqdn, dhcp6.sntp-servers,
- netbios-name-servers, netbios-scope, interface-mtu,
- rfc3442-classless-static-routes, ntp-servers;
-
-timeout 300;
diff --git a/overlay/zdt/scripts/setup.d/get_iam_sshkeys.py b/overlay/zdt/scripts/setup.d/get_iam_sshkeys.py
deleted file mode 100755
index e459e64..0000000
--- a/overlay/zdt/scripts/setup.d/get_iam_sshkeys.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/python3
-import sys
-import boto3
-import argparse
-
-parser = argparse.ArgumentParser(description="Get SSH keys from IAM users")
-parser.add_argument(
- "--user", dest="user", action="store", required=True, help="requested user"
-)
-parser.add_argument(
- "--group", action="store", required=True, help="IAM group to search"
-)
-parser.add_argument(
- "--iamRole",
- dest="iamRole",
- action="store",
- help="IAM role ARN to assume to search for IAM users",
-)
-parser.add_argument(
- "--allowedUser",
- dest="allowedUsers",
- action="append",
- default=["alpine"],
- help="Allowed users",
-)
-args = parser.parse_args()
-
-# Fail early if invalid user
-if not args.user in args.allowedUsers:
- sys.exit(0)
-
-session = boto3.Session()
-
-if args.iamRole:
- sts = session.client("sts")
- credentials = sts.assume_role(
- RoleArn=args.iamRole, RoleSessionName="sshdKeyLookup"
- )["Credentials"]
-
- assumed_role_session = boto3.Session(
- aws_access_key_id=credentials["AccessKeyId"],
- aws_secret_access_key=credentials["SecretAccessKey"],
- aws_session_token=credentials["SessionToken"],
- )
- iam = assumed_role_session.client("iam")
-
-else:
- iam = session.client("iam")
-
-try:
- for user in iam.get_group(GroupName=args.group)["Users"]:
- for key_desc in iam.list_ssh_public_keys(UserName=user["UserName"])[
- "SSHPublicKeys"
- ]:
- key = iam.get_ssh_public_key(
- UserName=user["UserName"],
- SSHPublicKeyId=key_desc["SSHPublicKeyId"],
- Encoding="SSH",
- )
- if key["SSHPublicKey"]["Status"] == "Active":
- print(key["SSHPublicKey"]["SSHPublicKeyBody"], user["UserName"])
-except:
- pass
diff --git a/overlay/zdt/scripts/setup.d/monit_alert.sh b/overlay/zdt/scripts/setup.d/monit_alert.sh
deleted file mode 100755
index 7d0501f..0000000
--- a/overlay/zdt/scripts/setup.d/monit_alert.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-
-LEVEL=${1}
-shift
-
-ATTACHMENT="$@"
-
-if [ -n "${MONIT_SERVICE}${MONIT_EVENT}" -a -n "$MONIT_DESCRIPTION" ]; then
- /usr/local/bin/cloudbender_sns_alarm.sh "$MONIT_SERVICE - $MONIT_EVENT" "$MONIT_DESCRIPTION" $LEVEL "$ATTACHMENT"
-fi
diff --git a/overlay/zdt/scripts/setup.d/monitrc b/overlay/zdt/scripts/setup.d/monitrc
deleted file mode 100644
index 336f61b..0000000
--- a/overlay/zdt/scripts/setup.d/monitrc
+++ /dev/null
@@ -1,20 +0,0 @@
-# Give instance 3 min to settle after boot
-set daemon 30
- # Monit starts after user-data so no delay required
- # with start delay 180
-
-set log syslog
-
-set httpd port 2812 and
- use address localhost
- allow localhost
- allow admin:localmonit
-
-# Basic rootfs check
-# >80%: emergency logrotate
-# >90%: warning
-check filesystem rootfs with path /
- if space usage > 80% then exec "/etc/periodic/hourly/logrotate"
- if space usage > 90% then exec "/usr/local/bin/monit_alert.sh warning"
-
-include /etc/monit.d/*.conf
diff --git a/overlay/zdt/scripts/setup.d/neofetch.conf b/overlay/zdt/scripts/setup.d/neofetch.conf
deleted file mode 100644
index 585fa39..0000000
--- a/overlay/zdt/scripts/setup.d/neofetch.conf
+++ /dev/null
@@ -1,35 +0,0 @@
-print_info() {
- echo -e "\n"
-
- prin "$(color 1)Welcome to Alpine - ZeroDownTime edition"
- echo
- prin "Release Notes:"
- prin " - "
- prin " - "
- echo
-
- info title
- info underline
-
- info "OS" distro
- info "Host" model
- info "Kernel" kernel
- info "Uptime" uptime
- # info "Packages" packages
- info "CPU" cpu
- info "GPU" gpu
- info "Memory" memory
-
- info "GPU Driver" gpu_driver # Linux/macOS only
- info "CPU Usage" cpu_usage
- info "Local IP" local_ip
- info "Disk" disk
-
- info underline
-}
-
-title_fqdn="off"
-memory_percent="on"
-colors=(1 2 15 15 15 15)
-image_source="/etc/neofetch-logo.txt"
-gap=0
diff --git a/overlay/zdt/scripts/setup.d/ps_mem.py b/overlay/zdt/scripts/setup.d/ps_mem.py
deleted file mode 100755
index 1c92ee3..0000000
--- a/overlay/zdt/scripts/setup.d/ps_mem.py
+++ /dev/null
@@ -1,666 +0,0 @@
-#!/usr/bin/env python3
-
-# Try to determine how much RAM is currently being used per program.
-# Note per _program_, not per process. So for example this script
-# will report RAM used by all httpd process together. In detail it reports:
-# sum(private RAM for program processes) + sum(Shared RAM for program processes)
-# The shared RAM is problematic to calculate, and this script automatically
-# selects the most accurate method available for your kernel.
-
-# Licence: LGPLv2
-# Author: P@draigBrady.com
-# Source: https://www.pixelbeat.org/scripts/ps_mem.py
-
-# V1.0 06 Jul 2005 Initial release
-# V1.1 11 Aug 2006 root permission required for accuracy
-# V1.2 08 Nov 2006 Add total to output
-# Use KiB,MiB,... for units rather than K,M,...
-# V1.3 22 Nov 2006 Ignore shared col from /proc/$pid/statm for
-# 2.6 kernels up to and including 2.6.9.
-# There it represented the total file backed extent
-# V1.4 23 Nov 2006 Remove total from output as it's meaningless
-# (the shared values overlap with other programs).
-# Display the shared column. This extra info is
-# useful, especially as it overlaps between programs.
-# V1.5 26 Mar 2007 Remove redundant recursion from human()
-# V1.6 05 Jun 2007 Also report number of processes with a given name.
-# Patch from riccardo.murri@gmail.com
-# V1.7 20 Sep 2007 Use PSS from /proc/$pid/smaps if available, which
-# fixes some over-estimation and allows totalling.
-# Enumerate the PIDs directly rather than using ps,
-# which fixes the possible race between reading
-# RSS with ps, and shared memory with this program.
-# Also we can show non truncated command names.
-# V1.8 28 Sep 2007 More accurate matching for stats in /proc/$pid/smaps
-# as otherwise could match libraries causing a crash.
-# Patch from patrice.bouchand.fedora@gmail.com
-# V1.9 20 Feb 2008 Fix invalid values reported when PSS is available.
-# Reported by Andrey Borzenkov
-# V3.13 17 Sep 2018
-# https://github.com/pixelb/ps_mem/commits/master/ps_mem.py
-
-# Notes:
-#
-# All interpreted programs where the interpreter is started
-# by the shell or with env, will be merged to the interpreter
-# (as that's what's given to exec). For e.g. all python programs
-# starting with "#!/usr/bin/env python3" will be grouped under python.
-# You can change this by using the full command line but that will
-# have the undesirable affect of splitting up programs started with
-# differing parameters (for e.g. mingetty tty[1-6]).
-#
-# For 2.6 kernels up to and including 2.6.13 and later 2.4 redhat kernels
-# (rmap vm without smaps) it can not be accurately determined how many pages
-# are shared between processes in general or within a program in our case:
-# http://lkml.org/lkml/2005/7/6/250
-# A warning is printed if overestimation is possible.
-# In addition for 2.6 kernels up to 2.6.9 inclusive, the shared
-# value in /proc/$pid/statm is the total file-backed extent of a process.
-# We ignore that, introducing more overestimation, again printing a warning.
-# Since kernel 2.6.23-rc8-mm1 PSS is available in smaps, which allows
-# us to calculate a more accurate value for the total RAM used by programs.
-#
-# Programs that use CLONE_VM without CLONE_THREAD are discounted by assuming
-# they're the only programs that have the same /proc/$PID/smaps file for
-# each instance. This will fail if there are multiple real instances of a
-# program that then use CLONE_VM without CLONE_THREAD, or if a clone changes
-# its memory map while we're checksumming each /proc/$PID/smaps.
-#
-# I don't take account of memory allocated for a program
-# by other programs. For e.g. memory used in the X server for
-# a program could be determined, but is not.
-#
-# FreeBSD is supported if linprocfs is mounted at /compat/linux/proc/
-# FreeBSD 8.0 supports up to a level of Linux 2.6.16
-
-import argparse
-import errno
-import os
-import sys
-import time
-import io
-
-# The following exits cleanly on Ctrl-C or EPIPE
-# while treating other exceptions as before.
-def std_exceptions(etype, value, tb):
- sys.excepthook = sys.__excepthook__
- if issubclass(etype, KeyboardInterrupt):
- pass
- elif issubclass(etype, IOError) and value.errno == errno.EPIPE:
- pass
- else:
- sys.__excepthook__(etype, value, tb)
-sys.excepthook = std_exceptions
-
-#
-# Define some global variables
-#
-
-PAGESIZE = os.sysconf("SC_PAGE_SIZE") / 1024 #KiB
-our_pid = os.getpid()
-
-have_pss = 0
-have_swap_pss = 0
-
-class Unbuffered(io.TextIOBase):
- def __init__(self, stream):
- super(Unbuffered, self).__init__()
- self.stream = stream
- def write(self, data):
- self.stream.write(data)
- self.stream.flush()
- def close(self):
- self.stream.close()
-
-class Proc:
- def __init__(self):
- uname = os.uname()
- if uname[0] == "FreeBSD":
- self.proc = '/compat/linux/proc'
- else:
- self.proc = '/proc'
-
- def path(self, *args):
- return os.path.join(self.proc, *(str(a) for a in args))
-
- def open(self, *args):
- try:
- if sys.version_info < (3,):
- return open(self.path(*args))
- else:
- return open(self.path(*args), errors='ignore')
- except (IOError, OSError):
- if type(args[0]) is not int:
- raise
- val = sys.exc_info()[1]
- if (val.errno == errno.ENOENT or # kernel thread or process gone
- val.errno == errno.EPERM or
- val.errno == errno.EACCES):
- raise LookupError
- raise
-
-proc = Proc()
-
-
-#
-# Functions
-#
-
-def parse_options():
- help_msg = 'Show program core memory usage.'
- parser = argparse.ArgumentParser(prog='ps_mem', description=help_msg)
- parser.add_argument('--version', action='version', version='3.13')
- parser.add_argument(
- '-s', '--split-args',
- action='store_true',
- help='Show and separate by, all command line arguments',
- )
- parser.add_argument(
- '-t', '--total',
- dest='only_total',
- action='store_true',
- help='Show only the total value',
- )
- parser.add_argument(
- '-d', '--discriminate-by-pid',
- action='store_true',
- help='Show by process rather than by program',
- )
- parser.add_argument(
- '-S', '--swap',
- dest='show_swap',
- action='store_true',
- help='Show swap information',
- )
- parser.add_argument(
- '-p',
- dest='pids',
- metavar='[,pid2,...pidN]',
- help='Only show memory usage PIDs in the specified list',
- )
- parser.add_argument(
- '-w',
- dest='watch',
- metavar='',
- type=int,
- help='Measure and show process memory every N seconds',
- )
- args = parser.parse_args()
-
- args.pids_to_show = []
- if args.pids:
- try:
- args.pids_to_show = [int(x) for x in args.pids.split(',')]
- except ValueError:
- parser.error('Invalid PID(s): %s' % args.pids)
-
- if args.watch is not None:
- if args.watch <= 0:
- parser.error('Seconds must be positive! (%s)' % args.watch)
-
- return (
- args.split_args,
- args.pids_to_show,
- args.watch,
- args.only_total,
- args.discriminate_by_pid,
- args.show_swap,
- )
-
-
-# (major,minor,release)
-def kernel_ver():
- kv = proc.open('sys/kernel/osrelease').readline().split(".")[:3]
- last = len(kv)
- if last == 2:
- kv.append('0')
- last -= 1
- while last > 0:
- for char in "-_":
- kv[last] = kv[last].split(char)[0]
- try:
- int(kv[last])
- except:
- kv[last] = 0
- last -= 1
- return (int(kv[0]), int(kv[1]), int(kv[2]))
-
-
-#return Private,Shared,Swap(Pss),unique_id
-#Note shared is always a subset of rss (trs is not always)
-def getMemStats(pid):
- global have_pss
- global have_swap_pss
- mem_id = pid #unique
- Private_lines = []
- Shared_lines = []
- Private_huge_lines = []
- Shared_huge_lines = []
- Pss_lines = []
- Rss = (int(proc.open(pid, 'statm').readline().split()[1])
- * PAGESIZE)
- Swap_lines = []
- Swap_pss_lines = []
-
- Swap = 0
-
- if os.path.exists(proc.path(pid, 'smaps')): # stat
- smaps = 'smaps'
- if os.path.exists(proc.path(pid, 'smaps_rollup')):
- smaps = 'smaps_rollup' # faster to process
- lines = proc.open(pid, smaps).readlines() # open
- # Note we checksum smaps as maps is usually but
- # not always different for separate processes.
- mem_id = hash(''.join(lines))
- for line in lines:
- # {Private,Shared}_Hugetlb is not included in Pss (why?)
- # so we need to account for separately.
- if line.startswith("Private_Hugetlb:"):
- Private_huge_lines.append(line)
- elif line.startswith("Shared_Hugetlb:"):
- Shared_huge_lines.append(line)
- elif line.startswith("Shared"):
- Shared_lines.append(line)
- elif line.startswith("Private"):
- Private_lines.append(line)
- elif line.startswith("Pss:"):
- have_pss = 1
- Pss_lines.append(line)
- elif line.startswith("Swap:"):
- Swap_lines.append(line)
- elif line.startswith("SwapPss:"):
- have_swap_pss = 1
- Swap_pss_lines.append(line)
- Shared = sum([int(line.split()[1]) for line in Shared_lines])
- Private = sum([int(line.split()[1]) for line in Private_lines])
- Shared_huge = sum([int(line.split()[1]) for line in Shared_huge_lines])
- Private_huge = sum([int(line.split()[1]) for line in Private_huge_lines])
- #Note Shared + Private = Rss above
- #The Rss in smaps includes video card mem etc.
- if have_pss:
- pss_adjust = 0.5 # add 0.5KiB as this avg error due to truncation
- Pss = sum([float(line.split()[1])+pss_adjust for line in Pss_lines])
- Shared = Pss - Private
- Private += Private_huge # Add after as PSS doesn't a/c for huge pages
- if have_swap_pss:
- # The kernel supports SwapPss, that shows proportional swap share.
- # Note that Swap - SwapPss is not Private Swap.
- Swap = sum([int(line.split()[1]) for line in Swap_pss_lines])
- else:
- # Note that Swap = Private swap + Shared swap.
- Swap = sum([int(line.split()[1]) for line in Swap_lines])
- elif (2,6,1) <= kernel_ver() <= (2,6,9):
- Shared = 0 #lots of overestimation, but what can we do?
- Shared_huge = 0
- Private = Rss
- else:
- Shared = int(proc.open(pid, 'statm').readline().split()[2])
- Shared *= PAGESIZE
- Shared_huge = 0
- Private = Rss - Shared
- return (Private, Shared, Shared_huge, Swap, mem_id)
-
-
-def getCmdName(pid, split_args, discriminate_by_pid, exe_only=False):
- cmdline = proc.open(pid, 'cmdline').read().split("\0")
- while cmdline[-1] == '' and len(cmdline) > 1:
- cmdline = cmdline[:-1]
-
- path = proc.path(pid, 'exe')
- try:
- path = os.readlink(path)
- # Some symlink targets were seen to contain NULs on RHEL 5 at least
- # https://github.com/pixelb/scripts/pull/10, so take string up to NUL
- path = path.split('\0')[0]
- except OSError:
- val = sys.exc_info()[1]
- if (val.errno == errno.ENOENT or # either kernel thread or process gone
- val.errno == errno.EPERM or
- val.errno == errno.EACCES):
- raise LookupError
- raise
-
- if split_args:
- return ' '.join(cmdline).replace('\n', ' ')
- if path.endswith(" (deleted)"):
- path = path[:-10]
- if os.path.exists(path):
- path += " [updated]"
- else:
- #The path could be have prelink stuff so try cmdline
- #which might have the full path present. This helped for:
- #/usr/libexec/notification-area-applet.#prelink#.fX7LCT (deleted)
- if os.path.exists(cmdline[0]):
- path = cmdline[0] + " [updated]"
- else:
- path += " [deleted]"
- exe = os.path.basename(path)
- if exe_only: return exe
-
- proc_status = proc.open(pid, 'status').readlines()
- cmd = proc_status[0][6:-1]
- if exe.startswith(cmd):
- cmd = exe #show non truncated version
- #Note because we show the non truncated name
- #one can have separated programs as follows:
- #584.0 KiB + 1.0 MiB = 1.6 MiB mozilla-thunder (exe -> bash)
- # 56.0 MiB + 22.2 MiB = 78.2 MiB mozilla-thunderbird-bin
- else:
- #Lookup the parent's exe and use that if matching
- #which will merge "Web Content" with "firefox" for example
- ppid = 0
- for l in range(10):
- ps_line = proc_status[l]
- if ps_line.startswith('PPid:'):
- ppid = int(ps_line[6:-1])
- break
- if ppid:
- try:
- p_exe = getCmdName(ppid, False, False, exe_only=True)
- except LookupError:
- pass
- else:
- if exe == p_exe:
- cmd = exe
- if sys.version_info >= (3,):
- cmd = cmd.encode(errors='replace').decode()
- if discriminate_by_pid:
- cmd = '%s [%d]' % (cmd, pid)
- return cmd
-
-
-#The following matches "du -h" output
-#see also human.py
-def human(num, power="Ki", units=None):
- if units is None:
- powers = ["Ki", "Mi", "Gi", "Ti"]
- while num >= 1000: #4 digits
- num /= 1024.0
- power = powers[powers.index(power)+1]
- return "%.1f %sB" % (num, power)
- else:
- return "%.f" % ((num * 1024) / units)
-
-
-def cmd_with_count(cmd, count):
- if count > 1:
- return "%s (%u)" % (cmd, count)
- else:
- return cmd
-
-#Warn of possible inaccuracies
-#RAM:
-#2 = accurate & can total
-#1 = accurate only considering each process in isolation
-#0 = some shared mem not reported
-#-1= all shared mem not reported
-#SWAP:
-#2 = accurate & can total
-#1 = accurate only considering each process in isolation
-#-1= not available
-def val_accuracy(show_swap):
- """http://wiki.apache.org/spamassassin/TopSharedMemoryBug"""
- kv = kernel_ver()
- pid = os.getpid()
- swap_accuracy = -1
- if kv[:2] == (2,4):
- if proc.open('meminfo').read().find("Inact_") == -1:
- return 1, swap_accuracy
- return 0, swap_accuracy
- elif kv[:2] == (2,6):
- if os.path.exists(proc.path(pid, 'smaps')):
- swap_accuracy = 1
- if proc.open(pid, 'smaps').read().find("Pss:")!=-1:
- return 2, swap_accuracy
- else:
- return 1, swap_accuracy
- if (2,6,1) <= kv <= (2,6,9):
- return -1, swap_accuracy
- return 0, swap_accuracy
- elif kv[0] > 2 and os.path.exists(proc.path(pid, 'smaps')):
- swap_accuracy = 1
- if show_swap and proc.open(pid, 'smaps').read().find("SwapPss:")!=-1:
- swap_accuracy = 2
- return 2, swap_accuracy
- else:
- return 1, swap_accuracy
-
-def show_val_accuracy( ram_inacc, swap_inacc, only_total, show_swap ):
- level = ("Warning","Error")[only_total]
-
- # Only show significant warnings
- if not show_swap:
- swap_inacc = 2
- elif only_total:
- ram_inacc = 2
-
- if ram_inacc == -1:
- sys.stderr.write(
- "%s: Shared memory is not reported by this system.\n" % level
- )
- sys.stderr.write(
- "Values reported will be too large, and totals are not reported\n"
- )
- elif ram_inacc == 0:
- sys.stderr.write(
- "%s: Shared memory is not reported accurately by this system.\n" % level
- )
- sys.stderr.write(
- "Values reported could be too large, and totals are not reported\n"
- )
- elif ram_inacc == 1:
- sys.stderr.write(
- "%s: Shared memory is slightly over-estimated by this system\n"
- "for each program, so totals are not reported.\n" % level
- )
-
- if swap_inacc == -1:
- sys.stderr.write(
- "%s: Swap is not reported by this system.\n" % level
- )
- elif swap_inacc == 1:
- sys.stderr.write(
- "%s: Swap is over-estimated by this system for each program,\n"
- "so totals are not reported.\n" % level
- )
-
- sys.stderr.close()
- if only_total:
- if show_swap:
- accuracy = swap_inacc
- else:
- accuracy = ram_inacc
- if accuracy != 2:
- sys.exit(1)
-
-
-def get_memory_usage(pids_to_show, split_args, discriminate_by_pid,
- include_self=False, only_self=False):
- cmds = {}
- shareds = {}
- shared_huges = {}
- mem_ids = {}
- count = {}
- swaps = {}
- for pid in os.listdir(proc.path('')):
- if not pid.isdigit():
- continue
- pid = int(pid)
-
- # Some filters
- if only_self and pid != our_pid:
- continue
- if pid == our_pid and not include_self:
- continue
- if pids_to_show and pid not in pids_to_show:
- continue
-
- try:
- cmd = getCmdName(pid, split_args, discriminate_by_pid)
- except LookupError:
- #operation not permitted
- #kernel threads don't have exe links or
- #process gone
- continue
-
- try:
- private, shared, shared_huge, swap, mem_id = getMemStats(pid)
- except RuntimeError:
- continue #process gone
- if shareds.get(cmd):
- if have_pss: #add shared portion of PSS together
- shareds[cmd] += shared
- elif shareds[cmd] < shared: #just take largest shared val
- shareds[cmd] = shared
- else:
- shareds[cmd] = shared
- if shared_huges.get(cmd):
- if shared_huges[cmd] < shared_huge: #just take largest shared_huge
- shared_huges[cmd] = shared_huge
- else:
- shared_huges[cmd] = shared_huge
- cmds[cmd] = cmds.setdefault(cmd, 0) + private
- if cmd in count:
- count[cmd] += 1
- else:
- count[cmd] = 1
- mem_ids.setdefault(cmd, {}).update({mem_id: None})
-
- # Swap (overcounting for now...)
- swaps[cmd] = swaps.setdefault(cmd, 0) + swap
-
- # Total swaped mem for each program
- total_swap = 0
-
- # Add shared mem for each program
- total = 0
-
- for cmd in cmds:
- cmd_count = count[cmd]
- if len(mem_ids[cmd]) == 1 and cmd_count > 1:
- # Assume this program is using CLONE_VM without CLONE_THREAD
- # so only account for one of the processes
- cmds[cmd] /= cmd_count
- if have_pss:
- shareds[cmd] /= cmd_count
- # overestimation possible if shared_huges shared across commands
- shareds[cmd] += shared_huges[cmd]
- cmds[cmd] = cmds[cmd] + shareds[cmd]
- total += cmds[cmd] # valid if PSS available
- total_swap += swaps[cmd]
-
- sorted_cmds = sorted(cmds.items(), key=lambda x:x[1])
- sorted_cmds = [x for x in sorted_cmds if x[1]]
-
- return sorted_cmds, shareds, count, total, swaps, total_swap
-
-def print_header(show_swap, discriminate_by_pid):
- output_string = " Private + Shared = RAM used"
- if show_swap:
- output_string += " Swap used"
- output_string += "\tProgram"
- if discriminate_by_pid:
- output_string += "[pid]"
- output_string += "\n\n"
- sys.stdout.write(output_string)
-
-
-def print_memory_usage(sorted_cmds, shareds, count, total, swaps, total_swap,
- show_swap):
- for cmd in sorted_cmds:
-
- output_string = "%9s + %9s = %9s"
- output_data = (human(cmd[1]-shareds[cmd[0]]),
- human(shareds[cmd[0]]), human(cmd[1]))
- if show_swap:
- output_string += " %9s"
- output_data += (human(swaps[cmd[0]]),)
- output_string += "\t%s\n"
- output_data += (cmd_with_count(cmd[0], count[cmd[0]]),)
-
- sys.stdout.write(output_string % output_data)
-
- # Only show totals if appropriate
- if have_swap_pss and show_swap: # kernel will have_pss
- sys.stdout.write("%s\n%s%9s%s%9s\n%s\n" %
- ("-" * 45, " " * 24, human(total), " " * 3,
- human(total_swap), "=" * 45))
- elif have_pss:
- sys.stdout.write("%s\n%s%9s\n%s\n" %
- ("-" * 33, " " * 24, human(total), "=" * 33))
-
-
-def verify_environment(pids_to_show):
- if os.geteuid() != 0 and not pids_to_show:
- sys.stderr.write("Sorry, root permission required, or specify pids with -p\n")
- sys.stderr.close()
- sys.exit(1)
-
- try:
- kernel_ver()
- except (IOError, OSError):
- val = sys.exc_info()[1]
- if val.errno == errno.ENOENT:
- sys.stderr.write(
- "Couldn't access " + proc.path('') + "\n"
- "Only GNU/Linux and FreeBSD (with linprocfs) are supported\n")
- sys.exit(2)
- else:
- raise
-
-def main():
- # Force the stdout and stderr streams to be unbuffered
- sys.stdout = Unbuffered(sys.stdout)
- sys.stderr = Unbuffered(sys.stderr)
-
- split_args, pids_to_show, watch, only_total, discriminate_by_pid, \
- show_swap = parse_options()
-
- verify_environment(pids_to_show)
-
- if not only_total:
- print_header(show_swap, discriminate_by_pid)
-
- if watch is not None:
- try:
- sorted_cmds = True
- while sorted_cmds:
- sorted_cmds, shareds, count, total, swaps, total_swap = \
- get_memory_usage(pids_to_show, split_args,
- discriminate_by_pid)
- if only_total and show_swap and have_swap_pss:
- sys.stdout.write(human(total_swap, units=1)+'\n')
- elif only_total and not show_swap and have_pss:
- sys.stdout.write(human(total, units=1)+'\n')
- elif not only_total:
- print_memory_usage(sorted_cmds, shareds, count, total,
- swaps, total_swap, show_swap)
-
- sys.stdout.flush()
- time.sleep(watch)
- else:
- sys.stdout.write('Process does not exist anymore.\n')
- except KeyboardInterrupt:
- pass
- else:
- # This is the default behavior
- sorted_cmds, shareds, count, total, swaps, total_swap = \
- get_memory_usage(pids_to_show, split_args,
- discriminate_by_pid)
- if only_total and show_swap and have_swap_pss:
- sys.stdout.write(human(total_swap, units=1)+'\n')
- elif only_total and not show_swap and have_pss:
- sys.stdout.write(human(total, units=1)+'\n')
- elif not only_total:
- print_memory_usage(sorted_cmds, shareds, count, total, swaps,
- total_swap, show_swap)
-
- # We must close explicitly, so that any EPIPE exception
- # is handled by our excepthook, rather than the default
- # one which is reenabled after this script finishes.
- sys.stdout.close()
-
- ram_accuracy, swap_accuracy = val_accuracy( show_swap )
- show_val_accuracy( ram_accuracy, swap_accuracy, only_total, show_swap )
-
-if __name__ == '__main__': main()
diff --git a/overlay/zdt/scripts/setup.d/route53.py b/overlay/zdt/scripts/setup.d/route53.py
deleted file mode 100755
index 20424de..0000000
--- a/overlay/zdt/scripts/setup.d/route53.py
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env python3
-import sys
-import boto3
-import json
-import argparse
-
-
-def update_dns(record_name, ips=[], ttl=180, action="UPSERT", record_type='A'):
- route53 = boto3.client("route53")
- zone_id = route53.list_hosted_zones_by_name(
- DNSName=".".join(record_name.split(".")[1:])
- )["HostedZones"][0]["Id"]
-
- changeset = {
- "Changes": [
- {
- "Action": action,
- "ResourceRecordSet": {
- "Name": record_name,
- "Type": record_type,
- "TTL": ttl,
- "ResourceRecords": [],
- },
- }
- ]
- }
- for ip in ips:
- changeset["Changes"][0]["ResourceRecordSet"]["ResourceRecords"].append(
- {"Value": ip}
- )
-
- route53.change_resource_record_sets(HostedZoneId=zone_id, ChangeBatch=changeset)
-
-
-parser = argparse.ArgumentParser(description='Update Route53 entries')
-parser.add_argument('--fqdn', dest='fqdn', action='store', required=True,
- help='FQDN for this record')
-parser.add_argument('--record', action='append', required=True,
- help='Value of a record')
-parser.add_argument('--type', dest='record_type', action='store', default='A',
- help='Record type')
-parser.add_argument('--ttl', dest='ttl', action='store', default=180, type=int,
- help='TTL of the entry')
-parser.add_argument('--delete', dest='delete', action='store_true',
- help='delete entry')
-
-args = parser.parse_args()
-action = "UPSERT"
-if args.delete:
- action = "DELETE"
-
-update_dns(args.fqdn, args.record, action=action, ttl=args.ttl, record_type=args.record_type)
diff --git a/overlay/zdt/scripts/setup.d/syslog-ng.conf b/overlay/zdt/scripts/setup.d/syslog-ng.conf
deleted file mode 100644
index 0570d26..0000000
--- a/overlay/zdt/scripts/setup.d/syslog-ng.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-# syslog-ng, format all json into messages
-# https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.23/administration-guide/63#TOPIC-1268643
-
-@version: 3.36
-@include "scl.conf"
-
-options { chain_hostnames(off); flush_lines(0); use_dns(no); use_fqdn(no);
- dns_cache(no); owner("root"); group("adm"); perm(0640);
- stats_freq(0); bad_hostname("^gconfd$"); frac-digits(6);
-};
-
-source s_sys { system(); internal();};
-
-destination d_mesg { file("/var/log/messages" template("$(format-json time=\"$UNIXTIME\" facility=\"$FACILITY\" host=\"$LOGHOST\" ident=\"$PROGRAM\" pid=\"$PID\" level=\"$PRIORITY\" message=\"$MESSAGE\")\n")); };
-
-# filter ipvs loggging each SYN to closed port
-# IPVS: rr: TCP 10.52.82.199:31021 - no destination available
-filter f_drop_ipvs { not (facility(kern) and match("IPVS: rr:.*no destination available" value("MESSAGE"))); };
-# "message":"net_ratelimit: 16 callbacks suppressed"
-filter f_drop_ipvs_ratelimit { not (facility(kern) and match("net_ratelimit:.*callbacks suppressed" value("MESSAGE"))); };
-
-log { source(s_sys); filter(f_drop_ipvs); filter(f_drop_ipvs_ratelimit); destination(d_mesg); };
diff --git a/overlay/zdt/scripts/setup.d/syslog-ng.logrotate.conf b/overlay/zdt/scripts/setup.d/syslog-ng.logrotate.conf
deleted file mode 100644
index 8628ee8..0000000
--- a/overlay/zdt/scripts/setup.d/syslog-ng.logrotate.conf
+++ /dev/null
@@ -1,13 +0,0 @@
-/var/log/messages
-{
- nodateext
- rotate 2
- missingok
- notifempty
- compress
- maxsize 32M
- sharedscripts
- postrotate
- rc-service syslog-ng reload > /dev/null
- endscript
-}
diff --git a/overlay/zdt/scripts/setup.d/zdt-ascii.txt b/overlay/zdt/scripts/setup.d/zdt-ascii.txt
deleted file mode 100644
index b3e74e5..0000000
--- a/overlay/zdt/scripts/setup.d/zdt-ascii.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-\x1b[38;2;32;120;108m ..
- ox@@@@@x
- -x@@@@@@@@@@@@x-
- .x@@@x- -x@@@x.
- ox@@@@x ox@@@@@@@@x- x@@@@xo
- @@@@@@@@ x@@@@@@@@@@@@x @@@@@@@@o
- o @@@@@x -x@@@@@@@@x- x@@@@@ o
- @@@x- @@@x- -x@@@o .x@@@.
- @@@@@@@x. x@@@@@@@@@@@@@ o@@@@@@@.
- @@@@@@@@@@xo @@@@@@ -x@@@@@xx@@@.
- @@@@@@@@@@@@@@x- .x@@@@@x- -@@@.
- @@@@@@@@ @@@@@@@@ x@@@@@x ox x@@@.
- @@@@ -@@ @@@@@@@@ @@@@ -x@@x .@@@@@.
- @@@@x @ @@@@@@@@ @@@x .-o .x@@@.
- @@@@@x @xoo@@@@ @@@@xx .xxx -@@@.
- @@@@@@@ @@@@ @@@@@ x@xo x@@@.
- @@@@@@@. -x@@@@@@ @@@@ x. -x@@@@@
- .@@@@@@@@@@@@@ @@@x .x@@@@@x
- x@@@@@@@@@ @@@@x@@@@@
- @@@@@@ @@@@@@.
- o@@ @@x
diff --git a/overlay/zdt/scripts/setup.d/zdt-sysctl.conf b/overlay/zdt/scripts/setup.d/zdt-sysctl.conf
deleted file mode 100644
index c9de88d..0000000
--- a/overlay/zdt/scripts/setup.d/zdt-sysctl.conf
+++ /dev/null
@@ -1,16 +0,0 @@
-net.core.somaxconn = 1024
-net.core.netdev_max_backlog = 4096
-net.core.rmem_max = 16777216
-net.core.wmem_max = 16777216
-net.ipv4.tcp_wmem = 4096 12582912 16777216
-net.ipv4.tcp_rmem = 4096 12582912 16777216
-net.ipv4.tcp_max_syn_backlog = 8192
-net.ipv4.tcp_tw_reuse = 1
-net.ipv4.tcp_retries2 = 9
-net.ipv4.tcp_slow_start_after_idle = 0
-net.netfilter.nf_conntrack_max = 524288
-net.ipv4.ip_no_pmtu_disc = 0
-net.ipv4.ip_forward_use_pmtu = 0
-kernel.panic = 10
-kernel.panic_on_oops = 1
-vm.oom_dump_tasks = 0