Add dev-util/bob

dev-util/bob/build.sh

@@ -0,0 +1,474 @@
+#!/bin/bash
+
+# - builds a generic image from a stage3 tarball and portage snapshot
+# 
+# based on work from JD Harrington https://github.com/psi/gentoo-aws
+# and Matsuu Takuto https://gist.github.com/870789
+
+#===============================================================================
+GENTOO_MIRROR="http://distfiles.gentoo.org"
+LOCAL_CACHE=/var/tmp
+CHROOT=/mnt/gentoo
+
+
+set -o nounset
+
+die() {
+	echo $@
+	exit 1
+}
+
+
+verify_digest() {
+	local FILE_NAME=$1
+	local TYPE=$2
+	local DIGEST=$3
+	local PURGE=$4
+
+	case $TYPE in
+		sha512)
+			FILE_DIGEST=$(sha512sum ${FILE_NAME} | awk '{print $1}') 
+			;;
+		md5)
+			FILE_DIGEST=$(md5sum ${FILE_NAME} | awk '{print $1}') 
+			;;
+		*)
+			echo "Unknown DIGEST"
+			return 1
+			;;
+	esac
+
+	if [ "${FILE_DIGEST}" = "${DIGEST}" ]; then
+		echo "${TYPE} checksum for ${FILE_NAME} verified."
+		return 0
+	else
+		echo "Invalid checksum for ${FILE_NAME}!"
+		[ "x${PURGE}" != "x" ] && { echo "Removing cached copy." ; rm -f ${FILE_NAME}; }
+		return 1
+	fi
+}
+
+
+fetch_file() {
+	local URL=$1
+	local DIGEST_URL=$2
+	
+	local FILE_NAME=$(basename ${URL})
+
+	local DIGEST_TYPE
+
+	# Check cache
+	if [ ! -d ${LOCAL_CACHE} ] || [ ! -w ${LOCAL_CACHE} ]; then
+		echo "Cannot write to cache ${LOCAL_CACHE}" 
+		return 1
+	fi
+
+	# If DIGEST requested get it
+	if [ ! -z ${DIGEST_URL} ]; then
+		DIGEST=$(curl -s -S ${DIGEST_URL} | grep -A1 -e "^# SHA512 HASH" | grep -o -E -e "^[0-9a-f]{128} *${FILE_NAME}$" | awk '{print $1}')
+		if [ -z ${DIGEST} ]; then
+			# Let's try md5sum before giving up
+			DIGEST=$(curl -s -S ${DIGEST_URL} | grep -o -E -e "^[0-9a-f]{32} *${FILE_NAME}$" | awk '{print $1}')
+			if [ -z ${DIGEST} ]; then
+				echo "Unable to get checksum for ${FILE_NAME}, abort"
+				return 2
+			fi
+			DIGEST_TYPE="md5"
+		else
+			DIGEST_TYPE="sha512"
+		fi
+	fi
+
+	# Do we have local copy
+	if [ -f ${LOCAL_CACHE}/${FILE_NAME} ]; then
+		verify_digest ${LOCAL_CACHE}/${FILE_NAME} $DIGEST_TYPE ${DIGEST} 1
+		if [ $? -eq 0 ]; then
+			cp ${LOCAL_CACHE}/${FILE_NAME} .
+			echo "Using cached ${LOCAL_CACHE}/${FILE_NAME}"
+			return 0
+		fi
+	fi
+
+	# We we are here either we didnt have a copy or the cached file was invalid
+	wget -q -O ${LOCAL_CACHE}/${FILE_NAME} ${URL}
+	if [ $? -eq 0 ]; then
+		echo "Downloaded ${URL} to ${LOCAL_CACHE}/${FILE_NAME}"
+
+		verify_digest ${LOCAL_CACHE}/${FILE_NAME} $DIGEST_TYPE ${DIGEST} 1
+		if [ $? -ne 0 ]; then
+			echo "Could not get a verified version of ${FILE_NAME}"
+			return 3
+		fi
+		cp ${LOCAL_CACHE}/${FILE_NAME} .
+	else
+		echo "Cannot download ${URL}!"
+		return 4
+	fi
+}
+
+
+# bootstrap, download stage3 and portage snapshot
+bootstrap() {
+
+	[ -d ${CHROOT} ] || die "${CHROOT} does not exists"
+	[ -w ${CHROOT} ] || die "${CHROOT} isn't writable"
+
+	pushd ${CHROOT} > /dev/null
+
+	# first install stage 3
+	if [ -d "usr" ] ; then
+		echo "There seems to be already files in ${CHROOT} !"
+		echo "Press <Ctrl+c> to abort, or <Return> to proceed without extracting stage3 ..."
+		read -r REPLY
+	else
+		STAGE_TARBALL=${GENTOO_MIRROR}/releases/${ARCH}/autobuilds/$(curl -s ${LATEST_STAGE_FILE} | grep -v "^#" | head -n 1) 
+
+		fetch_file "${STAGE_TARBALL}" "${STAGE_TARBALL}.DIGESTS" || die "Cannot get ${STAGE_TARBALL}"
+
+		echo "Extracting stage3 to ${CHROOT} ..."
+		tar jxpf $(basename ${STAGE_TARBALL}) || die "Extracting stage3 failed"
+
+		rm -f $(basename ${STAGE_TARBALL})
+	fi
+
+	# Portage snapshot
+	PORTAGE_SNAPSHOT="${GENTOO_MIRROR}/releases/snapshots/current/portage-latest.tar.bz2"
+	_PORTAGE_MOUNTED=0
+
+	if [ "x${BIND_PORTAGE}" != "x" ] ; then
+		mkdir -p ${CHROOT}/usr/portage
+		mount --bind ${BIND_PORTAGE} ${CHROOT}/usr/portage || die "Error mounting ${BIND_PORTAGE}"
+
+		# Remember we mounted portage
+		_PORTAGE_MOUNTED=1
+	else
+		# install latest portage snapshot
+		if [ -d "usr/portage" ] ; then
+			echo "There seems to be already portage files!"
+			echo "Press <Ctrl+c> to abort, or <Return> to proceed without extracting portage ..."
+			read -r REPLY
+		else
+			fetch_file "${PORTAGE_SNAPSHOT}" "${PORTAGE_SNAPSHOT}.md5sum"
+			echo "Extracting latest portage snapshot to ${CHROOT}/usr ..."
+			tar jxf $(basename ${PORTAGE_SNAPSHOT}) -C "${CHROOT}/usr" || die "Extracting portage snapshot failed"
+			rm -f portage-latest.tar.bz2
+		fi
+	fi
+
+	popd
+}
+
+
+# setup_chroot CHROOT
+setup_chroot() {
+
+	# mount pseudo filesystems
+	echo "Mounting pseudo filesystems..."
+	mount -t proc none ${CHROOT}/proc || die "Error mounting /proc"
+	mount --rbind /dev ${CHROOT}/dev || die "Error mounting /dev"
+	mount --rbind /sys ${CHROOT}/sys || die "Error mounting /sys"
+
+	# Needed at least to emerge python
+	mkdir -p ${CHROOT}/run/shm && mount -t tmpfs none ${CHROOT}/run/shm 
+
+	# resolve.conf
+	echo "Installing resolv.conf..."
+	cp -L /etc/resolv.conf ${CHROOT}/etc/resolv.conf || die "Can't copy resolv.conf"
+}
+
+
+# Actually prepare the install script running within chroot
+# and run it
+# https://www.gentoo.org/doc/en/handbook/handbook-amd64.xml?full=1#book_part1_chap6
+install_gentoo() {
+
+	if [ ${INTERACTIVE} = 1 ]; then
+		echo "Done. Entering chroot environment. All yours..."
+		chroot ${CHROOT} /bin/bash
+	else
+		# Install make.conf
+		if [ "x${MAKE_CONF}" != "x" ]; then
+			[ -r ${MAKE_CONF} ] || die "Cannot read custom make.conf ${MAKE_CONF}"
+
+			echo "Using custom make.conf"
+			cp ${MAKE_CONF} ${CHROOT}/etc/portage/ 
+		fi
+
+		[ -r ${KERNEL_CONF} ] || die "Cannot read kernel config ${KERNEL_CONF}"
+		cp ${KERNEL_CONF} ${CHROOT}/tmp/.config
+
+# Hack
+cp dracut.tgz ${CHROOT}/tmp/
+
+		# Install automatic CPU USE detection
+		echo "Installing CPU USE flags detection"
+		cat << 'EOF' > ${CHROOT}/etc/profile.d/portage.sh
+# use as many cores as available
+export MAKEOPTS="-j$(nproc)"
+
+# USE
+# Look at CPU and return all actually available USE flags
+get_cpu_use() {
+	local _USE=""
+
+	local AVAILABLE_USE="mmx mmxext sse sse2 sse3 ssse3 sse4 sse4_1 avx 3dnow 3dnowext"
+	local CPU_FLAGS=$(cat /proc/cpuinfo | grep flags | cut -d" " -f2- | uniq)
+
+	for f in ${AVAILABLE_USE}; do
+		if [ "$CPU_FLAGS" != "${CPU_FLAGS/$f/}" ]; then
+			_USE="$_USE $f"
+		fi
+	done
+	echo $_USE
+}
+
+export USE=$(get_cpu_use)
+EOF
+
+		# From here we create the install script and execute it within the chroot at the end
+		cat << 'EOF' > ${CHROOT}/tmp/install.sh
+#!/bin/bash
+set -e
+source /etc/profile
+export PS1="(chroot) $PS1"
+
+CPUS=$(nproc)
+[ $CPUS -ge 4 ] && JOBS="--jobs $((CPUS/2))"
+EOF
+		# Sync portage if not mounted
+		if [ ${_PORTAGE_MOUNTED} = 0 ]; then
+		cat << 'EOF' >> ${CHROOT}/tmp/install.sh
+echo "Syncing portage snapshot..."
+emerge --sync --quiet
+mkdir -p /usr/portage/distfiles
+chown portage: /usr/portage/distfiles
+EOF
+		fi
+
+		# eselect profile
+		cat << EOF >> ${CHROOT}/tmp/install.sh
+echo "Setting profile to ${ESELECT_PROFILE}"
+eselect profile set ${ESELECT_PROFILE}
+
+echo "Setting timezone to ${TIMEZONE}"
+echo ${TIMEZONE} > /etc/timezone
+emerge --config sys-libs/timezone-data
+
+echo "Setting locale to C"
+eselect locale set C
+source /etc/profile
+EOF
+
+		# Updating world, using our make.conf and settings
+		cat << 'EOF' >> ${CHROOT}/tmp/install.sh
+echo "Updating stage3 to latest versions and new USE flags..."
+emerge --update --deep --newuse --accept-properties=-interactive --with-bdeps=y ${JOBS} world
+dispatch-conf
+EOF
+
+		# Compile kernel
+
+		# Unmask dracut
+		echo "sys-kernel/dracut" >> ${CHROOT}/etc/portage/package.keywords
+
+		# Disable dhcp server part
+		echo "net-misc/dhcp -server" >> ${CHROOT}/etc/portage/package.use
+		# Disable thin provisioning in order to NOT have to add boost
+		echo "sys-fs/lvm2 -thin" >> ${CHROOT}/etc/portage/package.use
+
+		# Dracut needs dhcp rather than dhcpcd
+		cat << EOF >> ${CHROOT}/tmp/install.sh
+echo "Installing kernel sources, dracut and dependencies..."
+emerge -n sys-kernel/${KERNEL_FLAVOR}-sources
+mv /tmp/.config /usr/src/linux
+
+emerge -n ${JOBS} net-misc/dhcp net-misc/curl sys-apps/iproute2 sys-fs/cryptsetup app-crypt/gnupg sys-fs/lvm2 sys-fs/mdadm sys-boot/grub sys-kernel/dracut
+
+# Stuff for encrypted root
+emerge -n ${JOBS} net-misc/dropbear sys-fs/eudev 
+EOF
+		# Now lets actually build and install the kernel incl. initrd
+		cat << 'EOF' >> ${CHROOT}/tmp/install.sh
+echo "Compiling custom kernel..."
+cd /usr/src/linux
+make olddefconfig
+#make menuconfig
+make all ${MAKEOPTS}
+make modules_install
+
+NEW=$(make kernelversion)
+
+#Hack
+cd ..
+tar xfz /tmp/dracut.tgz
+cd dracut
+./dracut.sh -l --xz --force --strip -m "bash caps i18n terminfo base shutdown usrmount kernel-modules lvm mdraid crypt crypt-ssh rescue" /boot/initrd-${NEW} ${NEW}
+cd ../linux 
+
+#dracut --xz --force --strip /boot/initrd-${NEW} ${NEW}
+
+make install
+
+pushd /boot >/dev/null
+ln -sf System.map-${NEW} System.map
+popd > /dev/null
+
+echo "Installed custom kernel"
+
+echo "Writing /etc/fstab"
+echo -e "UUID=$(findmnt /boot -o uuid -n)\t/boot\txfs\tdiscard,noatime\t0 1" > /etc/fstab
+echo -e "UUID=$(findmnt / -o uuid -n)\t/\txfs\tdiscard,noatime\t0 1" >> /etc/fstab
+
+echo "Setting eth0 to dhcp"
+echo 'config_eth0="dhcp"' > /etc/conf.d/net
+cd /etc/init.d
+ln -s net.lo net.eth0
+rc-update add net.eth0 default
+
+echo "Enabling sshd..."
+rc-update add sshd default
+
+echo "Installing various tools..."
+emerge -n ${JOBS} sys-fs/xfsprogs sys-process/cronie app-admin/sudo app-editors/vim app-misc/screen app-portage/eix app-portage/gentoolkit net-misc/ntp app-admin/syslog-ng
+rc-update add cronie default
+rc-update add ntpd default
+rc-update add lvm boot
+rc-update add mdraid boot
+
+echo "Finally installing grub bootloader on /dev/sda..."
+echo -e "GRUB_HIDDEN_TIMEOUT=1" >> /etc/default/grub
+echo -e "GRUB_HIDDEN_TIMEOUT_QUIET=true" >> /etc/default/grub
+echo -e "GRUB_FORCE_HIDDEN_MENU=true" >> /etc/default/grub
+echo -e "GRUB_DISABLE_RECOVERY=true" >> /etc/default/grub
+echo -e "GRUB_TERMINAL=console" >> /etc/default/grub
+echo -e "GRUB_CMDLINE_LINUX_DEFAULT=\"nosplash rd.luks.uuid=$(blkid  | grep crypto_LUKS | awk '{print $2}' | sed -e 's/UUID=//' | sed -e 's/"//g') rd.lvm.vg=disks rd.luks.allow-discards cgroup_disable=memory rd.ip=dhcp rd.neednet=1 rd.luks.ssh=1\"" >> /etc/default/grub
+
+grub2-install /dev/sda
+grub2-mkconfig -o /boot/grub/grub.cfg
+EOF
+		chmod 755 ${CHROOT}/tmp/install.sh
+		chroot ${CHROOT} /tmp/install.sh
+
+		echo "Done !"
+		echo "Press <Return> to tear down the chroot environment once you are done."
+		echo "And dont' forget to either drop /root/.ssh/authorized_keys or other means of access..."
+		read -r REPLY
+
+	fi
+}
+
+
+# Clean up host
+cleanup() {
+	for mnt in ${CHROOT}/dev/pts ${CHROOT}/run/shm ${CHROOT}/dev ${CHROOT}/sys ${CHROOT}/proc; do
+		findmnt $mnt > /dev/null && umount $mnt
+	done
+	
+	if [ ${_PORTAGE_MOUNTED} != 0 ]; then
+		umount ${CHROOT}/usr/portage
+	else
+		rm -rf ${CHROOT}/usr/portage/distfiles/*
+	fi
+
+	# Clean up chroot
+	rm -rf ${CHROOT}/tmp/*
+	rm -rf ${CHROOT}/var/tmp/*
+}
+
+
+# print usage
+usage() {
+cat << EOF
+Usage: $0 [options]
+
+This script builds a full Gentoo chroot
+
+OPTIONS:
+-h Show this message
+-a arch, either i686 or x86_64, defaults to uname -m
+-p profile, [ hardened | hardened-no-multilib | default *]
+-t The timezone to use, default to UTC
+-r chroot location (default $CHROOT )
+-c local cache (default $LOCAL_CACHE)
+-b bind mount portage tree from, instead of downloading portage snapshot
+-i interactive, enter chroot only, do NOT run install script
+-m use custom make.conf
+-s kernel flavor, [ hardened* | gentoo ]
+-k kernel .config used to build custom kernel
+-d debug (set -x)
+EOF
+}
+
+
+DEBUG=0
+INTERACTIVE=0
+MAKE_CONF=""
+BIND_PORTAGE=""
+while getopts ":a:p:t:r:c:m:b:k:s:dhi" OPTIONS; do
+	case $OPTIONS in
+		a ) ARCH=$OPTARG;;
+		p ) PROFILE=$OPTARG;;
+		t ) TIMEZONE=$OPTARG;;
+		d ) DEBUG=1;;
+		b ) BIND_PORTAGE=$OPTARG;;
+		r ) CHROOT=$OPTARG;;
+		c ) LOCAL_CACHE=$OPTARG;;
+		i ) INTERACTIVE=1;;
+		m ) MAKE_CONF=$OPTARG;;
+		k ) KERNEL_CONF=$OPTARG;;
+		s ) KERNEL_FLAVOR=$OPTARG;;
+		? )
+			usage
+			exit
+			;;
+	esac
+done
+
+# Do some sanity checks first
+if [ "$(id -u)" != "0" ]; then
+	die "Sorry, but we need root permissions to create DEVICE nodes etc.!"
+fi
+
+ARCH=${ARCH-"$(uname -m)"}
+PROFILE=${PROFILE="default"}
+TIMEZONE=${TIMEZONE-"UTC"}
+KERNEL_CONF=${KERNEL_CONF-"/usr/src/linux/.config"}
+KERNEL_FLAVOR=${KERNEL_FLAVOR-"hardened"}
+
+if [ "${ARCH}" = "i686" ] ; then
+	# Why do they use x86 here ? :(
+	STAGE_PATH="${GENTOO_MIRROR}/releases/x86/autobuilds"
+elif [ "${ARCH}" = "x86_64" ] ; then
+	ARCH="amd64"
+	STAGE_PATH="${GENTOO_MIRROR}/releases/${ARCH}/autobuilds"
+elif [ "${ARCH}" = "amd64" ] ; then
+	STAGE_PATH="${GENTOO_MIRROR}/releases/${ARCH}/autobuilds"
+else
+	die "Unknown architecture!"
+fi
+
+if [ "${PROFILE}" = "hardened" ] ; then
+	LATEST_STAGE_FILE="${STAGE_PATH}/latest-stage3-${ARCH}-hardened.txt"
+	ESELECT_PROFILE="hardened/linux/${ARCH}"
+elif [ "${PROFILE}" = "hardened-no-multilib" ] ; then
+	LATEST_STAGE_FILE="${STAGE_PATH}/latest-stage3-${ARCH}-hardened+nomultilib.txt"
+	ESELECT_PROFILE="hardened/linux/${ARCH}/no-multilib"
+elif [ "${PROFILE}" = "default" ] ; then
+	LATEST_STAGE_FILE="${STAGE_PATH}/latest-stage3-${ARCH}.txt"
+	ESELECT_PROFILE="default/linux/${ARCH}/13.0"
+else
+	die "Unknown profile!"
+fi
+
+if [ ${DEBUG} -eq 1 ]; then
+	set -x
+fi
+
+bootstrap 
+
+# From here make sure we don't leave stuff around on the host
+trap "cleanup" INT TERM EXIT
+
+setup_chroot
+
+install_gentoo

dev-util/bob/server/make.conf

@@ -0,0 +1,32 @@
+# Please consult /usr/share/portage/config/make.conf.example for a more
+# detailed example.
+# CFLAGS="-march=amdfam10 -O2 -pipe"
+CFLAGS="-O2 -pipe -march=native"
+CXXFLAGS="${CFLAGS}"
+LDFLAGS="-Wl,-O1 -Wl,--as-needed"
+
+# _ARCH=`uname -m`
+# WARNING: Changing your CHOST is not something that should be done lightly.
+# Please consult http://www.gentoo.org/doc/en/change-chost.xml before changing.
+CHOST="x86_64-pc-linux-gnu"
+# These are the USE flags that were used in addition to what is provided by the
+# profile used for building.
+
+# All other CPU related USE flags are set via /etc/profile.d/portage_env.sh
+DO_USE="smp"
+
+# Disable things
+USE_NOT="-tcpd"
+
+# No SystemD and friends
+USE_NOT="${USE_NOT} -systemd -consolekit -policykit -networkmanager -udisks"
+
+USE="${DO_USE} ${USE_NOT}"
+
+FEATURES="clean-logs fail-clean parallel-install skiprocheck userfetch parallel-fetch strict"
+
+PAX_MARKINGS="XT"
+
+PORTAGE_NICENESS=10
+CLEAN_DELAY="3"
+

dev-util/bob/server/parted.txt

@@ -0,0 +1,6 @@
+u MiB
+mklabel gpt
+mkpart grub 1 5
+mkpart boot 5 128
+mkpart root 128 102400
+set 1 bios_grub on

dev-util/bob/server/setup_fs.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+parted -a opt /dev/sda -s $(cat parted.txt)
+partprobe 
+
+pvcreate /dev/sda3
+vgcreate disks /dev/sda3
+lvcreate -L16G /dev/mapper/disks -n root
+
+mkfs.xfs -f /dev/sda2
+mkfs.xfs /dev/disks/root
+
+mkdir /mnt/gentoo
+mount /dev/disks/root /mnt/gentoo
+mkdir /mnt/gentoo/boot
+mount /dev/sda2 /mnt/gentoo/boot

dev-util/bob/server/setup_raid.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+[ -r parted.txt ] || exit 1
+
+parted -a opt /dev/sda -s $(cat parted.txt)
+parted -a opt /dev/sdb -s $(cat parted.txt)
+partprobe 
+mdadm --create /dev/md0 --level=mirror --raid-devices=2 /dev/sda2 /dev/sdb2
+mdadm --create /dev/md1 --level=mirror --raid-devices=2 /dev/sda3 /dev/sdb3
+
+cryptsetup -v --cipher aes-xts-plain64 --key-size 512 --hash sha512 --iter-time 5000 --use-random luksFormat /dev/md1
+cryptsetup luksOpen /dev/md1 crypt_root
+
+pvcreate /dev/mapper/crypt_root
+vgcreate disks /dev/mapper/crypt_root
+lvcreate -L16G /dev/mapper/disks -n root
+
+mkfs.xfs -f /dev/md0
+mkfs.xfs -f /dev/disks/root
+
+mkdir -p /mnt/gentoo
+mount /dev/disks/root /mnt/gentoo
+mkdir -p /mnt/gentoo/boot
+mount /dev/md0 /mnt/gentoo/boot

dev-util/bob/urls.txt

@@ -0,0 +1,8 @@
+https://github.com/psi/gentoo-aws/blob/93d3bf2498c572832778a853b9307e47ef7f5b5b/scripts/build-stage3-image.sh
+https://raw.github.com/gist/870789/bf45e6c5b370445fc3c19111af0dd5eec4097480/create_gentoo_ami.sh
+http://www.agaffney.org/quickstart.php
+http://dev.gentoo.org/~swift/docs/gensetup-guide.xml
+http://rich0gentoo.wordpress.com/2010/04/02/gentoo-on-ec2-from-scratch/
+
+http://gentoo.arcticnetwork.ca/releases/amd64/autobuilds/
+