Migrate full ami build to builder script
This commit is contained in:
parent
5b2f32c9c8
commit
b53492723d
60
Makefile
60
Makefile
@ -1,60 +0,0 @@
|
|||||||
# vim: ts=8 noet:
|
|
||||||
|
|
||||||
NVME_SCRIPTS := $(subst scripts/,build/,$(wildcard scripts/nvme/*))
|
|
||||||
CORE_PROFILES := $(wildcard profiles/*/*)
|
|
||||||
TARGET_PROFILES := $(wildcard profiles/*.conf)
|
|
||||||
|
|
||||||
PROFILE :=
|
|
||||||
BUILD :=
|
|
||||||
BUILDS := $(BUILD)
|
|
||||||
LEVEL :=
|
|
||||||
|
|
||||||
# by default, use the 'packer' in the path
|
|
||||||
PACKER := packer
|
|
||||||
export PACKER
|
|
||||||
|
|
||||||
|
|
||||||
check_defined = \
|
|
||||||
$(strip $(foreach 1,$1, \
|
|
||||||
$(call __check_defined,$1,$(strip $(value 2)))))
|
|
||||||
__check_defined = \
|
|
||||||
$(if $(value $1),, \
|
|
||||||
$(error Undefined $1$(if $2, ($2))$(if $(value @), \
|
|
||||||
required by target `$@')))
|
|
||||||
|
|
||||||
|
|
||||||
.PHONY: amis prune release-readme clean
|
|
||||||
|
|
||||||
amis: build/packer.json build/profile/$(PROFILE) build
|
|
||||||
@:$(call check_defined, PROFILE, target profile name)
|
|
||||||
build/builder make-amis $(PROFILE) $(BUILDS)
|
|
||||||
|
|
||||||
prune: build
|
|
||||||
@:$(call check_defined, LEVEL, pruning level)
|
|
||||||
@:$(call check_defined, PROFILE, target profile name)
|
|
||||||
build/builder prune-amis $(LEVEL) $(PROFILE) $(BUILD)
|
|
||||||
|
|
||||||
release-readme: releases/README.md
|
|
||||||
releases/README.md: build
|
|
||||||
@:$(call check_defined, PROFILE, target profile name)
|
|
||||||
@:$(call require_var, PROFILE)
|
|
||||||
build/builder gen-release-readme $(PROFILE)
|
|
||||||
|
|
||||||
build:
|
|
||||||
python3 -m venv build
|
|
||||||
[ -d build/profile ] || mkdir -p build/profile
|
|
||||||
build/bin/pip install -U pip pyhocon pyyaml boto3
|
|
||||||
|
|
||||||
echo -e "#!/bin/sh\n$$(pwd)/build/bin/python scripts/builder.py \$$@" > $@
|
|
||||||
chmod +x $@
|
|
||||||
|
|
||||||
build/packer.json: packer.conf build
|
|
||||||
build/builder convert-packer-config
|
|
||||||
|
|
||||||
.PHONY: build/profile/$(PROFILE)
|
|
||||||
build/profile/$(PROFILE): build $(CORE_PROFILES) $(TARGET_PROFILES)
|
|
||||||
@:$(call check_defined, PROFILE, target profile name)
|
|
||||||
build/builder resolve-profiles $(PROFILE)
|
|
||||||
|
|
||||||
clean:
|
|
||||||
rm -rf build
|
|
21
README.md
21
README.md
@ -36,7 +36,6 @@ include as much detailed information as possible.
|
|||||||
|
|
||||||
* [Packer](https://packer.io) >= 1.4.1
|
* [Packer](https://packer.io) >= 1.4.1
|
||||||
* [Python 3.x](https://python.org) (3.7 is known to work)
|
* [Python 3.x](https://python.org) (3.7 is known to work)
|
||||||
* `make` (GNU Make is known to work)
|
|
||||||
* an AWS account with an existing subnet in an AWS Virtual Private Cloud
|
* an AWS account with an existing subnet in an AWS Virtual Private Cloud
|
||||||
|
|
||||||
### Profile Configuration
|
### Profile Configuration
|
||||||
@ -59,18 +58,12 @@ two methods on the list.*
|
|||||||
|
|
||||||
To build all build targets in a target profile, simply...
|
To build all build targets in a target profile, simply...
|
||||||
```
|
```
|
||||||
make PROFILE=<profile>
|
./scripts/builder.py amis <profile>
|
||||||
```
|
```
|
||||||
|
|
||||||
You can also build specfic build targets within a profile:
|
You can also build specfic build targets within a profile:
|
||||||
```
|
```
|
||||||
make PROFILE=<profile> BUILDS="<build1> <build2>"
|
./scripts/builder.py amis <profile> <build1> <build2>
|
||||||
```
|
|
||||||
|
|
||||||
If the `packer` binary is not in your `PATH`, or you would like to specify a
|
|
||||||
different one, use...
|
|
||||||
```
|
|
||||||
make PACKER=<packer-path> PROFILE=<profile>
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Before each build, new Alpine Linux *releases* are detected and the version's
|
Before each build, new Alpine Linux *releases* are detected and the version's
|
||||||
@ -98,7 +91,7 @@ pruning:
|
|||||||
|
|
||||||
To prune a profile (or optionally one build target of a profile)...
|
To prune a profile (or optionally one build target of a profile)...
|
||||||
```
|
```
|
||||||
make prune LEVEL=<level> PROFILE=<profile> [BUILD=<build>]
|
./scripts/builder.py prune-amis <profile> [<build>]
|
||||||
```
|
```
|
||||||
|
|
||||||
Any AMIs in the account which are "unknown" (to the profile/build target, at
|
Any AMIs in the account which are "unknown" (to the profile/build target, at
|
||||||
@ -110,14 +103,14 @@ This make target updates the [releases README](releases/README.md), primarily
|
|||||||
for updating the list of our pre-built AMIs. This may-or-may-not be useful for
|
for updating the list of our pre-built AMIs. This may-or-may-not be useful for
|
||||||
other target profiles.
|
other target profiles.
|
||||||
```
|
```
|
||||||
make release-readme PROFILE=<profile>
|
./scripts/builder.py gen-release-readme <profile>
|
||||||
```
|
```
|
||||||
|
|
||||||
### Cleaning up the Build Environment
|
### Cleaning up the Build Environment
|
||||||
|
|
||||||
`make clean` will remove the temporary `build` subdirectory, which contains the
|
`git clean -dxf` will remove the temporary `build` subdirectory, which contains
|
||||||
resolved profile and Packer configs, the Python virtual environment, and other
|
the resolved profile and Packer configs, the Python virtual environment, and
|
||||||
temporary build-related artifacts.
|
other temporary build-related artifacts.
|
||||||
|
|
||||||
## Caveats
|
## Caveats
|
||||||
|
|
||||||
|
@ -287,8 +287,7 @@ class MakeAMIs:
|
|||||||
print(text, end="")
|
print(text, end="")
|
||||||
|
|
||||||
if res.returncode == 0:
|
if res.returncode == 0:
|
||||||
subprocess.run([os.path.join(root, "build", "builder"),
|
UpdateReleases().update_readme(args.profile, build, root)
|
||||||
"update-releases", args.profile, build])
|
|
||||||
else:
|
else:
|
||||||
if "is used by an existing AMI" in out.getvalue():
|
if "is used by an existing AMI" in out.getvalue():
|
||||||
continue
|
continue
|
||||||
@ -537,17 +536,20 @@ class ResolveProfiles:
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"profile", help="name of profile to build", nargs="*")
|
"profile", help="name of profile to build", nargs="*")
|
||||||
|
|
||||||
def run(self, args, root):
|
def resolve_profiles(self, profiles, root):
|
||||||
builder = ConfigBuilder(
|
builder = ConfigBuilder(
|
||||||
os.path.join(root, "profiles"),
|
os.path.join(root, "profiles"),
|
||||||
os.path.join(root, "build", "profile"))
|
os.path.join(root, "build", "profile"))
|
||||||
|
|
||||||
if args.profile:
|
if profiles:
|
||||||
for profile in args.profile:
|
for profile in profiles:
|
||||||
builder.build_profile(profile)
|
builder.build_profile(profile)
|
||||||
else:
|
else:
|
||||||
builder.build_all()
|
builder.build_all()
|
||||||
|
|
||||||
|
def run(self, args, root):
|
||||||
|
self.resolve_profiles(args.profile, root)
|
||||||
|
|
||||||
|
|
||||||
class UpdateReleases:
|
class UpdateReleases:
|
||||||
"""Update release YAML
|
"""Update release YAML
|
||||||
@ -566,18 +568,21 @@ class UpdateReleases:
|
|||||||
return dict(zip(parsed[0::2], parsed[1::2]))
|
return dict(zip(parsed[0::2], parsed[1::2]))
|
||||||
|
|
||||||
def run(self, args, root):
|
def run(self, args, root):
|
||||||
|
self.update_readme(args.profile, args.build, root)
|
||||||
|
|
||||||
|
def update_readme(self, profile, build, root):
|
||||||
release_dir = os.path.join(root, "releases")
|
release_dir = os.path.join(root, "releases")
|
||||||
if not os.path.exists(release_dir):
|
if not os.path.exists(release_dir):
|
||||||
os.makedirs(release_dir)
|
os.makedirs(release_dir)
|
||||||
|
|
||||||
release_yaml = os.path.join(release_dir, f"{args.profile}.yaml")
|
release_yaml = os.path.join(release_dir, f"{profile}.yaml")
|
||||||
releases = {}
|
releases = {}
|
||||||
if os.path.exists(release_yaml):
|
if os.path.exists(release_yaml):
|
||||||
with open(release_yaml, "r") as data:
|
with open(release_yaml, "r") as data:
|
||||||
releases = yaml.safe_load(data)
|
releases = yaml.safe_load(data)
|
||||||
|
|
||||||
manifest_json = os.path.join(
|
manifest_json = os.path.join(
|
||||||
root, "build", "profile", args.profile, args.build,
|
root, "build", "profile", profile, build,
|
||||||
"manifest.json")
|
"manifest.json")
|
||||||
with open(manifest_json, "r") as data:
|
with open(manifest_json, "r") as data:
|
||||||
manifest = json.load(data)
|
manifest = json.load(data)
|
||||||
@ -585,16 +590,16 @@ class UpdateReleases:
|
|||||||
data = manifest["builds"][0]["custom_data"]
|
data = manifest["builds"][0]["custom_data"]
|
||||||
release = data["release"]
|
release = data["release"]
|
||||||
|
|
||||||
if args.build not in releases:
|
if build not in releases:
|
||||||
releases[args.build] = {}
|
releases[build] = {}
|
||||||
|
|
||||||
if release not in releases[args.build]:
|
if release not in releases[build]:
|
||||||
releases[args.build][release] = {}
|
releases[build][release] = {}
|
||||||
|
|
||||||
releases[args.build][release][data["ami_name"]] = {
|
releases[build][release][data["ami_name"]] = {
|
||||||
"description": data["ami_desc"],
|
"description": data["ami_desc"],
|
||||||
"profile": args.profile,
|
"profile": profile,
|
||||||
"profile_build": args.build,
|
"profile_build": build,
|
||||||
"version": data["version"],
|
"version": data["version"],
|
||||||
"release": release,
|
"release": release,
|
||||||
"arch": data["arch"],
|
"arch": data["arch"],
|
||||||
@ -628,6 +633,34 @@ class ConvertPackerJSON:
|
|||||||
source, dest, "json", 2, False)
|
source, dest, "json", 2, False)
|
||||||
|
|
||||||
|
|
||||||
|
class FullBuild:
|
||||||
|
"""Make all of the AMIs for a profile
|
||||||
|
"""
|
||||||
|
|
||||||
|
command_name = "amis"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def add_args(parser):
|
||||||
|
parser.add_argument("--region", "-r", default="us-west-2",
|
||||||
|
help="region to use for build")
|
||||||
|
parser.add_argument("profile", help="name of profile to build")
|
||||||
|
parser.add_argument("builds", nargs="*",
|
||||||
|
help="name of builds within a profile to build")
|
||||||
|
|
||||||
|
def run(self, args, root):
|
||||||
|
print("Converting packer.conf to JSON...", file=sys.stderr)
|
||||||
|
ConvertPackerJSON().run(args, root)
|
||||||
|
|
||||||
|
print("Resolving profiles...", file=sys.stderr)
|
||||||
|
ResolveProfiles().resolve_profiles([args.profile], root)
|
||||||
|
|
||||||
|
print("Running packer...", file=sys.stderr)
|
||||||
|
MakeAMIs().run(args, root)
|
||||||
|
|
||||||
|
print("Updating release readme...", file=sys.stderr)
|
||||||
|
GenReleaseReadme().run(args, root)
|
||||||
|
|
||||||
|
|
||||||
def find_repo_root():
|
def find_repo_root():
|
||||||
path = os.getcwd()
|
path = os.getcwd()
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user