Build: Move reusable parts of docker build scripts into new scripts

This commit is contained in:
Mike Salvatore 2021-07-27 13:08:55 -04:00
parent 3e638298e6
commit 5576e4e241
5 changed files with 203 additions and 167 deletions

View File

@ -1,20 +1,16 @@
WORKSPACE=${WORKSPACE:-$HOME}
DEFAULT_REPO_MONKEY_HOME=$WORKSPACE/git/monkey
BUILD_DIR="$PWD/monkey"
GIT=$WORKSPACE/git
DEFAULT_REPO_MONKEY_HOME=$GIT/monkey
ISLAND_PATH="$BUILD_DIR/monkey_island"
ISLAND_BINARIES_PATH="$ISLAND_PATH/cc/binaries"
MONKEY_ORIGIN_URL="https://github.com/guardicore/monkey.git"
CONFIG_URL="https://raw.githubusercontent.com/guardicore/monkey/develop/deployment_scripts/config"
NODE_SRC=https://deb.nodesource.com/setup_12.x
ISLAND_DIR_COPY_TIMEOUT=60 #Seconds
OUTPUT_NAME_TGZ="$PWD/infection_monkey_docker_$(date +%Y%m%d_%H%M%S).tgz"
log_message() {
echo -e "\n\n"
echo -e "MONKEY ISLAND BUILDER: $1"
}
exit_if_missing_argument() {
if [ -z "$2" ] || [ "${2:0:1}" == "-" ]; then
@ -24,20 +20,20 @@ exit_if_missing_argument() {
}
echo_help() {
echo "usage: build_appimage.sh [--help] [--agent-binary-dir <PATH>] [--branch <BRANCH>]"
echo "usage: build_package.sh [--help] [--agent-binary-dir <PATH>] [--branch <BRANCH>]"
echo " [--monkey-repo <PATH>] [--version <MONKEY_VERSION>]"
echo ""
echo "Creates an AppImage package for Infection Monkey."
echo "Creates a package for Infection Monkey."
echo ""
echo "--agent-binary-dir A directory containing the agent binaries that"
echo " you'd like to include with the AppImage. If this"
echo " you'd like to include with the package. If this"
echo " parameter is unspecified, the latest release"
echo " binaries will be downloaded from GitHub."
echo ""
echo "--as-root Throw caution to the wind and allow this script"
echo " to be run as root."
echo ""
echo "--branch The git branch you'd like the AppImage to be"
echo "--branch The git branch you'd like the package to be"
echo " built from. (Default: develop)"
echo ""
echo "--monkey-repo A directory containing the Infection Monkey git"
@ -48,8 +44,10 @@ echo_help() {
echo " will have no effect."
echo " (Default: $DEFAULT_REPO_MONKEY_HOME)"
echo ""
echo "--version A version number for the AppImage package."
echo "--version A version number for the package."
echo " (Default: dev)"
echo ""
echo "--package Which package to build (\"appimage\" or \"docker.\""
exit 0
}
@ -64,11 +62,6 @@ has_sudo() {
return $?
}
log_message() {
echo -e "\n\n"
echo -e "DOCKER IMAGE BUILDER: $1"
}
handle_error() {
echo "Fix the errors above and rerun the script"
exit 1
@ -81,7 +74,7 @@ install_nodejs() {
sudo apt-get install -y nodejs
}
install_build_prereqs() {
install_common_build_prereqs() {
sudo apt-get update
sudo apt-get upgrade -y
@ -90,8 +83,13 @@ install_build_prereqs() {
install_nodejs
}
install_docker() {
sudo apt-get install -y docker.io
is_valid_git_repo() {
pushd "$1" 2>/dev/null || return 1
git status >/dev/null 2>&1
success="$?"
popd || exit 1
return $success
}
clone_monkey_repo() {
@ -106,140 +104,13 @@ clone_monkey_repo() {
git clone -c core.autocrlf=false --single-branch --recurse-submodules -b "$branch" "$MONKEY_ORIGIN_URL" "$repo_dir" 2>&1 || handle_error
}
is_valid_git_repo() {
pushd "$1" 2>/dev/null || return 1
git status >/dev/null 2>&1
success="$?"
popd || exit 1
install_build_prereqs() {
sudo apt-get update
sudo apt-get upgrade -y
return $success
}
setup_build_dir() {
local agent_binary_dir=$1
local monkey_repo=$2
mkdir "$BUILD_DIR"
copy_entrypoint_to_build_dir
copy_monkey_island_to_build_dir "$monkey_repo/monkey"
add_agent_binaries_to_build_dir "$agent_binary_dir"
generate_ssl_cert
build_frontend
}
copy_entrypoint_to_build_dir() {
cp ./entrypoint.sh "$BUILD_DIR"
chmod 755 "$BUILD_DIR/entrypoint.sh"
}
copy_monkey_island_to_build_dir() {
local src=$1
cp "$src"/__init__.py "$BUILD_DIR"
cp "$src"/monkey_island.py "$BUILD_DIR"
cp -v -r "$src"/common "$BUILD_DIR/"
rsync \
-avr \
--exclude=monkey_island/cc/ui/node_modules \
--exclude=monkey_island/cc/ui/.npm \
"$src"/monkey_island "$BUILD_DIR/"
cp ./server_config.json "$BUILD_DIR"/monkey_island/cc/
}
add_agent_binaries_to_build_dir() {
local agent_binary_dir=$1
if [ -z "$agent_binary_dir" ]; then
download_monkey_agent_binaries
else
copy_agent_binaries_to_appdir "$agent_binary_dir"
fi
make_linux_binaries_executable
}
download_monkey_agent_binaries() {
log_message "Downloading monkey agent binaries to ${ISLAND_BINARIES_PATH}"
load_monkey_binary_config
mkdir -p "${ISLAND_BINARIES_PATH}" || handle_error
curl -L -o "${ISLAND_BINARIES_PATH}/${LINUX_32_BINARY_NAME}" "${LINUX_32_BINARY_URL}"
curl -L -o "${ISLAND_BINARIES_PATH}/${LINUX_64_BINARY_NAME}" "${LINUX_64_BINARY_URL}"
curl -L -o "${ISLAND_BINARIES_PATH}/${WINDOWS_32_BINARY_NAME}" "${WINDOWS_32_BINARY_URL}"
curl -L -o "${ISLAND_BINARIES_PATH}/${WINDOWS_64_BINARY_NAME}" "${WINDOWS_64_BINARY_URL}"
}
load_monkey_binary_config() {
tmpfile=$(mktemp)
log_message "Downloading prebuilt binary configuration"
curl -L -s -o "$tmpfile" "$CONFIG_URL"
log_message "Loading configuration"
source "$tmpfile"
}
copy_agent_binaries_to_appdir() {
cp "$1"/* "$ISLAND_BINARIES_PATH/"
}
make_linux_binaries_executable() {
chmod a+x "$ISLAND_BINARIES_PATH"/monkey-linux-*
}
generate_ssl_cert() {
log_message "Generating certificate"
chmod u+x "${ISLAND_PATH}"/linux/create_certificate.sh
"${ISLAND_PATH}"/linux/create_certificate.sh "${ISLAND_PATH}"/cc
}
build_frontend() {
pushd "$ISLAND_PATH/cc/ui" || handle_error
log_message "Generating front end"
npm ci
npm run dist
popd || handle_error
remove_node_modules
}
remove_node_modules() {
# Node has served its purpose. We don't need to deliver the node modules with
# the AppImage.
rm -rf "$ISLAND_PATH"/cc/ui/node_modules
rm -rf "$ISLAND_PATH"/cc/ui/.npm
}
build_docker_image() {
local version=$1
docker_image_name=guardicore/monkey-island:$version
tar_name=./dk.monkeyisland.$version.tar
build_docker_image_tar "$docker_image_name" "$tar_name"
build_docker_image_tgz "$tar_name" "$version"
}
build_docker_image_tar() {
sudo docker build . -t "$1"
sudo docker save "$1" > "$2"
}
build_docker_image_tgz() {
mkdir tgz
cp "$1" ./tgz
cp ./DOCKER_README.md ./tgz/README.md
tar -C ./tgz -cvf "$OUTPUT_NAME_TGZ" --gzip .
# monkey island prereqs
sudo apt-get install -y curl libcurl4 openssl git build-essential moreutils
install_nodejs
}
agent_binary_dir=""
@ -247,6 +118,7 @@ as_root=false
branch="develop"
monkey_repo="$DEFAULT_REPO_MONKEY_HOME"
monkey_version="dev"
package=""
while (( "$#" )); do
@ -282,6 +154,12 @@ while (( "$#" )); do
monkey_version=$2
shift 2
;;
--package)
exit_if_missing_argument "$1" "$2"
package=$2
shift 2
;;
*)
echo "Error: Unsupported parameter $1" >&2
exit 1
@ -289,7 +167,10 @@ while (( "$#" )); do
esac
done
log_message "Building Monkey Island Docker image."
if ! [[ $package =~ ^(appimage|docker)$ ]]; then
log_message "Invalid package: $package."
exit 1
fi
if ! $as_root && is_root; then
log_message "Please don't run this script as root"
@ -302,15 +183,20 @@ Run \`sudo -v\`, enter your password, and then re-run this script."
exit 1
fi
install_build_prereqs
install_docker
log_message "Building Monkey Island: $package"
source "./$package/$package.sh"
if ! is_valid_git_repo "$monkey_repo"; then
clone_monkey_repo "$monkey_repo" "$branch"
fi
setup_build_dir "$agent_binary_dir" "$monkey_repo"
build_docker_image "$monkey_version"
install_build_prereqs
install_package_specific_build_prereqs "$WORKSPACE"
log_message "Docker build script finished."
setup_build_dir "$agent_binary_dir" "$monkey_repo"
build_package "$monkey_version"
log_message "Finished building package: $package"
exit 0

86
build_scripts/common.sh Normal file
View File

@ -0,0 +1,86 @@
copy_monkey_island_to_build_dir() {
local src=$1
local build_dir=$2
cp "$src"/__init__.py "$build_dir"
cp "$src"/monkey_island.py "$build_dir"
cp -r "$src"/common "$build_dir/"
rsync \
-ar \
--exclude=monkey_island/cc/ui/node_modules \
--exclude=monkey_island/cc/ui/.npm \
"$src"/monkey_island "$build_dir/"
}
add_agent_binaries_to_build_dir() {
local agent_binary_dir=$1
local island_binaries_path="$2/monkey_island/cc/binaries/"
if [ -z "$agent_binary_dir" ]; then
download_monkey_agent_binaries $island_binaries_path
else
copy_agent_binaries_to_build_dir "$agent_binary_dir" "$island_binaries_path"
fi
make_linux_binaries_executable "$island_binaries_path"
}
download_monkey_agent_binaries() {
local island_binaries_path=$1
log_message "Downloading monkey agent binaries to ${island_binaries_path}"
load_monkey_binary_config
mkdir -p "${island_binaries_path}" || handle_error
curl -L -o "${island_binaries_path}/${LINUX_32_BINARY_NAME}" "${LINUX_32_BINARY_URL}"
curl -L -o "${island_binaries_path}/${LINUX_64_BINARY_NAME}" "${LINUX_64_BINARY_URL}"
curl -L -o "${island_binaries_path}/${WINDOWS_32_BINARY_NAME}" "${WINDOWS_32_BINARY_URL}"
curl -L -o "${island_binaries_path}/${WINDOWS_64_BINARY_NAME}" "${WINDOWS_64_BINARY_URL}"
}
load_monkey_binary_config() {
tmpfile=$(mktemp)
log_message "Downloading prebuilt binary configuration"
curl -L -s -o "$tmpfile" "$CONFIG_URL"
log_message "Loading configuration"
source "$tmpfile"
}
copy_agent_binaries_to_build_dir() {
cp "$1"/* "$2/"
}
make_linux_binaries_executable() {
chmod a+x "$1"/monkey-linux-*
}
generate_ssl_cert() {
local island_path="$1/monkey_island"
log_message "Generating certificate"
chmod u+x "$island_path"/linux/create_certificate.sh
"$island_path"/linux/create_certificate.sh "$island_path"/cc
}
build_frontend() {
local ui_dir="$1/monkey_island/cc/ui"
pushd "$ui_dir" || handle_error
log_message "Generating front end"
npm ci
npm run dist
popd || handle_error
remove_node_modules "$ui_dir"
}
remove_node_modules() {
# Node has served its purpose. We don't need to deliver the node modules with
# the package.
rm -rf "$1/node_modules"
rm -rf "$1/.npm"
}

3
build_scripts/docker/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
dk.monkeyisland*.tar
infection_monkey_docker_*.tgz
tgz/

View File

@ -3,10 +3,11 @@
# This is a utility script to clean up after a failed or successful Docker
# image build in order to speed up development and debugging
BUILD_DIR=$HOME/docker
DOCKER_DIR="$(realpath $(dirname $BASH_SOURCE[0]))"
rm -rf $HOME/git/monkey
rm -rf $BUILD_DIR/monkey
rm -rf $BUILD_DIR/tgz
rm $BUILD_DIR/dk.monkeyisland.*.tar
rm $BUILD_DIR/infection_monkey_docker*.tgz
rm -rf "$HOME/git/monkey"
rm -rf "$DOCKER_DIR/monkey"
rm -rf "$DOCKER_DIR/tgz"
rm "$DOCKER_DIR"/dk.monkeyisland.*.tar
rm "$DOCKER_DIR"/infection_monkey_docker*.tgz

60
build_scripts/docker/docker.sh Executable file
View File

@ -0,0 +1,60 @@
DOCKER_DIR="$(realpath $(dirname $BASH_SOURCE[0]))"
OUTPUT_NAME_TGZ="$DOCKER_DIR/infection_monkey_docker_$(date +%Y%m%d_%H%M%S).tgz"
source "$DOCKER_DIR/../common.sh"
install_package_specific_build_prereqs() {
sudo apt-get install -y docker.io
}
setup_build_dir() {
local agent_binary_dir=$1
local monkey_repo=$2
local build_dir=$DOCKER_DIR/monkey
mkdir "$build_dir"
copy_entrypoint_to_build_dir "$build_dir"
copy_monkey_island_to_build_dir "$monkey_repo/monkey" "$build_dir"
copy_server_config_to_build_dir "$build_dir"
add_agent_binaries_to_build_dir "$agent_binary_dir" "$build_dir"
generate_ssl_cert "$build_dir"
build_frontend "$build_dir"
}
copy_entrypoint_to_build_dir() {
cp "$DOCKER_DIR"/entrypoint.sh "$1"
chmod 755 "$1/entrypoint.sh"
}
copy_server_config_to_build_dir() {
cp "$DOCKER_DIR"/server_config.json "$1"/monkey_island/cc
}
build_package() {
local version=$1
pushd ./docker
docker_image_name="guardicore/monkey-island:$version"
tar_name="$DOCKER_DIR/dk.monkeyisland.$version.tar"
build_docker_image_tar "$docker_image_name" "$tar_name"
build_docker_image_tgz "$tar_name" "$version"
popd
}
build_docker_image_tar() {
sudo docker build . -t "$1"
sudo docker save "$1" > "$2"
}
build_docker_image_tgz() {
mkdir tgz
mv "$1" ./tgz
cp ./DOCKER_README.md ./tgz/README.md
tar -C ./tgz -cvf "$OUTPUT_NAME_TGZ" --gzip .
}