From 5576e4e241495cd832eb4f3f8d74e932c7fd1228 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 27 Jul 2021 13:08:55 -0400 Subject: [PATCH] Build: Move reusable parts of docker build scripts into new scripts --- .../build_docker.sh => build_package.sh} | 208 ++++-------------- build_scripts/common.sh | 86 ++++++++ build_scripts/docker/.gitignore | 3 + build_scripts/docker/clean.sh | 13 +- build_scripts/docker/docker.sh | 60 +++++ 5 files changed, 203 insertions(+), 167 deletions(-) rename build_scripts/{docker/build_docker.sh => build_package.sh} (52%) create mode 100644 build_scripts/common.sh create mode 100644 build_scripts/docker/.gitignore create mode 100755 build_scripts/docker/docker.sh diff --git a/build_scripts/docker/build_docker.sh b/build_scripts/build_package.sh similarity index 52% rename from build_scripts/docker/build_docker.sh rename to build_scripts/build_package.sh index 797e8d06d..a7ec5d4e7 100755 --- a/build_scripts/docker/build_docker.sh +++ b/build_scripts/build_package.sh @@ -1,20 +1,16 @@ WORKSPACE=${WORKSPACE:-$HOME} +DEFAULT_REPO_MONKEY_HOME=$WORKSPACE/git/monkey -BUILD_DIR="$PWD/monkey" -GIT=$WORKSPACE/git - -DEFAULT_REPO_MONKEY_HOME=$GIT/monkey - -ISLAND_PATH="$BUILD_DIR/monkey_island" -ISLAND_BINARIES_PATH="$ISLAND_PATH/cc/binaries" MONKEY_ORIGIN_URL="https://github.com/guardicore/monkey.git" CONFIG_URL="https://raw.githubusercontent.com/guardicore/monkey/develop/deployment_scripts/config" NODE_SRC=https://deb.nodesource.com/setup_12.x -ISLAND_DIR_COPY_TIMEOUT=60 #Seconds -OUTPUT_NAME_TGZ="$PWD/infection_monkey_docker_$(date +%Y%m%d_%H%M%S).tgz" +log_message() { + echo -e "\n\n" + echo -e "MONKEY ISLAND BUILDER: $1" +} exit_if_missing_argument() { if [ -z "$2" ] || [ "${2:0:1}" == "-" ]; then @@ -24,20 +20,20 @@ exit_if_missing_argument() { } echo_help() { - echo "usage: build_appimage.sh [--help] [--agent-binary-dir ] [--branch ]" + echo "usage: build_package.sh [--help] [--agent-binary-dir ] [--branch ]" echo " [--monkey-repo ] [--version ]" echo "" - echo "Creates an AppImage package for Infection Monkey." + echo "Creates a package for Infection Monkey." echo "" echo "--agent-binary-dir A directory containing the agent binaries that" - echo " you'd like to include with the AppImage. If this" + echo " you'd like to include with the package. If this" echo " parameter is unspecified, the latest release" echo " binaries will be downloaded from GitHub." echo "" echo "--as-root Throw caution to the wind and allow this script" echo " to be run as root." echo "" - echo "--branch The git branch you'd like the AppImage to be" + echo "--branch The git branch you'd like the package to be" echo " built from. (Default: develop)" echo "" echo "--monkey-repo A directory containing the Infection Monkey git" @@ -48,8 +44,10 @@ echo_help() { echo " will have no effect." echo " (Default: $DEFAULT_REPO_MONKEY_HOME)" echo "" - echo "--version A version number for the AppImage package." + echo "--version A version number for the package." echo " (Default: dev)" + echo "" + echo "--package Which package to build (\"appimage\" or \"docker.\"" exit 0 } @@ -64,11 +62,6 @@ has_sudo() { return $? } -log_message() { - echo -e "\n\n" - echo -e "DOCKER IMAGE BUILDER: $1" -} - handle_error() { echo "Fix the errors above and rerun the script" exit 1 @@ -81,7 +74,7 @@ install_nodejs() { sudo apt-get install -y nodejs } -install_build_prereqs() { +install_common_build_prereqs() { sudo apt-get update sudo apt-get upgrade -y @@ -90,8 +83,13 @@ install_build_prereqs() { install_nodejs } -install_docker() { - sudo apt-get install -y docker.io +is_valid_git_repo() { + pushd "$1" 2>/dev/null || return 1 + git status >/dev/null 2>&1 + success="$?" + popd || exit 1 + + return $success } clone_monkey_repo() { @@ -106,140 +104,13 @@ clone_monkey_repo() { git clone -c core.autocrlf=false --single-branch --recurse-submodules -b "$branch" "$MONKEY_ORIGIN_URL" "$repo_dir" 2>&1 || handle_error } -is_valid_git_repo() { - pushd "$1" 2>/dev/null || return 1 - git status >/dev/null 2>&1 - success="$?" - popd || exit 1 +install_build_prereqs() { + sudo apt-get update + sudo apt-get upgrade -y - return $success -} - -setup_build_dir() { - local agent_binary_dir=$1 - local monkey_repo=$2 - - mkdir "$BUILD_DIR" - - copy_entrypoint_to_build_dir - - copy_monkey_island_to_build_dir "$monkey_repo/monkey" - add_agent_binaries_to_build_dir "$agent_binary_dir" - - generate_ssl_cert - - build_frontend -} - -copy_entrypoint_to_build_dir() { - cp ./entrypoint.sh "$BUILD_DIR" - chmod 755 "$BUILD_DIR/entrypoint.sh" -} - -copy_monkey_island_to_build_dir() { - local src=$1 - - cp "$src"/__init__.py "$BUILD_DIR" - cp "$src"/monkey_island.py "$BUILD_DIR" - cp -v -r "$src"/common "$BUILD_DIR/" - - rsync \ - -avr \ - --exclude=monkey_island/cc/ui/node_modules \ - --exclude=monkey_island/cc/ui/.npm \ - "$src"/monkey_island "$BUILD_DIR/" - - cp ./server_config.json "$BUILD_DIR"/monkey_island/cc/ -} - -add_agent_binaries_to_build_dir() { - local agent_binary_dir=$1 - - if [ -z "$agent_binary_dir" ]; then - download_monkey_agent_binaries - else - copy_agent_binaries_to_appdir "$agent_binary_dir" - fi - - make_linux_binaries_executable -} - -download_monkey_agent_binaries() { - log_message "Downloading monkey agent binaries to ${ISLAND_BINARIES_PATH}" - - load_monkey_binary_config - - mkdir -p "${ISLAND_BINARIES_PATH}" || handle_error - curl -L -o "${ISLAND_BINARIES_PATH}/${LINUX_32_BINARY_NAME}" "${LINUX_32_BINARY_URL}" - curl -L -o "${ISLAND_BINARIES_PATH}/${LINUX_64_BINARY_NAME}" "${LINUX_64_BINARY_URL}" - curl -L -o "${ISLAND_BINARIES_PATH}/${WINDOWS_32_BINARY_NAME}" "${WINDOWS_32_BINARY_URL}" - curl -L -o "${ISLAND_BINARIES_PATH}/${WINDOWS_64_BINARY_NAME}" "${WINDOWS_64_BINARY_URL}" -} - -load_monkey_binary_config() { - tmpfile=$(mktemp) - - log_message "Downloading prebuilt binary configuration" - curl -L -s -o "$tmpfile" "$CONFIG_URL" - - log_message "Loading configuration" - source "$tmpfile" -} - -copy_agent_binaries_to_appdir() { - cp "$1"/* "$ISLAND_BINARIES_PATH/" -} - -make_linux_binaries_executable() { - chmod a+x "$ISLAND_BINARIES_PATH"/monkey-linux-* -} - -generate_ssl_cert() { - log_message "Generating certificate" - - chmod u+x "${ISLAND_PATH}"/linux/create_certificate.sh - "${ISLAND_PATH}"/linux/create_certificate.sh "${ISLAND_PATH}"/cc -} - -build_frontend() { - pushd "$ISLAND_PATH/cc/ui" || handle_error - - log_message "Generating front end" - npm ci - npm run dist - - popd || handle_error - - remove_node_modules -} - -remove_node_modules() { - # Node has served its purpose. We don't need to deliver the node modules with - # the AppImage. - rm -rf "$ISLAND_PATH"/cc/ui/node_modules - rm -rf "$ISLAND_PATH"/cc/ui/.npm -} - -build_docker_image() { - local version=$1 - - docker_image_name=guardicore/monkey-island:$version - tar_name=./dk.monkeyisland.$version.tar - - build_docker_image_tar "$docker_image_name" "$tar_name" - build_docker_image_tgz "$tar_name" "$version" -} - -build_docker_image_tar() { - sudo docker build . -t "$1" - sudo docker save "$1" > "$2" -} - -build_docker_image_tgz() { - mkdir tgz - cp "$1" ./tgz - cp ./DOCKER_README.md ./tgz/README.md - tar -C ./tgz -cvf "$OUTPUT_NAME_TGZ" --gzip . + # monkey island prereqs + sudo apt-get install -y curl libcurl4 openssl git build-essential moreutils + install_nodejs } agent_binary_dir="" @@ -247,6 +118,7 @@ as_root=false branch="develop" monkey_repo="$DEFAULT_REPO_MONKEY_HOME" monkey_version="dev" +package="" while (( "$#" )); do @@ -282,6 +154,12 @@ while (( "$#" )); do monkey_version=$2 shift 2 ;; + --package) + exit_if_missing_argument "$1" "$2" + + package=$2 + shift 2 + ;; *) echo "Error: Unsupported parameter $1" >&2 exit 1 @@ -289,7 +167,10 @@ while (( "$#" )); do esac done -log_message "Building Monkey Island Docker image." +if ! [[ $package =~ ^(appimage|docker)$ ]]; then + log_message "Invalid package: $package." + exit 1 +fi if ! $as_root && is_root; then log_message "Please don't run this script as root" @@ -302,15 +183,20 @@ Run \`sudo -v\`, enter your password, and then re-run this script." exit 1 fi -install_build_prereqs -install_docker +log_message "Building Monkey Island: $package" + +source "./$package/$package.sh" if ! is_valid_git_repo "$monkey_repo"; then clone_monkey_repo "$monkey_repo" "$branch" fi -setup_build_dir "$agent_binary_dir" "$monkey_repo" -build_docker_image "$monkey_version" +install_build_prereqs +install_package_specific_build_prereqs "$WORKSPACE" -log_message "Docker build script finished." + +setup_build_dir "$agent_binary_dir" "$monkey_repo" +build_package "$monkey_version" + +log_message "Finished building package: $package" exit 0 diff --git a/build_scripts/common.sh b/build_scripts/common.sh new file mode 100644 index 000000000..c5a062f5a --- /dev/null +++ b/build_scripts/common.sh @@ -0,0 +1,86 @@ +copy_monkey_island_to_build_dir() { + local src=$1 + local build_dir=$2 + + cp "$src"/__init__.py "$build_dir" + cp "$src"/monkey_island.py "$build_dir" + cp -r "$src"/common "$build_dir/" + + rsync \ + -ar \ + --exclude=monkey_island/cc/ui/node_modules \ + --exclude=monkey_island/cc/ui/.npm \ + "$src"/monkey_island "$build_dir/" +} + +add_agent_binaries_to_build_dir() { + local agent_binary_dir=$1 + local island_binaries_path="$2/monkey_island/cc/binaries/" + + if [ -z "$agent_binary_dir" ]; then + download_monkey_agent_binaries $island_binaries_path + else + copy_agent_binaries_to_build_dir "$agent_binary_dir" "$island_binaries_path" + fi + + make_linux_binaries_executable "$island_binaries_path" +} + +download_monkey_agent_binaries() { + local island_binaries_path=$1 + log_message "Downloading monkey agent binaries to ${island_binaries_path}" + + load_monkey_binary_config + + mkdir -p "${island_binaries_path}" || handle_error + curl -L -o "${island_binaries_path}/${LINUX_32_BINARY_NAME}" "${LINUX_32_BINARY_URL}" + curl -L -o "${island_binaries_path}/${LINUX_64_BINARY_NAME}" "${LINUX_64_BINARY_URL}" + curl -L -o "${island_binaries_path}/${WINDOWS_32_BINARY_NAME}" "${WINDOWS_32_BINARY_URL}" + curl -L -o "${island_binaries_path}/${WINDOWS_64_BINARY_NAME}" "${WINDOWS_64_BINARY_URL}" +} + +load_monkey_binary_config() { + tmpfile=$(mktemp) + + log_message "Downloading prebuilt binary configuration" + curl -L -s -o "$tmpfile" "$CONFIG_URL" + + log_message "Loading configuration" + source "$tmpfile" +} + +copy_agent_binaries_to_build_dir() { + cp "$1"/* "$2/" +} + +make_linux_binaries_executable() { + chmod a+x "$1"/monkey-linux-* +} + +generate_ssl_cert() { + local island_path="$1/monkey_island" + log_message "Generating certificate" + + chmod u+x "$island_path"/linux/create_certificate.sh + "$island_path"/linux/create_certificate.sh "$island_path"/cc +} + +build_frontend() { + local ui_dir="$1/monkey_island/cc/ui" + pushd "$ui_dir" || handle_error + + log_message "Generating front end" + npm ci + npm run dist + + popd || handle_error + + remove_node_modules "$ui_dir" +} + +remove_node_modules() { + # Node has served its purpose. We don't need to deliver the node modules with + # the package. + rm -rf "$1/node_modules" + rm -rf "$1/.npm" +} diff --git a/build_scripts/docker/.gitignore b/build_scripts/docker/.gitignore new file mode 100644 index 000000000..2edc32417 --- /dev/null +++ b/build_scripts/docker/.gitignore @@ -0,0 +1,3 @@ +dk.monkeyisland*.tar +infection_monkey_docker_*.tgz +tgz/ diff --git a/build_scripts/docker/clean.sh b/build_scripts/docker/clean.sh index f5cfacc28..82c769c02 100755 --- a/build_scripts/docker/clean.sh +++ b/build_scripts/docker/clean.sh @@ -3,10 +3,11 @@ # This is a utility script to clean up after a failed or successful Docker # image build in order to speed up development and debugging -BUILD_DIR=$HOME/docker +DOCKER_DIR="$(realpath $(dirname $BASH_SOURCE[0]))" -rm -rf $HOME/git/monkey -rm -rf $BUILD_DIR/monkey -rm -rf $BUILD_DIR/tgz -rm $BUILD_DIR/dk.monkeyisland.*.tar -rm $BUILD_DIR/infection_monkey_docker*.tgz + +rm -rf "$HOME/git/monkey" +rm -rf "$DOCKER_DIR/monkey" +rm -rf "$DOCKER_DIR/tgz" +rm "$DOCKER_DIR"/dk.monkeyisland.*.tar +rm "$DOCKER_DIR"/infection_monkey_docker*.tgz diff --git a/build_scripts/docker/docker.sh b/build_scripts/docker/docker.sh new file mode 100755 index 000000000..7e7c9ae91 --- /dev/null +++ b/build_scripts/docker/docker.sh @@ -0,0 +1,60 @@ +DOCKER_DIR="$(realpath $(dirname $BASH_SOURCE[0]))" +OUTPUT_NAME_TGZ="$DOCKER_DIR/infection_monkey_docker_$(date +%Y%m%d_%H%M%S).tgz" + +source "$DOCKER_DIR/../common.sh" + +install_package_specific_build_prereqs() { + sudo apt-get install -y docker.io +} + +setup_build_dir() { + local agent_binary_dir=$1 + local monkey_repo=$2 + local build_dir=$DOCKER_DIR/monkey + + mkdir "$build_dir" + + copy_entrypoint_to_build_dir "$build_dir" + + copy_monkey_island_to_build_dir "$monkey_repo/monkey" "$build_dir" + copy_server_config_to_build_dir "$build_dir" + add_agent_binaries_to_build_dir "$agent_binary_dir" "$build_dir" + + generate_ssl_cert "$build_dir" + + build_frontend "$build_dir" +} + +copy_entrypoint_to_build_dir() { + cp "$DOCKER_DIR"/entrypoint.sh "$1" + chmod 755 "$1/entrypoint.sh" +} + +copy_server_config_to_build_dir() { + cp "$DOCKER_DIR"/server_config.json "$1"/monkey_island/cc +} + +build_package() { + local version=$1 + pushd ./docker + + docker_image_name="guardicore/monkey-island:$version" + tar_name="$DOCKER_DIR/dk.monkeyisland.$version.tar" + + build_docker_image_tar "$docker_image_name" "$tar_name" + build_docker_image_tgz "$tar_name" "$version" + + popd +} + +build_docker_image_tar() { + sudo docker build . -t "$1" + sudo docker save "$1" > "$2" +} + +build_docker_image_tgz() { + mkdir tgz + mv "$1" ./tgz + cp ./DOCKER_README.md ./tgz/README.md + tar -C ./tgz -cvf "$OUTPUT_NAME_TGZ" --gzip . +}