From ae1427f6a6479f65496f5f97e092f756f63c9de4 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 27 Jul 2021 14:04:35 -0400 Subject: [PATCH] Build: Remove duplicate code from appimage build script --- CHANGELOG.md | 1 + build_scripts/appimage/appimage.sh | 137 +++++++++ build_scripts/appimage/build_appimage.sh | 372 ----------------------- build_scripts/appimage/clean.sh | 8 +- build_scripts/build_package.sh | 4 - build_scripts/common.sh | 2 + 6 files changed, 145 insertions(+), 379 deletions(-) create mode 100755 build_scripts/appimage/appimage.sh delete mode 100755 build_scripts/appimage/build_appimage.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index e21e31019..cf3507068 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Create/check data directory on Island init. #1170 - The formatting of some log messages to make them more readable. #1283 - Some unit tests to run faster. #1125 +- Moved appimage/ to build_scripts/appimage/. #1140 ### Removed - Relevant dead code as reported by Vulture. #1149 diff --git a/build_scripts/appimage/appimage.sh b/build_scripts/appimage/appimage.sh new file mode 100755 index 000000000..eb4bee130 --- /dev/null +++ b/build_scripts/appimage/appimage.sh @@ -0,0 +1,137 @@ +#!/bin/bash + +APP_TOOL_URL=https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage +PYTHON_VERSION="3.7.11" +PYTHON_APPIMAGE_URL="https://github.com/niess/python-appimage/releases/download/python3.7/python${PYTHON_VERSION}-cp37-cp37m-manylinux1_x86_64.AppImage" +APPIMAGE_DIR="$(realpath $(dirname $BASH_SOURCE[0]))" + +source "$APPIMAGE_DIR/../common.sh" + +install_package_specific_build_prereqs() { + log_message "Installing appimagetool" + WORKSPACE_BIN_DIR="$1/bin" + APP_TOOL_BIN="$WORKSPACE_BIN_DIR/appimagetool" + + mkdir -p "$WORKSPACE_BIN_DIR" + curl -L -o "$APP_TOOL_BIN" "$APP_TOOL_URL" + chmod u+x "$APP_TOOL_BIN" + + PATH=$PATH:$WORKSPACE_BIN_DIR +} + +setup_build_dir() { + local agent_binary_dir=$1 + local monkey_repo=$2 + local appdir=$APPIMAGE_DIR/squashfs-root + local build_dir="$appdir/usr/src" + + pushd $APPIMAGE_DIR + + setup_python_37_appdir $build_dir + + mkdir -p "$build_dir" + + copy_monkey_island_to_build_dir "$monkey_repo/monkey" $build_dir + copy_server_config_to_build_dir $build_dir + add_agent_binaries_to_build_dir "$agent_binary_dir" "$build_dir" + + install_monkey_island_python_dependencies "$appdir" "$build_dir" + install_mongodb "$build_dir" + + generate_ssl_cert "$build_dir" + build_frontend "$build_dir" + + add_monkey_icon "$appdir" "$monkey_repo" + add_desktop_file "$appdir" + add_apprun "$appdir" + + popd +} + +setup_python_37_appdir() { + PYTHON_APPIMAGE="python${PYTHON_VERSION}_x86_64.AppImage" + + log_message "downloading Python3.7 Appimage" + curl -L -o "$PYTHON_APPIMAGE" "$PYTHON_APPIMAGE_URL" + + chmod u+x "$PYTHON_APPIMAGE" + + "./$PYTHON_APPIMAGE" --appimage-extract + rm "$PYTHON_APPIMAGE" +} + +copy_server_config_to_build_dir() { + cp "$APPIMAGE_DIR"/server_config.json.standard "$1"/monkey_island/cc/server_config.json +} + +install_monkey_island_python_dependencies() { + local appdir=$1 + local build_dir=$2 + log_message "Installing island requirements" + + log_message "Installing pipenv" + "$appdir"/AppRun -m pip install pipenv || handle_error + + requirements_island="$build_dir/monkey_island/requirements.txt" + generate_requirements_from_pipenv_lock "$appdir" "$build_dir" "$requirements_island" + + log_message "Installing island python requirements" + "$appdir"/AppRun -m pip install -r "${requirements_island}" --ignore-installed || handle_error +} + +generate_requirements_from_pipenv_lock () { + local appdir=$1 + local build_dir=$2 + local requirements_island=$3 + + log_message "Generating a requirements.txt file with 'pipenv lock -r'" + pushd "$build_dir/monkey_island" + "$appdir"/AppRun -m pipenv --python "$appdir/AppRun" lock -r > "$requirements_island" || handle_error + popd +} + +install_mongodb() { + local build_dir=$1 + local mongo_path="$build_dir/monkey_island/bin/mongodb" + log_message "Installing MongoDB" + + mkdir -p "$mongo_path" + "$build_dir/monkey_island/linux/install_mongo.sh" "${mongo_path}" || handle_error +} + +add_monkey_icon() { + local appdir=$1 + local monkey_repo=$2 + + unlink "$appdir"/python.png + mkdir -p "$appdir"/usr/share/icons + cp "$monkey_repo"/monkey/monkey_island/cc/ui/src/images/monkey-icon.svg "$appdir"/usr/share/icons/infection-monkey.svg + ln -s "$appdir"/usr/share/icons/infection-monkey.svg "$appdir"/infection-monkey.svg +} + +add_desktop_file() { + local appdir=$1 + + unlink "$appdir"/python*.desktop + cp ./infection-monkey.desktop "$appdir"/usr/share/applications + ln -s "$appdir"/usr/share/applications/infection-monkey.desktop "$appdir"/infection-monkey.desktop +} + +add_apprun() { + cp ./AppRun "$1" +} + +build_package() { + log_message "Building AppImage" + pushd "$APPIMAGE_DIR" + + ARCH="x86_64" appimagetool "$APPIMAGE_DIR/squashfs-root" + apply_version_to_appimage "$1" + + popd +} + +apply_version_to_appimage() { + log_message "Renaming Infection_Monkey-x86_64.AppImage -> Infection_Monkey-$1-x86_64.AppImage" + mv "Infection_Monkey-x86_64.AppImage" "Infection_Monkey-$1-x86_64.AppImage" +} diff --git a/build_scripts/appimage/build_appimage.sh b/build_scripts/appimage/build_appimage.sh deleted file mode 100755 index 5ed46dfea..000000000 --- a/build_scripts/appimage/build_appimage.sh +++ /dev/null @@ -1,372 +0,0 @@ -#!/bin/bash - -WORKSPACE=${WORKSPACE:-$HOME} - -APPDIR="$PWD/squashfs-root" -INSTALL_DIR="$APPDIR/usr/src" - -GIT=$WORKSPACE/git - -DEFAULT_REPO_MONKEY_HOME=$GIT/monkey - -ISLAND_PATH="$INSTALL_DIR/monkey_island" -MONGO_PATH="$ISLAND_PATH/bin/mongodb" -ISLAND_BINARIES_PATH="$ISLAND_PATH/cc/binaries" - -MONKEY_ORIGIN_URL="https://github.com/guardicore/monkey.git" -CONFIG_URL="https://raw.githubusercontent.com/guardicore/monkey/develop/deployment_scripts/config" -NODE_SRC=https://deb.nodesource.com/setup_12.x -APP_TOOL_URL=https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage -PYTHON_VERSION="3.7.11" -PYTHON_APPIMAGE_URL="https://github.com/niess/python-appimage/releases/download/python3.7/python${PYTHON_VERSION}-cp37-cp37m-manylinux1_x86_64.AppImage" -ISLAND_DIR_COPY_TIMEOUT=60 #Seconds - -exit_if_missing_argument() { - if [ -z "$2" ] || [ "${2:0:1}" == "-" ]; then - echo "Error: Argument for $1 is missing" >&2 - exit 1 - fi -} - -echo_help() { - echo "usage: build_appimage.sh [--help] [--agent-binary-dir ] [--branch ]" - echo " [--monkey-repo ] [--version ]" - echo "" - echo "Creates an AppImage package for Infection Monkey." - echo "" - echo "--agent-binary-dir A directory containing the agent binaries that" - echo " you'd like to include with the AppImage. If this" - echo " parameter is unspecified, the latest release" - echo " binaries will be downloaded from GitHub." - echo "" - echo "--as-root Throw caution to the wind and allow this script" - echo " to be run as root." - echo "" - echo "--branch The git branch you'd like the AppImage to be" - echo " built from. (Default: develop)" - echo "" - echo "--monkey-repo A directory containing the Infection Monkey git" - echo " repository. If the directory is empty or does" - echo " not exist, a new repo will be cloned from GitHub." - echo " If the directory is already a valid GitHub repo," - echo " it will be used as-is and the --branch parameter" - echo " will have no effect." - echo " (Default: $DEFAULT_REPO_MONKEY_HOME)" - echo "" - echo "--version A version number for the AppImage package." - echo " (Default: dev)" - - exit 0 -} - -is_root() { - return "$(id -u)" -} - -has_sudo() { - # 0 true, 1 false - sudo -nv > /dev/null 2>&1 - return $? -} - -handle_error() { - echo "Fix the errors above and rerun the script" - exit 1 -} - -log_message() { - echo -e "\n\n" - echo -e "APPIMAGE BUILDER: $1" -} - -install_nodejs() { - log_message "Installing nodejs" - - curl -sL $NODE_SRC | sudo -E bash - - sudo apt-get install -y nodejs -} - -install_build_prereqs() { - sudo apt-get update - sudo apt-get upgrade -y - - # monkey island prereqs - sudo apt-get install -y curl libcurl4 openssl git build-essential moreutils - install_nodejs -} - -install_appimage_tool() { - log_message "Installing appimagetool" - APP_TOOL_BIN=$WORKSPACE/bin/appimagetool - - mkdir -p "$WORKSPACE"/bin - curl -L -o "$APP_TOOL_BIN" "$APP_TOOL_URL" - chmod u+x "$APP_TOOL_BIN" - - PATH=$PATH:$WORKSPACE/bin -} - -is_valid_git_repo() { - pushd "$1" 2>/dev/null || return 1 - git status >/dev/null 2>&1 - success="$?" - popd || exit 1 - - return $success -} - -clone_monkey_repo() { - local repo_dir=$1 - local branch=$2 - - if [[ ! -d "$repo_dir" ]]; then - mkdir -p "$repo_dir" - fi - - log_message "Cloning files from git" - git clone -c core.autocrlf=false --single-branch --recurse-submodules -b "$branch" "$MONKEY_ORIGIN_URL" "$repo_dir" 2>&1 || handle_error -} - -setup_appdir() { - local agent_binary_dir=$1 - local monkey_repo=$2 - - setup_python_37_appdir - - copy_monkey_island_to_appdir "$monkey_repo"/monkey - add_agent_binaries_to_appdir "$agent_binary_dir" - - install_monkey_island_python_dependencies - install_mongodb - - generate_ssl_cert - build_frontend - - add_monkey_icon "$monkey_repo"/monkey - add_desktop_file - add_apprun -} - -setup_python_37_appdir() { - PYTHON_APPIMAGE="python${PYTHON_VERSION}_x86_64.AppImage" - rm -rf "$APPDIR" || true - - log_message "downloading Python3.7 Appimage" - curl -L -o "$PYTHON_APPIMAGE" "$PYTHON_APPIMAGE_URL" - - chmod u+x "$PYTHON_APPIMAGE" - - ./"$PYTHON_APPIMAGE" --appimage-extract - rm "$PYTHON_APPIMAGE" - mkdir -p "$INSTALL_DIR" -} - -copy_monkey_island_to_appdir() { - cp "$1"/__init__.py "$INSTALL_DIR" - cp "$1"/monkey_island.py "$INSTALL_DIR" - cp -r "$1"/common "$INSTALL_DIR/" - if ! timeout "${ISLAND_DIR_COPY_TIMEOUT}" cp -r "$1"/monkey_island "$INSTALL_DIR/"; then - log_message "Copying island files takes too long. Maybe you're copying a dev folder instead of a fresh repository?" - exit 1 - fi - cp ./server_config.json.standard "$INSTALL_DIR"/monkey_island/cc/ - - # TODO: This is a workaround that may be able to be removed after PR #848 is - # merged. See monkey_island/cc/environment_singleton.py for more information. - cp ./server_config.json.standard "$INSTALL_DIR"/monkey_island/cc/server_config.json -} - -install_monkey_island_python_dependencies() { - log_message "Installing island requirements" - - log_message "Installing pipenv" - "$APPDIR"/AppRun -m pip install pipenv || handle_error - - requirements_island="$ISLAND_PATH/requirements.txt" - generate_requirements_from_pipenv_lock "$requirements_island" - - log_message "Installing island python requirements" - "$APPDIR"/AppRun -m pip install -r "${requirements_island}" --ignore-installed || handle_error -} - -generate_requirements_from_pipenv_lock () { - log_message "Generating a requirements.txt file with 'pipenv lock -r'" - cd "$ISLAND_PATH" || exit 1 - "$APPDIR"/AppRun -m pipenv --python "$APPDIR/AppRun" lock -r > "$1" || handle_error - cd - || exit 1 -} - -add_agent_binaries_to_appdir() { - if [ -z "$1" ]; then - download_monkey_agent_binaries_to_appdir - else - copy_agent_binaries_to_appdir "$1" - fi - - make_linux_binaries_executable -} - -download_monkey_agent_binaries_to_appdir() { - log_message "Downloading monkey agent binaries to ${ISLAND_BINARIES_PATH}" - - load_monkey_binary_config - - mkdir -p "${ISLAND_BINARIES_PATH}" || handle_error - curl -L -o "${ISLAND_BINARIES_PATH}/${LINUX_32_BINARY_NAME}" "${LINUX_32_BINARY_URL}" - curl -L -o "${ISLAND_BINARIES_PATH}/${LINUX_64_BINARY_NAME}" "${LINUX_64_BINARY_URL}" - curl -L -o "${ISLAND_BINARIES_PATH}/${WINDOWS_32_BINARY_NAME}" "${WINDOWS_32_BINARY_URL}" - curl -L -o "${ISLAND_BINARIES_PATH}/${WINDOWS_64_BINARY_NAME}" "${WINDOWS_64_BINARY_URL}" -} - -copy_agent_binaries_to_appdir() { - cp "$1"/* "$ISLAND_BINARIES_PATH/" -} - -make_linux_binaries_executable() { - chmod a+x "$ISLAND_BINARIES_PATH"/monkey-linux-* -} - -load_monkey_binary_config() { - tmpfile=$(mktemp) - - log_message "Downloading prebuilt binary configuration" - curl -L -s -o "$tmpfile" "$CONFIG_URL" - - log_message "Loading configuration" - source "$tmpfile" -} - -install_mongodb() { - log_message "Installing MongoDB" - - mkdir -p "$MONGO_PATH" - "${ISLAND_PATH}"/linux/install_mongo.sh "${MONGO_PATH}" || handle_error -} - -generate_ssl_cert() { - log_message "Generating certificate" - - chmod u+x "${ISLAND_PATH}"/linux/create_certificate.sh - "${ISLAND_PATH}"/linux/create_certificate.sh "${ISLAND_PATH}"/cc -} - -build_frontend() { - pushd "$ISLAND_PATH/cc/ui" || handle_error - - log_message "Generating front end" - npm ci - npm run dist - - popd || handle_error - - remove_node_modules -} - -remove_node_modules() { - # Node has served its purpose. We don't need to deliver the node modules with - # the AppImage. - rm -rf "$ISLAND_PATH"/cc/ui/node_modules -} - -add_monkey_icon() { - unlink "$APPDIR"/python.png - mkdir -p "$APPDIR"/usr/share/icons - cp "$1"/monkey_island/cc/ui/src/images/monkey-icon.svg "$APPDIR"/usr/share/icons/infection-monkey.svg - ln -s "$APPDIR"/usr/share/icons/infection-monkey.svg "$APPDIR"/infection-monkey.svg -} - -add_desktop_file() { - unlink "$APPDIR/python${PYTHON_VERSION}.desktop" - cp ./infection-monkey.desktop "$APPDIR"/usr/share/applications - ln -s "$APPDIR"/usr/share/applications/infection-monkey.desktop "$APPDIR"/infection-monkey.desktop -} - -add_apprun() { - cp ./AppRun "$APPDIR" -} - -build_appimage() { - log_message "Building AppImage" - ARCH="x86_64" appimagetool "$APPDIR" - apply_version_to_appimage "$1" -} - -apply_version_to_appimage() { - log_message "Renaming Infection_Monkey-x86_64.AppImage -> Infection_Monkey-$1-x86_64.AppImage" - mv "Infection_Monkey-x86_64.AppImage" "Infection_Monkey-$1-x86_64.AppImage" -} - -agent_binary_dir="" -as_root=false -branch="develop" -monkey_repo="$DEFAULT_REPO_MONKEY_HOME" -monkey_version="dev" - - -while (( "$#" )); do - case "$1" in - --agent-binary-dir) - exit_if_missing_argument "$1" "$2" - - agent_binary_dir=$2 - shift 2 - ;; - --as-root) - as_root=true - shift - ;; - --branch) - exit_if_missing_argument "$1" "$2" - - branch=$2 - shift 2 - ;; - -h|--help) - echo_help - ;; - --monkey-repo) - exit_if_missing_argument "$1" "$2" - - monkey_repo=$2 - shift 2 - ;; - --version) - exit_if_missing_argument "$1" "$2" - - monkey_version=$2 - shift 2 - ;; - *) - echo "Error: Unsupported parameter $1" >&2 - exit 1 - ;; - esac -done - -log_message "Building Monkey Island AppImage package." - -if ! $as_root && is_root; then - log_message "Please don't run this script as root" - exit 1 -fi - -if ! has_sudo; then - log_message "You need root permissions for some of this script operations. \ -Run \`sudo -v\`, enter your password, and then re-run this script." - exit 1 -fi - - -install_build_prereqs -install_appimage_tool - -if ! is_valid_git_repo "$monkey_repo"; then - clone_monkey_repo "$monkey_repo" "$branch" -fi - -setup_appdir "$agent_binary_dir" "$monkey_repo" - -build_appimage "$monkey_version" - -log_message "AppImage build script finished." -exit 0 diff --git a/build_scripts/appimage/clean.sh b/build_scripts/appimage/clean.sh index f48837d37..ee9303a10 100755 --- a/build_scripts/appimage/clean.sh +++ b/build_scripts/appimage/clean.sh @@ -3,7 +3,9 @@ # This is a utility script to clean up after a failed or successful AppImage build # in order to speed up development and debugging. -rm -rf "$HOME/.monkey_island" -rm -rf "$HOME/appimage/squashfs-root" +APPIMAGE_DIR="$(realpath $(dirname $BASH_SOURCE[0]))" + rm -rf "$HOME/git/monkey" -rm $HOME/appimage/Infection_Monkey*x86_64.AppImage +rm -rf "$HOME/.monkey_island" +rm -rf "$APPIMAGE_DIR/squashfs-root" +rm "$APPIMAGE_DIR"/Infection_Monkey*x86_64.AppImage diff --git a/build_scripts/build_package.sh b/build_scripts/build_package.sh index a7ec5d4e7..ee466d68c 100755 --- a/build_scripts/build_package.sh +++ b/build_scripts/build_package.sh @@ -1,10 +1,6 @@ WORKSPACE=${WORKSPACE:-$HOME} DEFAULT_REPO_MONKEY_HOME=$WORKSPACE/git/monkey - - - MONKEY_ORIGIN_URL="https://github.com/guardicore/monkey.git" -CONFIG_URL="https://raw.githubusercontent.com/guardicore/monkey/develop/deployment_scripts/config" NODE_SRC=https://deb.nodesource.com/setup_12.x log_message() { diff --git a/build_scripts/common.sh b/build_scripts/common.sh index c5a062f5a..85f794128 100644 --- a/build_scripts/common.sh +++ b/build_scripts/common.sh @@ -1,3 +1,5 @@ +CONFIG_URL="https://raw.githubusercontent.com/guardicore/monkey/develop/deployment_scripts/config" + copy_monkey_island_to_build_dir() { local src=$1 local build_dir=$2