Merge pull request #1101 from guardicore/appimage-v2

Appimage v2

CHANGELOG.md

@@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - PostgreSQL fingerprinter. #892
 - A runtime-configurable option to specify a data directory where runtime
   configuration and other artifacts can be stored. #994
-- Scripts to build a prototype AppImage for Monkey Island. #1069
+- Scripts to build an AppImage for Monkey Island. #1069, #1090
 
 ### Changed
 - server_config.json can be selected at runtime. #963

appimage/.gitignore

@@ -0,0 +1 @@
+*.AppImage

appimage/AppRun

@@ -0,0 +1,29 @@
+#! /bin/bash
+
+# Export APPRUN if running from an extracted image
+self="$(readlink -f -- $0)"
+here="${self%/*}"
+APPDIR="${APPDIR:-${here}}"
+
+# Export TCl/Tk
+export TCL_LIBRARY="${APPDIR}/usr/share/tcltk/tcl8.4"
+export TK_LIBRARY="${APPDIR}/usr/share/tcltk/tk8.4"
+export TKPATH="${TK_LIBRARY}"
+
+# Export SSL certificate
+export SSL_CERT_FILE="${APPDIR}/opt/_internal/certs.pem"
+
+# Call the entry point
+for opt in "$@"
+do
+    [ "${opt:0:1}" != "-" ] && break
+    if [[ "${opt}" =~ "I" ]] || [[ "${opt}" =~ "E" ]]; then
+        # Environment variables are disabled ($PYTHONHOME). Let's run in a safe
+        # mode from the raw Python binary inside the AppImage
+        "$APPDIR/opt/python3.7/bin/python3.7" "$@"
+        exit "$?"
+    fi
+done
+
+(PYTHONHOME="${APPDIR}/opt/python3.7" exec "/bin/bash" "${APPDIR}/usr/src/monkey_island/linux/run_appimage.sh")
+exit "$?"

appimage/README.md

@@ -2,8 +2,8 @@
 
 ## About
 
-This directory contains the necessary artifacts for building a prototype
-monkey_island AppImage using appimage-builder.
+This directory contains the necessary artifacts for building an Infection
+Monkey AppImage
 
 ## Building an AppImage
 
@@ -18,19 +18,18 @@ NOTE: This script is intended to be run from a clean VM. You can also manually
 remove build artifacts by removing the following files and directories.
 
 - $HOME/.monkey_island (optional)
-- $HOME/monkey-appdir
+- $HOME/squashfs-root
 - $HOME/git/monkey
-- $HOME/appimage/appimage-builder-cache
-- $HOME/appimage/"Monkey\ Island-\*-x86-64.Appimage"
+- $HOME/appimage/Infection_Monkey-x86_64.AppImage
 
 After removing the above files and directories, you can again execute `bash
 build_appimage.sh`.
 
 ## Running the AppImage
 
-The build script will produce an AppImage executible named something like
-`Monkey Island-VERSION-x86-64.AppImage`. Simply execute this file and you're
-off to the races.
+The build script will produce an AppImage executible named
+`Infection_Monkey-x86_64.AppImage`. Simply execute this file and you're off to
+the races.
 
 A new directory, `$HOME/.monkey_island` will be created to store runtime
 artifacts.

appimage/build_appimage.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-python_cmd="python3.7"
-APPDIR="$HOME/monkey-appdir"
+APPDIR="$HOME/squashfs-root"
+CONFIG_URL="https://raw.githubusercontent.com/guardicore/monkey/develop/deployment_scripts/config"
 INSTALL_DIR="$APPDIR/usr/src"
 
 GIT=$HOME/git
@@ -13,6 +13,10 @@ ISLAND_PATH="$INSTALL_DIR/monkey_island"
 MONGO_PATH="$ISLAND_PATH/bin/mongodb"
 ISLAND_BINARIES_PATH="$ISLAND_PATH/cc/binaries"
 
+NODE_SRC=https://deb.nodesource.com/setup_12.x
+APP_TOOL_URL=https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage
+PYTHON_APPIMAGE_URL="https://github.com/niess/python-appimage/releases/download/python3.7/python3.7.9-cp37-cp37m-manylinux1_x86_64.AppImage"
+
 is_root() {
   return "$(id -u)"
 }
@@ -33,21 +37,7 @@ log_message() {
   echo -e "DEPLOYMENT SCRIPT: $1"
 }
 
-setup_appdir() {
-    rm -rf "$APPDIR" || true
-    mkdir -p "$INSTALL_DIR"
-}
-
-install_pip_37() {
-    pip_url=https://bootstrap.pypa.io/get-pip.py
-    curl $pip_url -o get-pip.py
-    ${python_cmd} get-pip.py
-    rm get-pip.py
-}
-
 install_nodejs() {
-  NODE_SRC=https://deb.nodesource.com/setup_12.x
-
   log_message "Installing nodejs"
 
   curl -sL $NODE_SRC | sudo -E bash -
@@ -56,28 +46,17 @@ install_nodejs() {
 
 install_build_prereqs() {
     sudo apt update
-    sudo apt upgrade
+    sudo apt upgrade -y
 
-    # appimage-builder prereqs
-    sudo apt install -y python3 python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
-
-    #monkey island prereqs
-    sudo apt install -y curl libcurl4 python3.7 python3.7-dev openssl git build-essential moreutils
-    install_pip_37
+    # monkey island prereqs
+    sudo apt install -y curl libcurl4 openssl git build-essential moreutils
     install_nodejs
 }
 
-install_appimage_builder() {
-    sudo pip3 install appimage-builder
-
-    install_appimage_tool
-}
-
 install_appimage_tool() {
     APP_TOOL_BIN=$HOME/bin/appimagetool
-    APP_TOOL_URL=https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage
 
-    mkdir "$HOME"/bin
+    mkdir -p "$HOME"/bin
     curl -L -o "$APP_TOOL_BIN" "$APP_TOOL_URL"
     chmod u+x "$APP_TOOL_BIN"
 
@@ -88,7 +67,7 @@ load_monkey_binary_config() {
   tmpfile=$(mktemp)
 
   log_message "downloading configuration"
-  curl -L -s -o "$tmpfile" "$config_url"
+  curl -L -s -o "$tmpfile" "$CONFIG_URL"
 
   log_message "loading configuration"
   source "$tmpfile"
@@ -100,17 +79,49 @@ clone_monkey_repo() {
   fi
 
   log_message "Cloning files from git"
-  branch=${2:-"develop"}
+  branch=${1:-"develop"}
   git clone --single-branch --recurse-submodules -b "$branch" "${MONKEY_GIT_URL}" "${REPO_MONKEY_HOME}" 2>&1 || handle_error
 
-  chmod 774 -R "${MONKEY_HOME}"
+  chmod 774 -R "${REPO_MONKEY_HOME}"
+}
+
+setup_appdir() {
+	setup_python_37_appdir
+
+	copy_monkey_island_to_appdir
+	download_monkey_agent_binaries
+
+	install_monkey_island_python_dependencies
+	install_mongodb
+
+	generate_ssl_cert
+	build_frontend
+
+	add_monkey_icon
+	add_desktop_file
+	add_apprun
+}
+
+setup_python_37_appdir() {
+    PYTHON_APPIMAGE="python3.7.9_x86_64.AppImage"
+    rm -rf "$APPDIR" || true
+
+    log_message "downloading Python3.7 Appimage"
+    curl -L -o "$PYTHON_APPIMAGE" "$PYTHON_APPIMAGE_URL"
+
+    chmod u+x "$PYTHON_APPIMAGE"
+
+    ./"$PYTHON_APPIMAGE" --appimage-extract
+    rm "$PYTHON_APPIMAGE"
+    mv ./squashfs-root "$APPDIR"
+    mkdir -p "$INSTALL_DIR"
 }
 
 copy_monkey_island_to_appdir() {
   cp "$REPO_MONKEY_SRC"/__init__.py "$INSTALL_DIR"
   cp "$REPO_MONKEY_SRC"/monkey_island.py "$INSTALL_DIR"
-  cp -r "$REPO_MONKEY_SRC"/common "$INSTALL_DIR"
-  cp -r "$REPO_MONKEY_SRC"/monkey_island "$INSTALL_DIR"
+  cp -r "$REPO_MONKEY_SRC"/common "$INSTALL_DIR/"
+  cp -r "$REPO_MONKEY_SRC"/monkey_island "$INSTALL_DIR/"
   cp ./run_appimage.sh "$INSTALL_DIR"/monkey_island/linux/
   cp ./island_logger_config.json "$INSTALL_DIR"/
   cp ./server_config.json.standard "$INSTALL_DIR"/monkey_island/cc/
@@ -128,7 +139,7 @@ install_monkey_island_python_dependencies() {
   #       dependencies and should not be installed as a runtime requirement.
   cat "$requirements_island" | grep -Piv "virtualenv|pyinstaller" | sponge "$requirements_island"
 
-  ${python_cmd} -m pip install -r "${requirements_island}"  --ignore-installed --prefix /usr --root="$APPDIR" || handle_error
+  "$APPDIR"/AppRun -m pip install -r "${requirements_island}"  --ignore-installed || handle_error
 }
 
 download_monkey_agent_binaries() {
@@ -168,24 +179,26 @@ build_frontend() {
     popd || handle_error
 }
 
+add_monkey_icon() {
+	unlink "$APPDIR"/python.png
+	mkdir -p "$APPDIR"/usr/share/icons
+	cp "$REPO_MONKEY_SRC"/monkey_island/cc/ui/src/images/monkey-icon.svg "$APPDIR"/usr/share/icons/infection-monkey.svg
+	ln -s "$APPDIR"/usr/share/icons/infection-monkey.svg "$APPDIR"/infection-monkey.svg
+}
+
+add_desktop_file() {
+	unlink "$APPDIR"/python3.7.9.desktop
+	cp ./infection-monkey.desktop "$APPDIR"/usr/share/applications
+	ln -s "$APPDIR"/usr/share/applications/infection-monkey.desktop "$APPDIR"/infection-monkey.desktop
+}
+
+add_apprun() {
+	cp ./AppRun "$APPDIR"
+}
+
 build_appimage() {
     log_message "Building AppImage"
-    appimage-builder --recipe monkey_island_builder.yml --log DEBUG --skip-appimage
-
-    # There is a bug or unwanted behavior in appimage-builder that causes issues
-    # if 32-bit binaries are present in the appimage. To work around this, we:
-    #   1. Build the AppDir with appimage-builder and skip building the appimage
-    #   2. Add the 32-bit binaries to the AppDir
-    #   3. Build the AppImage with appimage-builder from the already-built AppDir
-    #
-    # Note that appimage-builder replaces the interpreter on the monkey agent binaries
-    # when building the AppDir. This is unwanted as the monkey agents may execute in
-    # environments where the AppImage isn't loaded.
-    #
-    # See https://github.com/AppImageCrafters/appimage-builder/issues/93 for more info.
-    download_monkey_agent_binaries
-
-    appimage-builder --recipe monkey_island_builder.yml --log DEBUG --skip-build
+    ARCH="x86_64" appimagetool "$APPDIR"
 }
 
 if is_root; then
@@ -199,33 +212,14 @@ Run \`sudo -v\`, enter your password, and then re-run this script."
   exit 1
 fi
 
-config_url="https://raw.githubusercontent.com/mssalvatore/monkey/linux-deploy-binaries/deployment_scripts/config"
-
-setup_appdir
 
 install_build_prereqs
-install_appimage_builder
-
+install_appimage_tool
 
 load_monkey_binary_config
 clone_monkey_repo "$@"
 
-copy_monkey_island_to_appdir
-
-# Create folders
-log_message "Creating island dirs under $ISLAND_PATH"
-mkdir -p "${MONGO_PATH}" || handle_error
-
-install_monkey_island_python_dependencies
-
-install_mongodb
-
-generate_ssl_cert
-
-build_frontend
-
-mkdir -p "$APPDIR"/usr/share/icons
-cp "$REPO_MONKEY_SRC"/monkey_island/cc/ui/src/images/monkey-icon.svg "$APPDIR"/usr/share/icons/monkey-icon.svg
+setup_appdir
 
 build_appimage
 

appimage/infection-monkey.desktop

@@ -0,0 +1,8 @@
+[Desktop Entry]
+Type=Application
+Name=Infection Monkey
+Exec=bash
+Comment=An automated breach and attack simulation platform
+Icon=infection-monkey
+Categories=Development;
+Terminal=true

appimage/run_appimage.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+PYTHON_CMD="$APPDIR"/opt/python3.7/bin/python3.7
+DOT_MONKEY="$HOME"/.monkey_island/
+
+configure_default_logging() {
+    if [ ! -f "$DOT_MONKEY"/island_logger_config.json ]; then
+        cp "$APPDIR"/usr/src/island_logger_config.json "$DOT_MONKEY"
+    fi
+}
+
+configure_default_server() {
+    if [ ! -f "$DOT_MONKEY"/server_config.json ]; then
+        cp "$APPDIR"/usr/src/monkey_island/cc/server_config.json.standard "$DOT_MONKEY"/server_config.json
+    fi
+}
+
+# shellcheck disable=SC2174
+mkdir --mode=0700 --parents "$DOT_MONKEY"
+
+DB_DIR="$DOT_MONKEY"/db
+mkdir --parents "$DB_DIR"
+
+configure_default_logging
+configure_default_server
+
+cd "$APPDIR"/usr/src || exit 1
+./monkey_island/bin/mongodb/bin/mongod --dbpath "$DB_DIR" &
+${PYTHON_CMD} ./monkey_island.py --server-config "$DOT_MONKEY"/server_config.json --logger-config "$DOT_MONKEY"/island_logger_config.json

deployment_scripts/appimage/monkey_island_builder.yml

@@ -1,40 +0,0 @@
-version: 1
-
-AppDir:
-  path: '../monkey-appdir'
-
-  app_info:
-    id: org.guardicore.monkey-island
-    name: Monkey Island
-    icon: monkey-icon
-    version: 1.10.0
-    exec: bin/bash
-    exec_args: "$APPDIR/usr/src/monkey_island/linux/run_appimage.sh"
-
-
-  apt:
-    arch: amd64
-    sources:
-      - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic main restricted
-        key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3B4FE6ACC0B21F32
-      - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic universe
-      - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-security main restricted
-      - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-security universe
-      - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted
-      - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-updates universe
-
-
-    include:
-      - bash
-      - python3.7
-
-  runtime:
-    env:
-      PATH: '${APPDIR}/usr/bin:${PATH}'
-      PYTHONHOME: '${APPDIR}/usr'
-      PYTHONPATH: '${APPDIR}/usr/lib/python3.7/site-packages'
-
-AppImage:
-  update-information: None
-  sign-key: None
-  arch: x86_64

deployment_scripts/appimage/run_appimage.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-DOT_MONKEY=$HOME/.monkey_island/
-
-configure_default_logging() {
-    if [ ! -f $DOT_MONKEY/island_logger_config.json ]; then
-        cp $APPDIR/usr/src/island_logger_config.json $DOT_MONKEY
-    fi
-}
-
-configure_default_server() {
-    if [ ! -f $DOT_MONKEY/server_config.json ]; then
-        cp $APPDIR/usr/src/monkey_island/cc/server_config.json.standard $DOT_MONKEY/server_config.json
-    fi
-}
-
-
-
-# Detecting command that calls python 3.7
-python_cmd=""
-if [[ $(python --version 2>&1) == *"Python 3.7"* ]]; then
-  python_cmd="python"
-fi
-if [[ $(python37 --version 2>&1) == *"Python 3.7"* ]]; then
-  python_cmd="python37"
-fi
-if [[ $(python3.7 --version 2>&1) == *"Python 3.7"* ]]; then
-  python_cmd="python3.7"
-fi
-
-mkdir --mode=0700 --parents $DOT_MONKEY
-
-DB_DIR=$DOT_MONKEY/db
-mkdir -p $DB_DIR
-
-configure_default_logging
-configure_default_server
-
-cd $APPDIR/usr/src
-./monkey_island/bin/mongodb/bin/mongod --dbpath $DB_DIR &
-${python_cmd} ./monkey_island.py --server-config $DOT_MONKEY/server_config.json --logger-config $DOT_MONKEY/island_logger_config.json

monkey/monkey_island/cc/resources/local_run.py

@@ -30,14 +30,15 @@ def run_local_monkey():
     if not result:
         return False, "OS Type not found"
 
-    monkey_path = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "binaries", result["filename"])
-    target_path = os.path.join(env_singleton.env.get_config().data_dir_abs_path, result["filename"])
+    src_path = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "binaries", result["filename"])
+    dest_dir = env_singleton.env.get_config().data_dir_abs_path
+    dest_path = os.path.join(dest_dir, result["filename"])
 
     # copy the executable to temp path (don't run the monkey from its current location as it may
     # delete itself)
     try:
-        copyfile(monkey_path, target_path)
-        os.chmod(target_path, stat.S_IRWXU | stat.S_IRWXG)
+        copyfile(src_path, dest_path)
+        os.chmod(dest_path, stat.S_IRWXU | stat.S_IRWXG)
     except Exception as exc:
         logger.error("Copy file failed", exc_info=True)
         return False, "Copy file failed: %s" % exc
@@ -46,11 +47,11 @@ def run_local_monkey():
     try:
         args = [
             '"%s" m0nk3y -s %s:%s'
-            % (target_path, local_ip_addresses()[0], env_singleton.env.get_island_port())
+            % (dest_path, local_ip_addresses()[0], env_singleton.env.get_island_port())
         ]
         if sys.platform == "win32":
             args = "".join(args)
-        subprocess.Popen(args, shell=True).pid
+        subprocess.Popen(args, cwd=dest_dir, shell=True).pid
     except Exception as exc:
         logger.error("popen failed", exc_info=True)
         return False, "popen failed: %s" % exc

monkey/monkey_island/cc/resources/pba_file_download.py

@@ -1,7 +1,7 @@
 import flask_restful
 from flask import send_from_directory
 
-from monkey_island.cc.services.post_breach_files import ABS_UPLOAD_PATH
+import monkey_island.cc.environment.environment_singleton as env_singleton
 
 __author__ = "VakarisZ"
 
@@ -13,4 +13,4 @@ class PBAFileDownload(flask_restful.Resource):
 
     # Used by monkey. can't secure.
     def get(self, path):
-        return send_from_directory(ABS_UPLOAD_PATH, path)
+        return send_from_directory(env_singleton.env.get_config().data_dir_abs_path, path)

monkey/monkey_island/cc/resources/pba_file_upload.py

@@ -1,15 +1,16 @@
 import copy
 import logging
 import os
+from pathlib import Path
 
 import flask_restful
 from flask import Response, request, send_from_directory
 from werkzeug.utils import secure_filename
 
+import monkey_island.cc.environment.environment_singleton as env_singleton
 from monkey_island.cc.resources.auth.auth import jwt_required
 from monkey_island.cc.services.config import ConfigService
 from monkey_island.cc.services.post_breach_files import (
-    ABS_UPLOAD_PATH,
     PBA_LINUX_FILENAME_PATH,
     PBA_WINDOWS_FILENAME_PATH,
 )
@@ -29,7 +30,7 @@ class FileUpload(flask_restful.Resource):
 
     def __init__(self):
         # Create all directories on the way if they don't exist
-        ABS_UPLOAD_PATH.mkdir(parents=True, exist_ok=True)
+        Path(env_singleton.env.get_config().data_dir_abs_path).mkdir(parents=True, exist_ok=True)
 
     @jwt_required
     def get(self, file_type):
@@ -43,7 +44,7 @@ class FileUpload(flask_restful.Resource):
             filename = ConfigService.get_config_value(copy.deepcopy(PBA_LINUX_FILENAME_PATH))
         else:
             filename = ConfigService.get_config_value(copy.deepcopy(PBA_WINDOWS_FILENAME_PATH))
-        return send_from_directory(ABS_UPLOAD_PATH, filename)
+        return send_from_directory(env_singleton.env.get_config().data_dir_abs_path, filename)
 
     @jwt_required
     def post(self, file_type):
@@ -68,7 +69,7 @@ class FileUpload(flask_restful.Resource):
             PBA_LINUX_FILENAME_PATH if file_type == "PBAlinux" else PBA_WINDOWS_FILENAME_PATH
         )
         filename = ConfigService.get_config_value(filename_path)
-        file_path = ABS_UPLOAD_PATH.joinpath(filename)
+        file_path = Path(env_singleton.env.get_config().data_dir_abs_path).joinpath(filename)
         try:
             if os.path.exists(file_path):
                 os.remove(file_path)
@@ -87,7 +88,9 @@ class FileUpload(flask_restful.Resource):
         :return: filename string
         """
         filename = secure_filename(request_.files["filepond"].filename)
-        file_path = ABS_UPLOAD_PATH.joinpath(filename).absolute()
+        file_path = (
+            Path(env_singleton.env.get_config().data_dir_abs_path).joinpath(filename).absolute()
+        )
         request_.files["filepond"].save(str(file_path))
         ConfigService.set_config_value(
             (PBA_LINUX_FILENAME_PATH if is_linux else PBA_WINDOWS_FILENAME_PATH), filename

monkey/monkey_island/cc/services/post_breach_files.py

@@ -1,12 +1,11 @@
 import logging
 import os
-from pathlib import Path
 
 import monkey_island.cc.services.config
 
 __author__ = "VakarisZ"
 
-from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH
+import monkey_island.cc.environment.environment_singleton as env_singleton
 
 logger = logging.getLogger(__name__)
 
@@ -15,8 +14,6 @@ PBA_WINDOWS_FILENAME_PATH = ["monkey", "post_breach", "PBA_windows_filename"]
 PBA_LINUX_FILENAME_PATH = ["monkey", "post_breach", "PBA_linux_filename"]
 UPLOADS_DIR_NAME = "userUploads"
 
-ABS_UPLOAD_PATH = Path(MONKEY_ISLAND_ABS_PATH, "cc", UPLOADS_DIR_NAME)
-
 
 def remove_PBA_files():
     if monkey_island.cc.services.config.ConfigService.get_config():
@@ -33,7 +30,7 @@ def remove_PBA_files():
 
 
 def remove_file(file_name):
-    file_path = os.path.join(ABS_UPLOAD_PATH, file_name)
+    file_path = os.path.join(env_singleton.env.get_config().data_dir_abs_path, file_name)
     try:
         if os.path.exists(file_path):
             os.remove(file_path)