From 12c40c396888e15458d6c9bb96736733eae28178 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 8 Feb 2021 09:09:42 -0500 Subject: [PATCH 01/52] build: scripts for building MonkeyIsland as an appimage Adds the basic scripts for generating an AppImage. Code changes are required to allow Monkey Island to operate on a read-only filesystem. --- deployment_scripts/appimage/AppRun | 5 + deployment_scripts/appimage/build_appimage.sh | 238 ++++++++++++++++++ .../appimage/monkey_island.desktop | 10 + .../appimage/monkey_island_builder.yml | 44 ++++ deployment_scripts/appimage/run.sh | 21 ++ 5 files changed, 318 insertions(+) create mode 100644 deployment_scripts/appimage/AppRun create mode 100755 deployment_scripts/appimage/build_appimage.sh create mode 100644 deployment_scripts/appimage/monkey_island.desktop create mode 100644 deployment_scripts/appimage/monkey_island_builder.yml create mode 100644 deployment_scripts/appimage/run.sh diff --git a/deployment_scripts/appimage/AppRun b/deployment_scripts/appimage/AppRun new file mode 100644 index 000000000..29d0197c4 --- /dev/null +++ b/deployment_scripts/appimage/AppRun @@ -0,0 +1,5 @@ +#!/bin/bash + +cd ./usr/src + +bash monkey_island/linux/run.sh $HOME/.monkey_island/db diff --git a/deployment_scripts/appimage/build_appimage.sh b/deployment_scripts/appimage/build_appimage.sh new file mode 100755 index 000000000..407df9bb7 --- /dev/null +++ b/deployment_scripts/appimage/build_appimage.sh @@ -0,0 +1,238 @@ +#!/bin/bash + +python_cmd="python3.7" +APPDIR="$HOME/monkey-appdir" +INSTALL_DIR="$APPDIR/usr/src" + +GIT=$HOME/git + +REPO_MONKEY_HOME=$GIT/monkey +REPO_MONKEY_SRC=$REPO_MONKEY_HOME/monkey + +ISLAND_PATH="$INSTALL_DIR/monkey_island" +MONGO_PATH="$ISLAND_PATH/bin/mongodb" +ISLAND_BINARIES_PATH="$ISLAND_PATH/cc/binaries" +INFECTION_MONKEY_DIR="$MONKEY_HOME/monkey/infection_monkey" +MONKEY_BIN_DIR="$INFECTION_MONKEY_DIR/bin" + +is_root() { + return $(id -u) +} + +has_sudo() { + # 0 true, 1 false + return $(sudo -nv > /dev/null 2>&1) +} + +handle_error() { + echo "Fix the errors above and rerun the script" + exit 1 +} + +log_message() { + echo -e "\n\n" + echo -e "DEPLOYMENT SCRIPT: $1" +} + +setup_appdir() { + rm -rf $APPDIR | true + mkdir -p $INSTALL_DIR +} + +install_pip_37() { + pip_url=https://bootstrap.pypa.io/get-pip.py + curl $pip_url -o get-pip.py + ${python_cmd} get-pip.py + rm get-pip.py +} + +install_nodejs() { + log_message "Installing nodejs" + node_src=https://deb.nodesource.com/setup_12.x + curl -sL $node_src | sudo -E bash - + sudo apt-get install -y nodejs +} + +install_build_prereqs() { + # appimage-builder prereqs + sudo apt install -y python3 python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace + + #monkey island prereqs + sudo apt install -y curl libcurl4 python3.7 python3.7-dev openssl git build-essential + install_pip_37 + install_nodejs +} + +install_appimage_builder() { + sudo pip3 install appimage-builder + + install_appimage_tool +} + +install_appimage_tool() { + APP_TOOL_BIN=$HOME/bin/appimagetool + mkdir $HOME/bin + curl -L -o $APP_TOOL_BIN https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage + chmod u+x $APP_TOOL_BIN + + PATH=$PATH:$HOME/bin +} + +load_monkey_binary_config() { + tmpfile=$(mktemp) + + log_message "downloading configuration" + curl -L -s -o $tmpfile "$config_url" + + log_message "loading configuration" + source $tmpfile +} + +clone_monkey_repo() { + if [[ ! -d ${GIT} ]]; then + mkdir -p "${GIT}" + fi + + log_message "Cloning files from git" + branch=${2:-"develop"} + git clone --single-branch --recurse-submodules -b "$branch" "${MONKEY_GIT_URL}" "${REPO_MONKEY_HOME}" 2>&1 || handle_error + + chmod 774 -R "${MONKEY_HOME}" +} + +copy_monkey_island_to_appdir() { + cp $REPO_MONKEY_SRC/__init__.py $INSTALL_DIR + cp $REPO_MONKEY_SRC/monkey_island.py $INSTALL_DIR + cp -r $REPO_MONKEY_SRC/common $INSTALL_DIR + cp -r $REPO_MONKEY_SRC/monkey_island $INSTALL_DIR + cp ./run.sh $INSTALL_DIR/monkey_island/linux/ +} + +install_monkey_island_python_dependencies() { + log_message "Installing island requirements" + requirements_island="$ISLAND_PATH/requirements.txt" + ${python_cmd} -m pip install -r "${requirements_island}" --ignore-installed --prefix /usr --root=$APPDIR || handle_error + ${python_cmd} -m pip install pyjwt==1.7 --ignore-installed -U --prefix /usr --root=$APPDIR || handle_error + #${python_cmd} -m pip install PyNacl --user --upgrade || handle_error +} + +download_monkey_agent_binaries() { +log_message "Downloading monkey agent binaries to ${ISLAND_BINARIES_PATH}" + mkdir -p "${ISLAND_BINARIES_PATH}" || handle_error + curl -L -o ${ISLAND_BINARIES_PATH}/${LINUX_32_BINARY_NAME} ${LINUX_32_BINARY_URL} + curl -L -o ${ISLAND_BINARIES_PATH}/${LINUX_64_BINARY_NAME} ${LINUX_64_BINARY_URL} + curl -L -o ${ISLAND_BINARIES_PATH}/${WINDOWS_32_BINARY_NAME} ${WINDOWS_32_BINARY_URL} + curl -L -o ${ISLAND_BINARIES_PATH}/${WINDOWS_64_BINARY_NAME} ${WINDOWS_64_BINARY_URL} + + # Allow them to be executed + chmod a+x "$ISLAND_BINARIES_PATH/$LINUX_32_BINARY_NAME" + chmod a+x "$ISLAND_BINARIES_PATH/$LINUX_64_BINARY_NAME" +} + +install_mongodb() { + #log_message "Installing libcurl4" + #sudo apt-get install -y libcurl4 + + log_message "Installing MongoDB" + + mkdir -p $MONGO_PATH + "${ISLAND_PATH}"/linux/install_mongo.sh ${MONGO_PATH} || handle_error +} + +generate_ssl_cert() { + # Generate SSL certificate + log_message "Generating certificate" + + chmod u+x "${ISLAND_PATH}"/linux/create_certificate.sh + "${ISLAND_PATH}"/linux/create_certificate.sh ${ISLAND_PATH}/cc +} + +build_frontend() { + pushd "$ISLAND_PATH/cc/ui" || handle_error + npm install sass-loader node-sass webpack --save-dev + npm update + + log_message "Generating front end" + npm run dist + popd || handle_error +} + +download_monkey_helper_binaries() { + # Making dir for binaries + mkdir "${MONKEY_BIN_DIR}" + + download_sambacry_binaries + download_tracerout_binaries +} + +download_sambacry_binaries() { + # Download sambacry binaries + log_message "Downloading sambacry binaries" + curl -L -o ${MONKEY_BIN_DIR}/sc_monkey_runner64.so ${SAMBACRY_64_BINARY_URL} + curl -L -o ${MONKEY_BIN_DIR}/sc_monkey_runner32.so ${SAMBACRY_32_BINARY_URL} +} + +download_tracerout_binaries() { + # Download traceroute binaries + log_message "Downloading traceroute binaries" + curl -L -o ${MONKEY_BIN_DIR}/traceroute64 ${TRACEROUTE_64_BINARY_URL} + curl -L -o ${MONKEY_BIN_DIR}/traceroute32 ${TRACEROUTE_32_BINARY_URL} +} + +if is_root; then + log_message "Please don't run this script as root" + exit 1 +fi + +if ! has_sudo; then + log_message "You need root permissions for some of this script operations. \ +Run \`sudo -v\`, enter your password, and then re-run this script." + exit 1 +fi + +config_url="https://raw.githubusercontent.com/mssalvatore/monkey/linux-deploy-binaries/deployment_scripts/config" + +setup_appdir + +install_build_prereqs +install_appimage_builder + + +load_monkey_binary_config +clone_monkey_repo +download_monkey_helper_binaries + +copy_monkey_island_to_appdir +# Create folders +log_message "Creating island dirs under $ISLAND_PATH" +mkdir -p "${MONGO_PATH}" || handle_error + +#log_message "Installing python3-distutils" +#sudo apt-get install -y python3-distutils +install_monkey_island_python_dependencies + +#log_message "Installing monkey requirements" +#sudo apt-get install -y libffi-dev upx libssl-dev libc++1 +#requirements_monkey="$INFECTION_MONKEY_DIR/requirements.txt" +#${python_cmd} -m pip install -r "${requirements_monkey}" --user --upgrade || handle_error + +download_monkey_agent_binaries + +install_mongodb + +generate_ssl_cert + +build_frontend + +#cp ./AppRun $APPDIR + +mkdir -p $APPDIR/usr/share/icons +cp $REPO_MONKEY_SRC/monkey_island/cc/ui/src/images/monkey-icon.svg $APPDIR/usr/share/icons/monkey-icon.svg +#cp ./monkey_island.desktop $APPDIR + +log_message "Building AppImage" +appimage-builder --recipe monkey_island_builder.yml --skip-appimage + + +log_message "Deployment script finished." +exit 0 diff --git a/deployment_scripts/appimage/monkey_island.desktop b/deployment_scripts/appimage/monkey_island.desktop new file mode 100644 index 000000000..bfa2b8955 --- /dev/null +++ b/deployment_scripts/appimage/monkey_island.desktop @@ -0,0 +1,10 @@ +[Desktop Entry] +Version=1.10 +Type=Application +Name=Monkey Island +Comment=FILL ME IN +TryExec=fooview +Exec=AppRun +Icon=monkey_island.svg +Terminal=true; +Category=Security; diff --git a/deployment_scripts/appimage/monkey_island_builder.yml b/deployment_scripts/appimage/monkey_island_builder.yml new file mode 100644 index 000000000..f0848211a --- /dev/null +++ b/deployment_scripts/appimage/monkey_island_builder.yml @@ -0,0 +1,44 @@ +version: 1 + +AppDir: + path: '../monkey-appdir' + + app_info: + id: org.guardicore.monkey-island + name: Monkey Island + icon: monkey-icon + version: 1.10.0 + exec: bin/bash + exec_args: "$APPDIR/usr/src/monkey_island/linux/run.sh $HOME/.monkey_island/db" + + + apt: + arch: amd64 + sources: + - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic main restricted + key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3B4FE6ACC0B21F32 + - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic universe + - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-security main restricted + - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-security universe + - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted + - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-updates universe + + include: + - libc6 + - bash + - curl + - libcurl4 + - coreutils + - python3.7 + - nodejs + + runtime: + env: + PATH: '${APPDIR}/usr/bin:${PATH}' + PYTHONHOME: '${APPDIR}/usr' + PYTHONPATH: '${APPDIR}/usr/lib/python3.7/site-packages' + +AppImage: + update-information: None + sign-key: None + arch: x86_64 diff --git a/deployment_scripts/appimage/run.sh b/deployment_scripts/appimage/run.sh new file mode 100644 index 000000000..328535e78 --- /dev/null +++ b/deployment_scripts/appimage/run.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Detecting command that calls python 3.7 +python_cmd="" +if [[ $(python --version 2>&1) == *"Python 3.7"* ]]; then + python_cmd="python" +fi +if [[ $(python37 --version 2>&1) == *"Python 3.7"* ]]; then + python_cmd="python37" +fi +if [[ $(python3.7 --version 2>&1) == *"Python 3.7"* ]]; then + python_cmd="python3.7" +fi + +DB_DIR=${1:-"./monkey_island/bin/mongodb/db"} + +mkdir -p $DB_DIR + +cd $APPDIR/usr/src +./monkey_island/bin/mongodb/bin/mongod --dbpath $DB_DIR & +${python_cmd} ./monkey_island.py From 0230c26f19012ed21298b25bdfd62d6d2f522d2e Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 8 Feb 2021 11:18:18 -0500 Subject: [PATCH 02/52] cc: allow server_config.json to be specified at runtime --- monkey/monkey_island.py | 13 +++++++--- .../cc/environment/environment_config.py | 20 +++++++++------ .../cc/environment/environment_singleton.py | 25 +++++++++++-------- .../cc/environment/test_environment_config.py | 7 +++--- monkey/monkey_island/cc/main.py | 5 +++- 5 files changed, 44 insertions(+), 26 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index cd452066c..2e410cd9f 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -3,19 +3,24 @@ from gevent import monkey as gevent_monkey gevent_monkey.patch_all() from monkey_island.cc.main import main +from monkey_island.cc.environment.environment_config import DEFAULT_SERVER_CONFIG_PATH def parse_cli_args(): import argparse - parser = argparse.ArgumentParser(description="Infection Monkey Island CnC Server. See https://infectionmonkey.com") + parser = argparse.ArgumentParser(description="Infection Monkey Island CnC Server. See https://infectionmonkey.com", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-s", "--setup-only", action="store_true", help="Pass this flag to cause the Island to setup and exit without actually starting. " "This is useful for preparing Island to boot faster later-on, so for " "compiling/packaging Islands.") + parser.add_argument("-c", "--config", action="store", + help="The path to the server configuration file.", + default=DEFAULT_SERVER_CONFIG_PATH) args = parser.parse_args() - return args.setup_only + return (args.setup_only, args.config) if "__main__" == __name__: - is_setup_only = parse_cli_args() - main(is_setup_only) + (is_setup_only, config) = parse_cli_args() + main(is_setup_only, config) diff --git a/monkey/monkey_island/cc/environment/environment_config.py b/monkey/monkey_island/cc/environment/environment_config.py index 35dbafc8e..a680658a2 100644 --- a/monkey/monkey_island/cc/environment/environment_config.py +++ b/monkey/monkey_island/cc/environment/environment_config.py @@ -12,6 +12,7 @@ from monkey_island.cc.resources.auth.auth_user import User from monkey_island.cc.resources.auth.user_store import UserStore SERVER_CONFIG_FILENAME = "server_config.json" +DEFAULT_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', SERVER_CONFIG_FILENAME) class EnvironmentConfig: @@ -20,6 +21,7 @@ class EnvironmentConfig: deployment: str, user_creds: UserCreds, aws=None): + self.server_config_path = None self.server_config = server_config self.deployment = deployment self.user_creds = user_creds @@ -40,22 +42,24 @@ class EnvironmentConfig: aws=aws) def save_to_file(self): - file_path = EnvironmentConfig.get_config_file_path() - with open(file_path, 'w') as f: + with open(self.server_config_path, 'w') as f: f.write(json.dumps(self.to_dict(), indent=2)) @staticmethod - def get_from_file() -> EnvironmentConfig: - file_path = EnvironmentConfig.get_config_file_path() + def get_from_file(file_path=DEFAULT_SERVER_CONFIG_PATH) -> EnvironmentConfig: + file_path = os.path.expanduser(file_path) + if not Path(file_path).is_file(): server_config_generator.create_default_config_file(file_path) with open(file_path, 'r') as f: config_content = f.read() - return EnvironmentConfig.get_from_json(config_content) - @staticmethod - def get_config_file_path() -> str: - return os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', SERVER_CONFIG_FILENAME) + environment_config = EnvironmentConfig.get_from_json(config_content) + # TODO: Populating this property is not ideal. Revisit this when you + # make the logger config file configurable at runtime. + environment_config.server_config_path = file_path + + return environment_config def to_dict(self) -> Dict: config_dict = {'server_config': self.server_config, diff --git a/monkey/monkey_island/cc/environment/environment_singleton.py b/monkey/monkey_island/cc/environment/environment_singleton.py index 6b98d0b7c..aab06285f 100644 --- a/monkey/monkey_island/cc/environment/environment_singleton.py +++ b/monkey/monkey_island/cc/environment/environment_singleton.py @@ -1,7 +1,9 @@ import logging import monkey_island.cc.resources.auth.user_store as user_store -from monkey_island.cc.environment import EnvironmentConfig, aws, password, standard, testing +from monkey_island.cc.environment import (EnvironmentConfig, aws, password, + standard, testing) +from monkey_island.cc.environment.environment_config import DEFAULT_SERVER_CONFIG_PATH __author__ = 'itay.mizeretz' @@ -35,13 +37,16 @@ def set_to_standard(): env.save_config() user_store.UserStore.set_users(env.get_auth_users()) +def initialize_from_file(file_path): + try: + config = EnvironmentConfig.get_from_file(file_path) -try: - config = EnvironmentConfig.get_from_file() - __env_type = config.server_config - set_env(__env_type, config) - # noinspection PyUnresolvedReferences - logger.info('Monkey\'s env is: {0}'.format(env.__class__.__name__)) -except Exception: - logger.error('Failed initializing environment', exc_info=True) - raise + __env_type = config.server_config + set_env(__env_type, config) + # noinspection PyUnresolvedReferences + logger.info('Monkey\'s env is: {0}'.format(env.__class__.__name__)) + except Exception: + logger.error('Failed initializing environment', exc_info=True) + raise + +initialize_from_file(DEFAULT_SERVER_CONFIG_PATH) diff --git a/monkey/monkey_island/cc/environment/test_environment_config.py b/monkey/monkey_island/cc/environment/test_environment_config.py index ed9b0ef96..2f076a3a0 100644 --- a/monkey/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/monkey_island/cc/environment/test_environment_config.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock, patch import monkey_island.cc.test_common.environment.server_config_mocks as config_mocks from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH -from monkey_island.cc.environment.environment_config import EnvironmentConfig +from monkey_island.cc.environment.environment_config import EnvironmentConfig, DEFAULT_SERVER_CONFIG from monkey_island.cc.environment.user_creds import UserCreds @@ -46,6 +46,7 @@ class TestEnvironmentConfig(TestCase): env_config = EnvironmentConfig(server_config=config['server_config'], deployment=config['deployment'], user_creds=user_creds) + env_config.server_config_path = get_server_config_file_path_test_version() env_config.save_to_file() file_path = get_server_config_file_path_test_version() @@ -55,12 +56,12 @@ class TestEnvironmentConfig(TestCase): self.assertDictEqual(config, json.loads(content_from_file)) - def test_get_server_config_file_path(self): + def test_default_server_config_file_path(self): if platform.system() == "Windows": server_file_path = MONKEY_ISLAND_ABS_PATH + r"\cc\server_config.json" else: server_file_path = MONKEY_ISLAND_ABS_PATH + "/cc/server_config.json" - self.assertEqual(EnvironmentConfig.get_config_file_path(), server_file_path) + self.assertEqual(DEFAULT_SERVER_CONFIG_PATH, server_file_path) def test_get_from_dict(self): config_dict = config_mocks.CONFIG_WITH_CREDENTIALS diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index ce142edcc..27d928134 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -21,6 +21,7 @@ json_setup_logging(default_path=Path(MONKEY_ISLAND_ABS_PATH, 'cc', 'island_logge logger = logging.getLogger(__name__) import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 +from monkey_island.cc.environment.environment_config import DEFAULT_SERVER_CONFIG_PATH from common.version import get_version # noqa: E402 from monkey_island.cc.app import init_app # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 @@ -34,8 +35,10 @@ from monkey_island.cc.setup import setup # noqa: E402 MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" -def main(should_setup_only=False): +def main(should_setup_only=False, server_config_filename=DEFAULT_SERVER_CONFIG_PATH): logger.info("Starting bootloader server") + env_singleton.initialize_from_file(server_config_filename) + mongo_url = os.environ.get('MONGO_URL', env_singleton.env.get_mongo_url()) bootloader_server_thread = Thread(target=BootloaderHttpServer(mongo_url).serve_forever, daemon=True) From 1d73f6e860a1111d5557d607ce9fa232912cd2da Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 8 Feb 2021 13:04:33 -0500 Subject: [PATCH 03/52] cc: move DEFAULT_SERVER_CONFIG_PATH to consts.py --- .../cc/environment/environment_config.py | 4 +--- .../cc/environment/environment_singleton.py | 2 +- .../cc/environment/test_environment_config.py | 10 +--------- monkey/monkey_island/cc/main.py | 2 +- monkey/monkey_island/cc/server_utils/consts.py | 3 +++ monkey/monkey_island/cc/test_consts.py | 11 +++++++++++ 6 files changed, 18 insertions(+), 14 deletions(-) create mode 100644 monkey/monkey_island/cc/test_consts.py diff --git a/monkey/monkey_island/cc/environment/environment_config.py b/monkey/monkey_island/cc/environment/environment_config.py index a680658a2..006f4d233 100644 --- a/monkey/monkey_island/cc/environment/environment_config.py +++ b/monkey/monkey_island/cc/environment/environment_config.py @@ -6,13 +6,11 @@ from pathlib import Path from typing import Dict, List import monkey_island.cc.environment.server_config_generator as server_config_generator -from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH +from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH from monkey_island.cc.environment.user_creds import UserCreds from monkey_island.cc.resources.auth.auth_user import User from monkey_island.cc.resources.auth.user_store import UserStore -SERVER_CONFIG_FILENAME = "server_config.json" -DEFAULT_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', SERVER_CONFIG_FILENAME) class EnvironmentConfig: diff --git a/monkey/monkey_island/cc/environment/environment_singleton.py b/monkey/monkey_island/cc/environment/environment_singleton.py index aab06285f..6934c5cd2 100644 --- a/monkey/monkey_island/cc/environment/environment_singleton.py +++ b/monkey/monkey_island/cc/environment/environment_singleton.py @@ -3,7 +3,7 @@ import logging import monkey_island.cc.resources.auth.user_store as user_store from monkey_island.cc.environment import (EnvironmentConfig, aws, password, standard, testing) -from monkey_island.cc.environment.environment_config import DEFAULT_SERVER_CONFIG_PATH +from monkey_island.cc.consts import DEFAULT_SERVER_CONFIG_PATH __author__ = 'itay.mizeretz' diff --git a/monkey/monkey_island/cc/environment/test_environment_config.py b/monkey/monkey_island/cc/environment/test_environment_config.py index 2f076a3a0..d00fd1de1 100644 --- a/monkey/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/monkey_island/cc/environment/test_environment_config.py @@ -1,13 +1,12 @@ import json import os -import platform from typing import Dict from unittest import TestCase from unittest.mock import MagicMock, patch import monkey_island.cc.test_common.environment.server_config_mocks as config_mocks from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH -from monkey_island.cc.environment.environment_config import EnvironmentConfig, DEFAULT_SERVER_CONFIG +from monkey_island.cc.environment.environment_config import EnvironmentConfig from monkey_island.cc.environment.user_creds import UserCreds @@ -56,13 +55,6 @@ class TestEnvironmentConfig(TestCase): self.assertDictEqual(config, json.loads(content_from_file)) - def test_default_server_config_file_path(self): - if platform.system() == "Windows": - server_file_path = MONKEY_ISLAND_ABS_PATH + r"\cc\server_config.json" - else: - server_file_path = MONKEY_ISLAND_ABS_PATH + "/cc/server_config.json" - self.assertEqual(DEFAULT_SERVER_CONFIG_PATH, server_file_path) - def test_get_from_dict(self): config_dict = config_mocks.CONFIG_WITH_CREDENTIALS env_conf = EnvironmentConfig.get_from_dict(config_dict) diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index 27d928134..fa7ff524d 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -21,7 +21,7 @@ json_setup_logging(default_path=Path(MONKEY_ISLAND_ABS_PATH, 'cc', 'island_logge logger = logging.getLogger(__name__) import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 -from monkey_island.cc.environment.environment_config import DEFAULT_SERVER_CONFIG_PATH +from monkey_island.cc.consts import DEFAULT_SERVER_CONFIG_PATH from common.version import get_version # noqa: E402 from monkey_island.cc.app import init_app # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index c302f6fb7..b5e9b7dce 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -4,3 +4,6 @@ __author__ = 'itay.mizeretz' MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), 'monkey_island') DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5 + +_SERVER_CONFIG_FILENAME = "server_config.json" +DEFAULT_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', _SERVER_CONFIG_FILENAME) diff --git a/monkey/monkey_island/cc/test_consts.py b/monkey/monkey_island/cc/test_consts.py new file mode 100644 index 000000000..f1bb45805 --- /dev/null +++ b/monkey/monkey_island/cc/test_consts.py @@ -0,0 +1,11 @@ +import platform +import monkey_island.cc.consts as consts + + +def test_default_server_config_file_path(): + if platform.system() == "Windows": + server_file_path = consts.MONKEY_ISLAND_ABS_PATH + r"\cc\server_config.json" + else: + server_file_path = consts.MONKEY_ISLAND_ABS_PATH + "/cc/server_config.json" + + assert consts.DEFAULT_SERVER_CONFIG_PATH == server_file_path From 986219bd8657a9cd77e6419410207abfe93eeb47 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 8 Feb 2021 14:19:15 -0500 Subject: [PATCH 04/52] cc: rework EnvironmentConfig test 1. Rewrote in pytest 2. Removed reduntant tests 3. Added tests for add_user() and get_users() --- .../cc/environment/environment_singleton.py | 2 +- .../cc/environment/test_environment_config.py | 167 +++++++++++------- monkey/monkey_island/cc/main.py | 2 +- monkey/monkey_island/cc/test_consts.py | 2 +- 4 files changed, 102 insertions(+), 71 deletions(-) diff --git a/monkey/monkey_island/cc/environment/environment_singleton.py b/monkey/monkey_island/cc/environment/environment_singleton.py index 6934c5cd2..bbcc75584 100644 --- a/monkey/monkey_island/cc/environment/environment_singleton.py +++ b/monkey/monkey_island/cc/environment/environment_singleton.py @@ -3,7 +3,7 @@ import logging import monkey_island.cc.resources.auth.user_store as user_store from monkey_island.cc.environment import (EnvironmentConfig, aws, password, standard, testing) -from monkey_island.cc.consts import DEFAULT_SERVER_CONFIG_PATH +from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH __author__ = 'itay.mizeretz' diff --git a/monkey/monkey_island/cc/environment/test_environment_config.py b/monkey/monkey_island/cc/environment/test_environment_config.py index d00fd1de1..dea22dd3f 100644 --- a/monkey/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/monkey_island/cc/environment/test_environment_config.py @@ -1,92 +1,123 @@ import json import os from typing import Dict -from unittest import TestCase -from unittest.mock import MagicMock, patch + +import pytest import monkey_island.cc.test_common.environment.server_config_mocks as config_mocks -from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH from monkey_island.cc.environment.environment_config import EnvironmentConfig from monkey_island.cc.environment.user_creds import UserCreds -def get_server_config_file_path_test_version(): - return os.path.join(os.getcwd(), 'test_config.json') -class TestEnvironmentConfig(TestCase): +@pytest.fixture +def config_file(tmpdir): + return os.path.join(tmpdir, "test_config.json") - def test_get_from_json(self): - self._test_get_from_json(config_mocks.CONFIG_WITH_CREDENTIALS) - self._test_get_from_json(config_mocks.CONFIG_NO_CREDENTIALS) - self._test_get_from_json(config_mocks.CONFIG_PARTIAL_CREDENTIALS) - def _test_get_from_json(self, config: Dict): - config_json = json.dumps(config) - env_config_object = EnvironmentConfig.get_from_json(config_json) - self.assertEqual(config['server_config'], env_config_object.server_config) - self.assertEqual(config['deployment'], env_config_object.deployment) - if 'user' in config: - self.assertEqual(config['user'], env_config_object.user_creds.username) - if 'password_hash' in config: - self.assertEqual(config['password_hash'], env_config_object.user_creds.password_hash) - if 'aws' in config: - self.assertEqual(config['aws'], env_config_object.aws) +def test_get_with_credentials(config_file): + test_conf = config_mocks.CONFIG_WITH_CREDENTIALS - def test_save_to_file(self): - self._test_save_to_file(config_mocks.CONFIG_WITH_CREDENTIALS) - self._test_save_to_file(config_mocks.CONFIG_NO_CREDENTIALS) - self._test_save_to_file(config_mocks.CONFIG_PARTIAL_CREDENTIALS) + _write_test_config_to_tmp(config_file, test_conf) + config_dict = EnvironmentConfig.get_from_file(config_file).to_dict() - @patch.object(target=EnvironmentConfig, attribute="get_config_file_path", - new=MagicMock(return_value=get_server_config_file_path_test_version())) - def _test_save_to_file(self, config: Dict): - user_creds = UserCreds.get_from_dict(config) - env_config = EnvironmentConfig(server_config=config['server_config'], - deployment=config['deployment'], - user_creds=user_creds) - env_config.server_config_path = get_server_config_file_path_test_version() + assert len(config_dict.keys()) == 4 + assert config_dict["server_config"] == test_conf["server_config"] + assert config_dict["deployment"] == test_conf["deployment"] + assert config_dict["user"] == test_conf["user"] + assert config_dict["password_hash"] == test_conf["password_hash"] - env_config.save_to_file() - file_path = get_server_config_file_path_test_version() - with open(file_path, 'r') as f: - content_from_file = f.read() - os.remove(file_path) - self.assertDictEqual(config, json.loads(content_from_file)) +def test_get_with_no_credentials(config_file): + test_conf = config_mocks.CONFIG_NO_CREDENTIALS - def test_get_from_dict(self): - config_dict = config_mocks.CONFIG_WITH_CREDENTIALS - env_conf = EnvironmentConfig.get_from_dict(config_dict) - self.assertEqual(env_conf.server_config, config_dict['server_config']) - self.assertEqual(env_conf.deployment, config_dict['deployment']) - self.assertEqual(env_conf.user_creds.username, config_dict['user']) - self.assertEqual(env_conf.aws, None) + _write_test_config_to_tmp(config_file, test_conf) + config_dict = EnvironmentConfig.get_from_file(config_file).to_dict() - config_dict = config_mocks.CONFIG_BOGUS_VALUES - env_conf = EnvironmentConfig.get_from_dict(config_dict) - self.assertEqual(env_conf.server_config, config_dict['server_config']) - self.assertEqual(env_conf.deployment, config_dict['deployment']) - self.assertEqual(env_conf.user_creds.username, config_dict['user']) - self.assertEqual(env_conf.aws, config_dict['aws']) + assert len(config_dict.keys()) == 2 + assert config_dict["server_config"] == test_conf["server_config"] + assert config_dict["deployment"] == test_conf["deployment"] - def test_to_dict(self): - conf_json1 = json.dumps(config_mocks.CONFIG_WITH_CREDENTIALS) - self._test_to_dict(EnvironmentConfig.get_from_json(conf_json1)) - conf_json2 = json.dumps(config_mocks.CONFIG_NO_CREDENTIALS) - self._test_to_dict(EnvironmentConfig.get_from_json(conf_json2)) +def test_get_with_partial_credentials(config_file): + test_conf = config_mocks.CONFIG_PARTIAL_CREDENTIALS - conf_json3 = json.dumps(config_mocks.CONFIG_PARTIAL_CREDENTIALS) - self._test_to_dict(EnvironmentConfig.get_from_json(conf_json3)) + _write_test_config_to_tmp(config_file, test_conf) + config_dict = EnvironmentConfig.get_from_file(config_file).to_dict() - def _test_to_dict(self, env_config_object: EnvironmentConfig): - test_dict = {'server_config': env_config_object.server_config, - 'deployment': env_config_object.deployment} - user_creds = env_config_object.user_creds - if user_creds.username: - test_dict.update({'user': user_creds.username}) - if user_creds.password_hash: - test_dict.update({'password_hash': user_creds.password_hash}) + assert len(config_dict.keys()) == 3 + assert config_dict["server_config"] == test_conf["server_config"] + assert config_dict["deployment"] == test_conf["deployment"] + assert config_dict["user"] == test_conf["user"] - self.assertDictEqual(test_dict, env_config_object.to_dict()) + +def _write_test_config_to_tmp(config_file, config: Dict): + with open(config_file, "wt") as f: + json.dump(config, f) + + +def test_save_to_file(config_file): + server_config = "standard" + deployment = "develop" + user = "test_user" + password_hash = "abcdef" + aws = "test" + + environment_config = EnvironmentConfig( + server_config, deployment, UserCreds(user, password_hash), aws + ) + environment_config.server_config_path = config_file + + environment_config.save_to_file() + with open(config_file, "r") as f: + from_file = json.load(f) + + assert len(from_file.keys()) == 5 + assert from_file["server_config"] == server_config + assert from_file["deployment"] == deployment + assert from_file["user"] == user + assert from_file["password_hash"] == password_hash + assert from_file["aws"] == aws + + +def test_add_user(config_file): + server_config = "standard" + deployment = "develop" + user = "test_user" + password_hash = "abcdef" + + new_user = "new_user" + new_password_hash = "fedcba" + new_user_creds = UserCreds(new_user, new_password_hash) + + environment_config = EnvironmentConfig( + server_config, deployment, UserCreds(user, password_hash) + ) + environment_config.server_config_path = config_file + + environment_config.add_user(new_user_creds) + + with open(config_file, "r") as f: + from_file = json.load(f) + + assert len(from_file.keys()) == 4 + assert from_file["user"] == new_user + assert from_file["password_hash"] == new_password_hash + + +def test_get_users(): + server_config = "standard" + deployment = "develop" + user = "test_user" + password_hash = "abcdef" + + environment_config = EnvironmentConfig( + server_config, deployment, UserCreds(user, password_hash) + ) + + users = environment_config.get_users() + assert len(users) == 1 + assert users[0].id == 1 + assert users[0].username == user + assert users[0].secret == password_hash diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index fa7ff524d..b60a61497 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -21,7 +21,7 @@ json_setup_logging(default_path=Path(MONKEY_ISLAND_ABS_PATH, 'cc', 'island_logge logger = logging.getLogger(__name__) import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 -from monkey_island.cc.consts import DEFAULT_SERVER_CONFIG_PATH +from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH from common.version import get_version # noqa: E402 from monkey_island.cc.app import init_app # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 diff --git a/monkey/monkey_island/cc/test_consts.py b/monkey/monkey_island/cc/test_consts.py index f1bb45805..76a08a258 100644 --- a/monkey/monkey_island/cc/test_consts.py +++ b/monkey/monkey_island/cc/test_consts.py @@ -1,5 +1,5 @@ import platform -import monkey_island.cc.consts as consts +import monkey_island.cc.server_utils.consts as consts def test_default_server_config_file_path(): From 4b5415ac0b946b5b6a9e5beaf398057ab1901381 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 8 Feb 2021 15:00:46 -0500 Subject: [PATCH 05/52] cc: fix server_config_generator behavior --- .../cc/environment/server_config_generator.py | 5 +++-- .../cc/environment/test_environment_config.py | 12 ++++++++++++ monkey/monkey_island/cc/server_config.json.standard | 4 ++++ monkey/monkey_island/cc/server_utils/consts.py | 3 +++ 4 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 monkey/monkey_island/cc/server_config.json.standard diff --git a/monkey/monkey_island/cc/environment/server_config_generator.py b/monkey/monkey_island/cc/environment/server_config_generator.py index d5c645564..d52d5261d 100644 --- a/monkey/monkey_island/cc/environment/server_config_generator.py +++ b/monkey/monkey_island/cc/environment/server_config_generator.py @@ -1,7 +1,8 @@ from pathlib import Path +from monkey_island.cc.consts import DEFAULT_STANDARD_SERVER_CONFIG + def create_default_config_file(path): - default_config_path = f"{path}.default" - default_config = Path(default_config_path).read_text() + default_config = Path(DEFAULT_STANDARD_SERVER_CONFIG).read_text() Path(path).write_text(default_config) diff --git a/monkey/monkey_island/cc/environment/test_environment_config.py b/monkey/monkey_island/cc/environment/test_environment_config.py index dea22dd3f..f038eed74 100644 --- a/monkey/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/monkey_island/cc/environment/test_environment_config.py @@ -121,3 +121,15 @@ def test_get_users(): assert users[0].id == 1 assert users[0].username == user assert users[0].secret == password_hash + +def test_generate_default_file(config_file): + environment_config = EnvironmentConfig.get_from_file(config_file) + + assert os.path.isfile(config_file) + + assert environment_config.server_config == "password" + assert environment_config.deployment == "standard" + assert environment_config.user_creds.username == "" + assert environment_config.user_creds.password_hash == "" + assert environment_config.aws is None + environment_config.server_config_path == config_file diff --git a/monkey/monkey_island/cc/server_config.json.standard b/monkey/monkey_island/cc/server_config.json.standard new file mode 100644 index 000000000..7bdd9a163 --- /dev/null +++ b/monkey/monkey_island/cc/server_config.json.standard @@ -0,0 +1,4 @@ +{ + "server_config": "password", + "deployment": "standard" +} diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index b5e9b7dce..0be7e776d 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -7,3 +7,6 @@ DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5 _SERVER_CONFIG_FILENAME = "server_config.json" DEFAULT_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', _SERVER_CONFIG_FILENAME) + +_STANDARD_SERVER_CONFIG_FILENAME = "server_config.json.standard" +DEFAULT_STANDARD_SERVER_CONFIG = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', _STANDARD_SERVER_CONFIG_FILENAME) From 98b64da8966c0b4138bdc37b87fbaaef409581da Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 9 Feb 2021 12:33:01 -0500 Subject: [PATCH 06/52] cc: simplify constructor/factory interface for EnvironmentConfig The `get_from_json()` and `get_from_dict()` static methods were really just used for testing. The `EnvironmentConfig` class needs to store its file path so it can wite to the file if needed. In practical usage, `EnvironmentConfig` objects are initialized from files, so a simpler interface is for its constructor to take a file path. --- .../cc/environment/environment_config.py | 61 ++++----- .../cc/environment/environment_singleton.py | 2 +- .../cc/environment/server_config_generator.py | 2 +- .../cc/environment/test__init__.py | 66 +++++++--- .../cc/environment/test_environment_config.py | 116 ++++++++---------- .../environment/server_config_mocks.py | 41 ------- .../server_config_no_credentials.json | 4 + .../server_config_partial_credentials.json | 5 + .../server_config_standard_env.json | 4 + ...rver_config_standard_with_credentials.json | 6 + .../server_config_with_credentials.json | 6 + 11 files changed, 147 insertions(+), 166 deletions(-) delete mode 100644 monkey/monkey_island/cc/test_common/environment/server_config_mocks.py create mode 100644 monkey/monkey_island/cc/testing/environment/server_config_no_credentials.json create mode 100644 monkey/monkey_island/cc/testing/environment/server_config_partial_credentials.json create mode 100644 monkey/monkey_island/cc/testing/environment/server_config_standard_env.json create mode 100644 monkey/monkey_island/cc/testing/environment/server_config_standard_with_credentials.json create mode 100644 monkey/monkey_island/cc/testing/environment/server_config_with_credentials.json diff --git a/monkey/monkey_island/cc/environment/environment_config.py b/monkey/monkey_island/cc/environment/environment_config.py index 006f4d233..aec1a777f 100644 --- a/monkey/monkey_island/cc/environment/environment_config.py +++ b/monkey/monkey_island/cc/environment/environment_config.py @@ -6,45 +6,22 @@ from pathlib import Path from typing import Dict, List import monkey_island.cc.environment.server_config_generator as server_config_generator -from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH from monkey_island.cc.environment.user_creds import UserCreds from monkey_island.cc.resources.auth.auth_user import User from monkey_island.cc.resources.auth.user_store import UserStore - class EnvironmentConfig: - def __init__(self, - server_config: str, - deployment: str, - user_creds: UserCreds, - aws=None): - self.server_config_path = None - self.server_config = server_config - self.deployment = deployment - self.user_creds = user_creds - self.aws = aws + def __init__(self, file_path): + self._server_config_path = os.path.expanduser(file_path) + self.server_config = None + self.deployment = None + self.user_creds = None + self.aws = None - @staticmethod - def get_from_json(config_json: str) -> EnvironmentConfig: - data = json.loads(config_json) - return EnvironmentConfig.get_from_dict(data) + self._load_from_file(self._server_config_path) - @staticmethod - def get_from_dict(dict_data: Dict) -> EnvironmentConfig: - user_creds = UserCreds.get_from_dict(dict_data) - aws = dict_data['aws'] if 'aws' in dict_data else None - return EnvironmentConfig(server_config=dict_data['server_config'], - deployment=dict_data['deployment'], - user_creds=user_creds, - aws=aws) - - def save_to_file(self): - with open(self.server_config_path, 'w') as f: - f.write(json.dumps(self.to_dict(), indent=2)) - - @staticmethod - def get_from_file(file_path=DEFAULT_SERVER_CONFIG_PATH) -> EnvironmentConfig: + def _load_from_file(self, file_path): file_path = os.path.expanduser(file_path) if not Path(file_path).is_file(): @@ -52,12 +29,24 @@ class EnvironmentConfig: with open(file_path, 'r') as f: config_content = f.read() - environment_config = EnvironmentConfig.get_from_json(config_content) - # TODO: Populating this property is not ideal. Revisit this when you - # make the logger config file configurable at runtime. - environment_config.server_config_path = file_path + self._load_from_json(config_content) - return environment_config + def _load_from_json(self, config_json: str) -> EnvironmentConfig: + data = json.loads(config_json) + self._load_from_dict(data) + + def _load_from_dict(self, dict_data: Dict): + user_creds = UserCreds.get_from_dict(dict_data) + aws = dict_data['aws'] if 'aws' in dict_data else None + + self.server_config = dict_data['server_config'] + self.deployment = dict_data['deployment'] + self.user_creds = user_creds + self.aws = aws + + def save_to_file(self): + with open(self._server_config_path, 'w') as f: + f.write(json.dumps(self.to_dict(), indent=2)) def to_dict(self) -> Dict: config_dict = {'server_config': self.server_config, diff --git a/monkey/monkey_island/cc/environment/environment_singleton.py b/monkey/monkey_island/cc/environment/environment_singleton.py index bbcc75584..1d037051d 100644 --- a/monkey/monkey_island/cc/environment/environment_singleton.py +++ b/monkey/monkey_island/cc/environment/environment_singleton.py @@ -39,7 +39,7 @@ def set_to_standard(): def initialize_from_file(file_path): try: - config = EnvironmentConfig.get_from_file(file_path) + config = EnvironmentConfig(file_path) __env_type = config.server_config set_env(__env_type, config) diff --git a/monkey/monkey_island/cc/environment/server_config_generator.py b/monkey/monkey_island/cc/environment/server_config_generator.py index d52d5261d..3c0fb083e 100644 --- a/monkey/monkey_island/cc/environment/server_config_generator.py +++ b/monkey/monkey_island/cc/environment/server_config_generator.py @@ -1,6 +1,6 @@ from pathlib import Path -from monkey_island.cc.consts import DEFAULT_STANDARD_SERVER_CONFIG +from monkey_island.cc.server_utils.consts import DEFAULT_STANDARD_SERVER_CONFIG def create_default_config_file(path): diff --git a/monkey/monkey_island/cc/environment/test__init__.py b/monkey/monkey_island/cc/environment/test__init__.py index c55e1b65b..3e805b712 100644 --- a/monkey/monkey_island/cc/environment/test__init__.py +++ b/monkey/monkey_island/cc/environment/test__init__.py @@ -1,15 +1,43 @@ import json import os +import tempfile from typing import Dict from unittest import TestCase from unittest.mock import MagicMock, patch -import monkey_island.cc.test_common.environment.server_config_mocks as config_mocks +from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH from common.utils.exceptions import (AlreadyRegisteredError, CredentialsNotRequiredError, InvalidRegistrationCredentialsError, RegistrationNotNeededError) from monkey_island.cc.environment import Environment, EnvironmentConfig, UserCreds +TEST_RESOURCES_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing", "environment") + +WITH_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_with_credentials.json") +NO_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_no_credentials.json") +PARTIAL_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_partial_credentials.json") +STANDARD_WITH_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, + "server_config_standard_with_credentials.json") +STANDARD_ENV = os.path.join(TEST_RESOURCES_DIR, + "server_config_standard_env.json") + + +def get_tmp_file(): + with tempfile.NamedTemporaryFile(delete=False) as f: + return f.name + + +class StubEnvironmentConfig(EnvironmentConfig): + def __init__(self, server_config, deployment, user_creds): + self.server_config = server_config + self.deployment = deployment + self.user_creds = user_creds + self.server_config_path = get_tmp_file() + + def __del__(self): + os.remove(self.server_config_path) + + def get_server_config_file_path_test_version(): return os.path.join(os.getcwd(), 'test_config.json') @@ -18,7 +46,7 @@ class TestEnvironment(TestCase): class EnvironmentCredentialsNotRequired(Environment): def __init__(self): - config = EnvironmentConfig('test', 'test', UserCreds()) + config = StubEnvironmentConfig('test', 'test', UserCreds()) super().__init__(config) _credentials_required = False @@ -28,7 +56,7 @@ class TestEnvironment(TestCase): class EnvironmentCredentialsRequired(Environment): def __init__(self): - config = EnvironmentConfig('test', 'test', UserCreds()) + config = StubEnvironmentConfig('test', 'test', UserCreds()) super().__init__(config) _credentials_required = True @@ -38,7 +66,7 @@ class TestEnvironment(TestCase): class EnvironmentAlreadyRegistered(Environment): def __init__(self): - config = EnvironmentConfig('test', 'test', UserCreds('test_user', 'test_secret')) + config = StubEnvironmentConfig('test', 'test', UserCreds('test_user', 'test_secret')) super().__init__(config) _credentials_required = True @@ -75,35 +103,35 @@ class TestEnvironment(TestCase): def test_needs_registration(self): env = TestEnvironment.EnvironmentCredentialsRequired() - self._test_bool_env_method("needs_registration", env, config_mocks.CONFIG_WITH_CREDENTIALS, False) - self._test_bool_env_method("needs_registration", env, config_mocks.CONFIG_NO_CREDENTIALS, True) - self._test_bool_env_method("needs_registration", env, config_mocks.CONFIG_PARTIAL_CREDENTIALS, True) + self._test_bool_env_method("needs_registration", env, WITH_CREDENTIALS, False) + self._test_bool_env_method("needs_registration", env, NO_CREDENTIALS, True) + self._test_bool_env_method("needs_registration", env, PARTIAL_CREDENTIALS, True) env = TestEnvironment.EnvironmentCredentialsNotRequired() - self._test_bool_env_method("needs_registration", env, config_mocks.CONFIG_STANDARD_ENV, False) - self._test_bool_env_method("needs_registration", env, config_mocks.CONFIG_STANDARD_WITH_CREDENTIALS, False) + self._test_bool_env_method("needs_registration", env, STANDARD_ENV, False) + self._test_bool_env_method("needs_registration", env, STANDARD_WITH_CREDENTIALS, False) def test_is_registered(self): env = TestEnvironment.EnvironmentCredentialsRequired() - self._test_bool_env_method("_is_registered", env, config_mocks.CONFIG_WITH_CREDENTIALS, True) - self._test_bool_env_method("_is_registered", env, config_mocks.CONFIG_NO_CREDENTIALS, False) - self._test_bool_env_method("_is_registered", env, config_mocks.CONFIG_PARTIAL_CREDENTIALS, False) + self._test_bool_env_method("_is_registered", env, WITH_CREDENTIALS, True) + self._test_bool_env_method("_is_registered", env, NO_CREDENTIALS, False) + self._test_bool_env_method("_is_registered", env, PARTIAL_CREDENTIALS, False) env = TestEnvironment.EnvironmentCredentialsNotRequired() - self._test_bool_env_method("_is_registered", env, config_mocks.CONFIG_STANDARD_ENV, False) - self._test_bool_env_method("_is_registered", env, config_mocks.CONFIG_STANDARD_WITH_CREDENTIALS, False) + self._test_bool_env_method("_is_registered", env, STANDARD_ENV, False) + self._test_bool_env_method("_is_registered", env, STANDARD_WITH_CREDENTIALS, False) def test_is_credentials_set_up(self): env = TestEnvironment.EnvironmentCredentialsRequired() - self._test_bool_env_method("_is_credentials_set_up", env, config_mocks.CONFIG_NO_CREDENTIALS, False) - self._test_bool_env_method("_is_credentials_set_up", env, config_mocks.CONFIG_WITH_CREDENTIALS, True) - self._test_bool_env_method("_is_credentials_set_up", env, config_mocks.CONFIG_PARTIAL_CREDENTIALS, False) + self._test_bool_env_method("_is_credentials_set_up", env, NO_CREDENTIALS, False) + self._test_bool_env_method("_is_credentials_set_up", env, WITH_CREDENTIALS, True) + self._test_bool_env_method("_is_credentials_set_up", env, PARTIAL_CREDENTIALS, False) env = TestEnvironment.EnvironmentCredentialsNotRequired() - self._test_bool_env_method("_is_credentials_set_up", env, config_mocks.CONFIG_STANDARD_ENV, False) + self._test_bool_env_method("_is_credentials_set_up", env, STANDARD_ENV, False) def _test_bool_env_method(self, method_name: str, env: Environment, config: Dict, expected_result: bool): - env._config = EnvironmentConfig.get_from_json(json.dumps(config)) + env._config = EnvironmentConfig(config) method = getattr(env, method_name) if expected_result: self.assertTrue(method()) diff --git a/monkey/monkey_island/cc/environment/test_environment_config.py b/monkey/monkey_island/cc/environment/test_environment_config.py index f038eed74..e508112d4 100644 --- a/monkey/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/monkey_island/cc/environment/test_environment_config.py @@ -1,14 +1,28 @@ import json import os -from typing import Dict +import shutil import pytest -import monkey_island.cc.test_common.environment.server_config_mocks as config_mocks +from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH from monkey_island.cc.environment.environment_config import EnvironmentConfig from monkey_island.cc.environment.user_creds import UserCreds +TEST_RESOURCES_DIR = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", "testing", "environment" +) + +WITH_CREDENTIALS = os.path.join( + TEST_RESOURCES_DIR, "server_config_with_credentials.json" +) +NO_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_no_credentials.json") +PARTIAL_CREDENTIALS = os.path.join( + TEST_RESOURCES_DIR, "server_config_partial_credentials.json" +) +STANDARD_WITH_CREDENTIALS = os.path.join( + TEST_RESOURCES_DIR, "server_config_standard_with_credentials.json" +) @pytest.fixture @@ -16,86 +30,59 @@ def config_file(tmpdir): return os.path.join(tmpdir, "test_config.json") -def test_get_with_credentials(config_file): - test_conf = config_mocks.CONFIG_WITH_CREDENTIALS - - _write_test_config_to_tmp(config_file, test_conf) - config_dict = EnvironmentConfig.get_from_file(config_file).to_dict() +def test_get_with_credentials(): + config_dict = EnvironmentConfig(WITH_CREDENTIALS).to_dict() assert len(config_dict.keys()) == 4 - assert config_dict["server_config"] == test_conf["server_config"] - assert config_dict["deployment"] == test_conf["deployment"] - assert config_dict["user"] == test_conf["user"] - assert config_dict["password_hash"] == test_conf["password_hash"] + assert config_dict["server_config"] == "password" + assert config_dict["deployment"] == "develop" + assert config_dict["user"] == "test" + assert config_dict["password_hash"] == "abcdef" -def test_get_with_no_credentials(config_file): - test_conf = config_mocks.CONFIG_NO_CREDENTIALS - - _write_test_config_to_tmp(config_file, test_conf) - config_dict = EnvironmentConfig.get_from_file(config_file).to_dict() +def test_get_with_no_credentials(): + config_dict = EnvironmentConfig(NO_CREDENTIALS).to_dict() assert len(config_dict.keys()) == 2 - assert config_dict["server_config"] == test_conf["server_config"] - assert config_dict["deployment"] == test_conf["deployment"] + assert config_dict["server_config"] == "password" + assert config_dict["deployment"] == "develop" -def test_get_with_partial_credentials(config_file): - test_conf = config_mocks.CONFIG_PARTIAL_CREDENTIALS - - _write_test_config_to_tmp(config_file, test_conf) - config_dict = EnvironmentConfig.get_from_file(config_file).to_dict() +def test_get_with_partial_credentials(): + config_dict = EnvironmentConfig(PARTIAL_CREDENTIALS).to_dict() assert len(config_dict.keys()) == 3 - assert config_dict["server_config"] == test_conf["server_config"] - assert config_dict["deployment"] == test_conf["deployment"] - assert config_dict["user"] == test_conf["user"] - - -def _write_test_config_to_tmp(config_file, config: Dict): - with open(config_file, "wt") as f: - json.dump(config, f) + assert config_dict["server_config"] == "password" + assert config_dict["deployment"] == "develop" + assert config_dict["user"] == "test" def test_save_to_file(config_file): - server_config = "standard" - deployment = "develop" - user = "test_user" - password_hash = "abcdef" - aws = "test" - - environment_config = EnvironmentConfig( - server_config, deployment, UserCreds(user, password_hash), aws - ) - environment_config.server_config_path = config_file + shutil.copyfile(STANDARD_WITH_CREDENTIALS, config_file) + environment_config = EnvironmentConfig(config_file) + environment_config.aws = "test_aws" environment_config.save_to_file() + with open(config_file, "r") as f: from_file = json.load(f) assert len(from_file.keys()) == 5 - assert from_file["server_config"] == server_config - assert from_file["deployment"] == deployment - assert from_file["user"] == user - assert from_file["password_hash"] == password_hash - assert from_file["aws"] == aws + assert from_file["server_config"] == "standard" + assert from_file["deployment"] == "develop" + assert from_file["user"] == "test" + assert from_file["password_hash"] == "abcdef" + assert from_file["aws"] == "test_aws" def test_add_user(config_file): - server_config = "standard" - deployment = "develop" - user = "test_user" - password_hash = "abcdef" - new_user = "new_user" new_password_hash = "fedcba" new_user_creds = UserCreds(new_user, new_password_hash) - environment_config = EnvironmentConfig( - server_config, deployment, UserCreds(user, password_hash) - ) - environment_config.server_config_path = config_file + shutil.copyfile(STANDARD_WITH_CREDENTIALS, config_file) + environment_config = EnvironmentConfig(config_file) environment_config.add_user(new_user_creds) with open(config_file, "r") as f: @@ -107,23 +94,17 @@ def test_add_user(config_file): def test_get_users(): - server_config = "standard" - deployment = "develop" - user = "test_user" - password_hash = "abcdef" - - environment_config = EnvironmentConfig( - server_config, deployment, UserCreds(user, password_hash) - ) - + environment_config = EnvironmentConfig(STANDARD_WITH_CREDENTIALS) users = environment_config.get_users() + assert len(users) == 1 assert users[0].id == 1 - assert users[0].username == user - assert users[0].secret == password_hash + assert users[0].username == "test" + assert users[0].secret == "abcdef" + def test_generate_default_file(config_file): - environment_config = EnvironmentConfig.get_from_file(config_file) + environment_config = EnvironmentConfig(config_file) assert os.path.isfile(config_file) @@ -132,4 +113,3 @@ def test_generate_default_file(config_file): assert environment_config.user_creds.username == "" assert environment_config.user_creds.password_hash == "" assert environment_config.aws is None - environment_config.server_config_path == config_file diff --git a/monkey/monkey_island/cc/test_common/environment/server_config_mocks.py b/monkey/monkey_island/cc/test_common/environment/server_config_mocks.py deleted file mode 100644 index ddbff3118..000000000 --- a/monkey/monkey_island/cc/test_common/environment/server_config_mocks.py +++ /dev/null @@ -1,41 +0,0 @@ -# Username:test Password:test -CONFIG_WITH_CREDENTIALS = { - "server_config": "password", - "deployment": "develop", - "user": "test", - "password_hash": "9ece086e9bac491fac5c1d1046ca11d737b92a2b2ebd93f005d7b710110c0a678288166e7fbe796883a" - "4f2e9b3ca9f484f521d0ce464345cc1aec96779149c14" -} - -CONFIG_NO_CREDENTIALS = { - "server_config": "password", - "deployment": "develop" -} - -CONFIG_PARTIAL_CREDENTIALS = { - "server_config": "password", - "deployment": "develop", - "user": "test" -} - -CONFIG_BOGUS_VALUES = { - "server_config": "password", - "deployment": "develop", - "user": "test", - "aws": "test", - "test": "test", - "test2": "test2" -} - -CONFIG_STANDARD_ENV = { - "server_config": "standard", - "deployment": "develop" -} - -CONFIG_STANDARD_WITH_CREDENTIALS = { - "server_config": "standard", - "deployment": "develop", - "user": "test", - "password_hash": "9ece086e9bac491fac5c1d1046ca11d737b92a2b2ebd93f005d7b710110c0a678288166e7fbe796883a" - "4f2e9b3ca9f484f521d0ce464345cc1aec96779149c14" -} diff --git a/monkey/monkey_island/cc/testing/environment/server_config_no_credentials.json b/monkey/monkey_island/cc/testing/environment/server_config_no_credentials.json new file mode 100644 index 000000000..ecc4c1708 --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_no_credentials.json @@ -0,0 +1,4 @@ +{ + "server_config": "password", + "deployment": "develop" +} diff --git a/monkey/monkey_island/cc/testing/environment/server_config_partial_credentials.json b/monkey/monkey_island/cc/testing/environment/server_config_partial_credentials.json new file mode 100644 index 000000000..a9e283924 --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_partial_credentials.json @@ -0,0 +1,5 @@ +{ + "server_config": "password", + "deployment": "develop", + "user": "test" +} diff --git a/monkey/monkey_island/cc/testing/environment/server_config_standard_env.json b/monkey/monkey_island/cc/testing/environment/server_config_standard_env.json new file mode 100644 index 000000000..420f1b303 --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_standard_env.json @@ -0,0 +1,4 @@ +{ + "server_config": "standard", + "deployment": "develop" +} diff --git a/monkey/monkey_island/cc/testing/environment/server_config_standard_with_credentials.json b/monkey/monkey_island/cc/testing/environment/server_config_standard_with_credentials.json new file mode 100644 index 000000000..4bff379e8 --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_standard_with_credentials.json @@ -0,0 +1,6 @@ +{ + "server_config": "standard", + "deployment": "develop", + "user": "test", + "password_hash": "abcdef" +} diff --git a/monkey/monkey_island/cc/testing/environment/server_config_with_credentials.json b/monkey/monkey_island/cc/testing/environment/server_config_with_credentials.json new file mode 100644 index 000000000..54c0fa787 --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_with_credentials.json @@ -0,0 +1,6 @@ +{ + "server_config": "password", + "deployment": "develop", + "user": "test", + "password_hash": "abcdef" +} From a057dec1fe911973404613b6804321d7ca6eb5fb Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 10 Feb 2021 08:09:43 -0500 Subject: [PATCH 07/52] cc: use DEFAULT_SERVER_CONFIG_PATH in set_server_config --- monkey/monkey_island/cc/environment/set_server_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/environment/set_server_config.py b/monkey/monkey_island/cc/environment/set_server_config.py index 168fe13cd..2bb4336ae 100644 --- a/monkey/monkey_island/cc/environment/set_server_config.py +++ b/monkey/monkey_island/cc/environment/set_server_config.py @@ -14,7 +14,7 @@ def add_monkey_dir_to_sys_path(): add_monkey_dir_to_sys_path() -from monkey_island.cc.environment.environment_config import EnvironmentConfig # noqa: E402 isort:skip +from monkey_island.cc.consts import DEFAULT_SERVER_CONFIG_PATH # noqa: E402 isort:skip SERVER_CONFIG = "server_config" BACKUP_CONFIG_FILENAME = "./server_config.backup" @@ -26,7 +26,7 @@ logger.setLevel(logging.DEBUG) def main(): args = parse_args() - file_path = EnvironmentConfig.get_config_file_path() + file_path = DEFAULT_SERVER_CONFIG_PATH if args.server_config == "restore": restore_previous_config(file_path) From dd9e4bdefa892d1d0d71f2ea34f9e6d924ce4d52 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 10 Feb 2021 08:30:22 -0500 Subject: [PATCH 08/52] cc: address flake8 issues --- monkey/monkey_island.py | 4 ++-- .../cc/environment/environment_singleton.py | 2 ++ monkey/monkey_island/cc/environment/test__init__.py | 13 +++++++------ monkey/monkey_island/cc/main.py | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 2e410cd9f..f27efa01a 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -2,8 +2,8 @@ from gevent import monkey as gevent_monkey gevent_monkey.patch_all() -from monkey_island.cc.main import main -from monkey_island.cc.environment.environment_config import DEFAULT_SERVER_CONFIG_PATH +from monkey_island.cc.main import main # noqa: E402 +from monkey_island.cc.environment.environment_config import DEFAULT_SERVER_CONFIG_PATH # noqa: E402 def parse_cli_args(): diff --git a/monkey/monkey_island/cc/environment/environment_singleton.py b/monkey/monkey_island/cc/environment/environment_singleton.py index 1d037051d..accb104ce 100644 --- a/monkey/monkey_island/cc/environment/environment_singleton.py +++ b/monkey/monkey_island/cc/environment/environment_singleton.py @@ -37,6 +37,7 @@ def set_to_standard(): env.save_config() user_store.UserStore.set_users(env.get_auth_users()) + def initialize_from_file(file_path): try: config = EnvironmentConfig(file_path) @@ -49,4 +50,5 @@ def initialize_from_file(file_path): logger.error('Failed initializing environment', exc_info=True) raise + initialize_from_file(DEFAULT_SERVER_CONFIG_PATH) diff --git a/monkey/monkey_island/cc/environment/test__init__.py b/monkey/monkey_island/cc/environment/test__init__.py index 3e805b712..fde0a8b27 100644 --- a/monkey/monkey_island/cc/environment/test__init__.py +++ b/monkey/monkey_island/cc/environment/test__init__.py @@ -1,4 +1,3 @@ -import json import os import tempfile from typing import Dict @@ -6,10 +5,12 @@ from unittest import TestCase from unittest.mock import MagicMock, patch from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH -from common.utils.exceptions import (AlreadyRegisteredError, CredentialsNotRequiredError, - InvalidRegistrationCredentialsError, RegistrationNotNeededError) -from monkey_island.cc.environment import Environment, EnvironmentConfig, UserCreds - +from common.utils.exceptions import (AlreadyRegisteredError, + CredentialsNotRequiredError, + InvalidRegistrationCredentialsError, + RegistrationNotNeededError) +from monkey_island.cc.environment import (Environment, EnvironmentConfig, + UserCreds) TEST_RESOURCES_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing", "environment") @@ -19,7 +20,7 @@ PARTIAL_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_partial_cr STANDARD_WITH_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_standard_with_credentials.json") STANDARD_ENV = os.path.join(TEST_RESOURCES_DIR, - "server_config_standard_env.json") + "server_config_standard_env.json") def get_tmp_file(): diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index b60a61497..3da752ec9 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -21,7 +21,7 @@ json_setup_logging(default_path=Path(MONKEY_ISLAND_ABS_PATH, 'cc', 'island_logge logger = logging.getLogger(__name__) import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 -from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH +from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH # noqa: E402 from common.version import get_version # noqa: E402 from monkey_island.cc.app import init_app # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 From ea14bcc2f65fd50aa202eec0e98b7b83241d883f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 10 Feb 2021 08:33:19 -0500 Subject: [PATCH 09/52] cc: rename DEFAULT_STANDARD_SERVER_CONFIG -> DEFAULT_STANDARD_SERVER_CONFIG_PATH --- .../monkey_island/cc/environment/server_config_generator.py | 4 ++-- monkey/monkey_island/cc/server_utils/consts.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/environment/server_config_generator.py b/monkey/monkey_island/cc/environment/server_config_generator.py index 3c0fb083e..c91922605 100644 --- a/monkey/monkey_island/cc/environment/server_config_generator.py +++ b/monkey/monkey_island/cc/environment/server_config_generator.py @@ -1,8 +1,8 @@ from pathlib import Path -from monkey_island.cc.server_utils.consts import DEFAULT_STANDARD_SERVER_CONFIG +from monkey_island.cc.server_utils.consts import DEFAULT_STANDARD_SERVER_CONFIG_PATH def create_default_config_file(path): - default_config = Path(DEFAULT_STANDARD_SERVER_CONFIG).read_text() + default_config = Path(DEFAULT_STANDARD_SERVER_CONFIG_PATH).read_text() Path(path).write_text(default_config) diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 0be7e776d..904526618 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -9,4 +9,4 @@ _SERVER_CONFIG_FILENAME = "server_config.json" DEFAULT_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', _SERVER_CONFIG_FILENAME) _STANDARD_SERVER_CONFIG_FILENAME = "server_config.json.standard" -DEFAULT_STANDARD_SERVER_CONFIG = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', _STANDARD_SERVER_CONFIG_FILENAME) +DEFAULT_STANDARD_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', _STANDARD_SERVER_CONFIG_FILENAME) From fc2f8eca45d975ed44ba8ef74287172198c1424f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 10 Feb 2021 08:40:25 -0500 Subject: [PATCH 10/52] cc: remove unnecessary private constants in consts.py --- monkey/monkey_island/cc/server_utils/consts.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 904526618..cb91b6578 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -1,12 +1,14 @@ import os -__author__ = 'itay.mizeretz' +__author__ = "itay.mizeretz" -MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), 'monkey_island') +MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island") DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5 -_SERVER_CONFIG_FILENAME = "server_config.json" -DEFAULT_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', _SERVER_CONFIG_FILENAME) +DEFAULT_SERVER_CONFIG_PATH = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json" +) -_STANDARD_SERVER_CONFIG_FILENAME = "server_config.json.standard" -DEFAULT_STANDARD_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', _STANDARD_SERVER_CONFIG_FILENAME) +DEFAULT_STANDARD_SERVER_CONFIG_PATH = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json.standard" +) From fef44bcd05af8cd284f9882cd383d345bf6f487a Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 17 Feb 2021 09:55:06 -0500 Subject: [PATCH 11/52] cc: deploy "develop" environment by default --- .travis.yml | 2 +- .../monkey_island/cc/environment/server_config_generator.py | 4 ++-- .../monkey_island/cc/environment/test_environment_config.py | 2 +- ...{server_config.json.default => server_config.json.develop} | 0 monkey/monkey_island/cc/server_config.json.standard | 4 ---- monkey/monkey_island/cc/server_utils/consts.py | 4 ++-- 6 files changed, 6 insertions(+), 10 deletions(-) rename monkey/monkey_island/cc/{server_config.json.default => server_config.json.develop} (100%) delete mode 100644 monkey/monkey_island/cc/server_config.json.standard diff --git a/.travis.yml b/.travis.yml index 8ac8db204..509da86ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ os: linux before_install: # Init server_config.json to default -- cp monkey/monkey_island/cc/server_config.json.default monkey/monkey_island/cc/server_config.json +- cp monkey/monkey_island/cc/server_config.json.develop monkey/monkey_island/cc/server_config.json install: # Python diff --git a/monkey/monkey_island/cc/environment/server_config_generator.py b/monkey/monkey_island/cc/environment/server_config_generator.py index c91922605..211b745c5 100644 --- a/monkey/monkey_island/cc/environment/server_config_generator.py +++ b/monkey/monkey_island/cc/environment/server_config_generator.py @@ -1,8 +1,8 @@ from pathlib import Path -from monkey_island.cc.server_utils.consts import DEFAULT_STANDARD_SERVER_CONFIG_PATH +from monkey_island.cc.server_utils.consts import DEFAULT_DEVELOP_SERVER_CONFIG_PATH def create_default_config_file(path): - default_config = Path(DEFAULT_STANDARD_SERVER_CONFIG_PATH).read_text() + default_config = Path(DEFAULT_DEVELOP_SERVER_CONFIG_PATH).read_text() Path(path).write_text(default_config) diff --git a/monkey/monkey_island/cc/environment/test_environment_config.py b/monkey/monkey_island/cc/environment/test_environment_config.py index e508112d4..b06e26301 100644 --- a/monkey/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/monkey_island/cc/environment/test_environment_config.py @@ -109,7 +109,7 @@ def test_generate_default_file(config_file): assert os.path.isfile(config_file) assert environment_config.server_config == "password" - assert environment_config.deployment == "standard" + assert environment_config.deployment == "develop" assert environment_config.user_creds.username == "" assert environment_config.user_creds.password_hash == "" assert environment_config.aws is None diff --git a/monkey/monkey_island/cc/server_config.json.default b/monkey/monkey_island/cc/server_config.json.develop similarity index 100% rename from monkey/monkey_island/cc/server_config.json.default rename to monkey/monkey_island/cc/server_config.json.develop diff --git a/monkey/monkey_island/cc/server_config.json.standard b/monkey/monkey_island/cc/server_config.json.standard deleted file mode 100644 index 7bdd9a163..000000000 --- a/monkey/monkey_island/cc/server_config.json.standard +++ /dev/null @@ -1,4 +0,0 @@ -{ - "server_config": "password", - "deployment": "standard" -} diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index cb91b6578..44bcb5eba 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -9,6 +9,6 @@ DEFAULT_SERVER_CONFIG_PATH = os.path.join( MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json" ) -DEFAULT_STANDARD_SERVER_CONFIG_PATH = os.path.join( - MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json.standard" +DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json.develop" ) From 4cb28db3bc0c4a7a28fccffb0aa08d22bf8354a1 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 10 Feb 2021 07:29:28 -0500 Subject: [PATCH 12/52] cc: reformat island_logger.py for readability 1. Adjusted some spacing and indentation 2. Reformatted with Black --- .../monkey_island/cc/server_utils/island_logger.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/server_utils/island_logger.py b/monkey/monkey_island/cc/server_utils/island_logger.py index 2b4843876..826ddca78 100644 --- a/monkey/monkey_island/cc/server_utils/island_logger.py +++ b/monkey/monkey_island/cc/server_utils/island_logger.py @@ -2,10 +2,12 @@ import json import logging.config import os -__author__ = 'Maor.Rayzin' +__author__ = "Maor.Rayzin" -def json_setup_logging(default_path='logging.json', default_level=logging.INFO, env_key='LOG_CFG'): +def json_setup_logging( + default_path="logging.json", default_level=logging.INFO, env_key="LOG_CFG" +): """ Setup the logging configuration :param default_path: the default log configuration file path @@ -15,11 +17,13 @@ def json_setup_logging(default_path='logging.json', default_level=logging.INFO, """ path = default_path value = os.getenv(env_key, None) + if value: path = value + if os.path.exists(path): - with open(path, 'rt') as f: + with open(path, "rt") as f: config = json.load(f) - logging.config.dictConfig(config) + logging.config.dictConfig(config) else: logging.basicConfig(level=default_level) From e8bb2e6be2e4f62a3c383df1a5b776a8a39ec4cd Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 10 Feb 2021 12:39:11 -0500 Subject: [PATCH 13/52] cc: allow logger config to be specified at runtime --- monkey/monkey_island.py | 28 +++++++++++++++---- monkey/monkey_island/cc/main.py | 4 --- .../monkey_island/cc/server_utils/consts.py | 4 +++ .../cc/server_utils/island_logger.py | 8 ++++-- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index f27efa01a..224b4788f 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -2,25 +2,43 @@ from gevent import monkey as gevent_monkey gevent_monkey.patch_all() +import json # noqa: E402 + from monkey_island.cc.main import main # noqa: E402 -from monkey_island.cc.environment.environment_config import DEFAULT_SERVER_CONFIG_PATH # noqa: E402 +from monkey_island.cc.consts import DEFAULT_SERVER_CONFIG_PATH, DEFAULT_LOGGING_CONFIG_PATH # noqa: E402 +from monkey_island.cc.island_logger import json_setup_logging # noqa: E402 def parse_cli_args(): import argparse + parser = argparse.ArgumentParser(description="Infection Monkey Island CnC Server. See https://infectionmonkey.com", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-s", "--setup-only", action="store_true", help="Pass this flag to cause the Island to setup and exit without actually starting. " "This is useful for preparing Island to boot faster later-on, so for " "compiling/packaging Islands.") - parser.add_argument("-c", "--config", action="store", + parser.add_argument("--server-config", action="store", help="The path to the server configuration file.", default=DEFAULT_SERVER_CONFIG_PATH) + parser.add_argument("--logging-config", action="store", + help="The path to the logging configuration file.", + default=DEFAULT_LOGGING_CONFIG_PATH) args = parser.parse_args() - return (args.setup_only, args.config) + + return (args.setup_only, args.server_config, args.logging_config) if "__main__" == __name__: - (is_setup_only, config) = parse_cli_args() - main(is_setup_only, config) + (is_setup_only, server_config, logging_config) = parse_cli_args() + + # This is here in order to catch EVERYTHING, some functions are being called on + # imports, so the log init needs to be first. + try: + json_setup_logging(logging_config) + except(json.JSONDecodeError) as ex: + print(f"Error loading logging config: {ex}") + exit(1) + + from monkey_island.cc.main import main + main(is_setup_only, server_config) diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index 3da752ec9..2018da89c 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -13,11 +13,7 @@ if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path: sys.path.insert(0, MONKEY_ISLAND_DIR_BASE_PATH) from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH # noqa: E402 -from monkey_island.cc.server_utils.island_logger import json_setup_logging # noqa: E402 -# This is here in order to catch EVERYTHING, some functions are being called on imports the log init needs to be on top. -json_setup_logging(default_path=Path(MONKEY_ISLAND_ABS_PATH, 'cc', 'island_logger_default_config.json'), - default_level=logging.DEBUG) logger = logging.getLogger(__name__) import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 44bcb5eba..4c793b587 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -12,3 +12,7 @@ DEFAULT_SERVER_CONFIG_PATH = os.path.join( DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json.develop" ) + +DEFAULT_LOGGING_CONFIG_PATH = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", "island_logger_default_config.json" +) diff --git a/monkey/monkey_island/cc/server_utils/island_logger.py b/monkey/monkey_island/cc/server_utils/island_logger.py index 826ddca78..f8eca8cbb 100644 --- a/monkey/monkey_island/cc/server_utils/island_logger.py +++ b/monkey/monkey_island/cc/server_utils/island_logger.py @@ -2,11 +2,15 @@ import json import logging.config import os +from monkey_island.cc.consts import DEFAULT_LOGGING_CONFIG_PATH + __author__ = "Maor.Rayzin" def json_setup_logging( - default_path="logging.json", default_level=logging.INFO, env_key="LOG_CFG" + default_path=DEFAULT_LOGGING_CONFIG_PATH, + default_level=logging.INFO, + env_key="LOG_CFG", ): """ Setup the logging configuration @@ -15,7 +19,7 @@ def json_setup_logging( :param env_key: SYS ENV key to use for external configuration file path :return: """ - path = default_path + path = os.path.expanduser(default_path) value = os.getenv(env_key, None) if value: From 8b3703816d191c9e77060c2733149df7e111165e Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 10 Feb 2021 12:40:18 -0500 Subject: [PATCH 14/52] run black to format monkey_island.py --- monkey/monkey_island.py | 44 ++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 224b4788f..ea8c96efd 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -5,25 +5,40 @@ gevent_monkey.patch_all() import json # noqa: E402 from monkey_island.cc.main import main # noqa: E402 -from monkey_island.cc.consts import DEFAULT_SERVER_CONFIG_PATH, DEFAULT_LOGGING_CONFIG_PATH # noqa: E402 +from monkey_island.cc.consts import ( + DEFAULT_SERVER_CONFIG_PATH, + DEFAULT_LOGGING_CONFIG_PATH, +) # noqa: E402 from monkey_island.cc.island_logger import json_setup_logging # noqa: E402 def parse_cli_args(): import argparse - parser = argparse.ArgumentParser(description="Infection Monkey Island CnC Server. See https://infectionmonkey.com", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("-s", "--setup-only", action="store_true", - help="Pass this flag to cause the Island to setup and exit without actually starting. " - "This is useful for preparing Island to boot faster later-on, so for " - "compiling/packaging Islands.") - parser.add_argument("--server-config", action="store", - help="The path to the server configuration file.", - default=DEFAULT_SERVER_CONFIG_PATH) - parser.add_argument("--logging-config", action="store", - help="The path to the logging configuration file.", - default=DEFAULT_LOGGING_CONFIG_PATH) + parser = argparse.ArgumentParser( + description="Infection Monkey Island CnC Server. See https://infectionmonkey.com", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "-s", + "--setup-only", + action="store_true", + help="Pass this flag to cause the Island to setup and exit without actually starting. " + "This is useful for preparing Island to boot faster later-on, so for " + "compiling/packaging Islands.", + ) + parser.add_argument( + "--server-config", + action="store", + help="The path to the server configuration file.", + default=DEFAULT_SERVER_CONFIG_PATH, + ) + parser.add_argument( + "--logging-config", + action="store", + help="The path to the logging configuration file.", + default=DEFAULT_LOGGING_CONFIG_PATH, + ) args = parser.parse_args() return (args.setup_only, args.server_config, args.logging_config) @@ -36,9 +51,10 @@ if "__main__" == __name__: # imports, so the log init needs to be first. try: json_setup_logging(logging_config) - except(json.JSONDecodeError) as ex: + except (json.JSONDecodeError) as ex: print(f"Error loading logging config: {ex}") exit(1) from monkey_island.cc.main import main + main(is_setup_only, server_config) From 74e0dfddc573964b1441f5f70a231433fae61041 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 10 Feb 2021 13:39:12 -0500 Subject: [PATCH 15/52] cc: expand "~" in log file configuration --- .../cc/server_utils/island_logger.py | 12 +++++++ monkey/monkey_island/cc/test_island_logger.py | 29 ++++++++++++++++ .../cc/testing/logger_config.json | 33 +++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 monkey/monkey_island/cc/test_island_logger.py create mode 100644 monkey/monkey_island/cc/testing/logger_config.json diff --git a/monkey/monkey_island/cc/server_utils/island_logger.py b/monkey/monkey_island/cc/server_utils/island_logger.py index f8eca8cbb..a512e0217 100644 --- a/monkey/monkey_island/cc/server_utils/island_logger.py +++ b/monkey/monkey_island/cc/server_utils/island_logger.py @@ -1,6 +1,7 @@ import json import logging.config import os +from typing import Dict from monkey_island.cc.consts import DEFAULT_LOGGING_CONFIG_PATH @@ -28,6 +29,17 @@ def json_setup_logging( if os.path.exists(path): with open(path, "rt") as f: config = json.load(f) + _expanduser_log_file_paths(config) logging.config.dictConfig(config) else: logging.basicConfig(level=default_level) + + +def _expanduser_log_file_paths(config: Dict): + handlers = config.get("handlers", {}) + + for handler_settings in handlers.values(): + if "filename" in handler_settings: + handler_settings["filename"] = os.path.expanduser( + handler_settings["filename"] + ) diff --git a/monkey/monkey_island/cc/test_island_logger.py b/monkey/monkey_island/cc/test_island_logger.py new file mode 100644 index 000000000..bd9de46f4 --- /dev/null +++ b/monkey/monkey_island/cc/test_island_logger.py @@ -0,0 +1,29 @@ +import logging +import os + +from monkey_island.cc.consts import MONKEY_ISLAND_ABS_PATH +from monkey_island.cc.island_logger import json_setup_logging + +TEST_LOGGING_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing", + "logger_config.json") + + +def set_home_env(monkeypatch, tmpdir): + monkeypatch.setenv("HOME", tmpdir) + + +def test_expanduser_filename(monkeypatch, tmpdir): + INFO_LOG = os.path.join(tmpdir, "info.log") + TEST_STRING = "Hello, Monkey!" + + set_home_env(monkeypatch, tmpdir) + + json_setup_logging(TEST_LOGGING_CONFIG_PATH) + + logger = logging.getLogger("TestLogger") + logger.info(TEST_STRING) + + assert os.path.isfile(INFO_LOG) + with open(INFO_LOG, "r") as f: + line = f.readline() + assert TEST_STRING in line diff --git a/monkey/monkey_island/cc/testing/logger_config.json b/monkey/monkey_island/cc/testing/logger_config.json new file mode 100644 index 000000000..b3ad82641 --- /dev/null +++ b/monkey/monkey_island/cc/testing/logger_config.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(asctime)s - %(filename)s:%(lineno)s - %(funcName)10s() - %(levelname)s - %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "simple", + "stream": "ext://sys.stdout" + }, + "info_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "INFO", + "formatter": "simple", + "filename": "~/info.log", + "maxBytes": 10485760, + "backupCount": 20, + "encoding": "utf8" + } + }, + "root": { + "level": "DEBUG", + "handlers": [ + "console", + "info_file_handler" + ] + } +} From 5b781c50a4d51b7bbf992cd583ff944453afac39 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 10 Feb 2021 13:51:48 -0500 Subject: [PATCH 16/52] cc: rename DEFAULT_LOGGING_CONFIG_PATH -> DEFAULT_LOGGER_CONFIG_PATH --- monkey/monkey_island.py | 12 ++++++------ monkey/monkey_island/cc/server_utils/consts.py | 2 +- .../monkey_island/cc/server_utils/island_logger.py | 4 ++-- monkey/monkey_island/cc/test_island_logger.py | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index ea8c96efd..8bc887d89 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -7,7 +7,7 @@ import json # noqa: E402 from monkey_island.cc.main import main # noqa: E402 from monkey_island.cc.consts import ( DEFAULT_SERVER_CONFIG_PATH, - DEFAULT_LOGGING_CONFIG_PATH, + DEFAULT_LOGGER_CONFIG_PATH, ) # noqa: E402 from monkey_island.cc.island_logger import json_setup_logging # noqa: E402 @@ -34,23 +34,23 @@ def parse_cli_args(): default=DEFAULT_SERVER_CONFIG_PATH, ) parser.add_argument( - "--logging-config", + "--logger-config", action="store", help="The path to the logging configuration file.", - default=DEFAULT_LOGGING_CONFIG_PATH, + default=DEFAULT_LOGGER_CONFIG_PATH, ) args = parser.parse_args() - return (args.setup_only, args.server_config, args.logging_config) + return (args.setup_only, args.server_config, args.logger_config) if "__main__" == __name__: - (is_setup_only, server_config, logging_config) = parse_cli_args() + (is_setup_only, server_config, logger_config) = parse_cli_args() # This is here in order to catch EVERYTHING, some functions are being called on # imports, so the log init needs to be first. try: - json_setup_logging(logging_config) + json_setup_logging(logger_config) except (json.JSONDecodeError) as ex: print(f"Error loading logging config: {ex}") exit(1) diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 4c793b587..c12a42731 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -13,6 +13,6 @@ DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json.develop" ) -DEFAULT_LOGGING_CONFIG_PATH = os.path.join( +DEFAULT_LOGGER_CONFIG_PATH = os.path.join( MONKEY_ISLAND_ABS_PATH, "cc", "island_logger_default_config.json" ) diff --git a/monkey/monkey_island/cc/server_utils/island_logger.py b/monkey/monkey_island/cc/server_utils/island_logger.py index a512e0217..224e85364 100644 --- a/monkey/monkey_island/cc/server_utils/island_logger.py +++ b/monkey/monkey_island/cc/server_utils/island_logger.py @@ -3,13 +3,13 @@ import logging.config import os from typing import Dict -from monkey_island.cc.consts import DEFAULT_LOGGING_CONFIG_PATH +from monkey_island.cc.consts import DEFAULT_LOGGER_CONFIG_PATH __author__ = "Maor.Rayzin" def json_setup_logging( - default_path=DEFAULT_LOGGING_CONFIG_PATH, + default_path=DEFAULT_LOGGER_CONFIG_PATH, default_level=logging.INFO, env_key="LOG_CFG", ): diff --git a/monkey/monkey_island/cc/test_island_logger.py b/monkey/monkey_island/cc/test_island_logger.py index bd9de46f4..b5618a95c 100644 --- a/monkey/monkey_island/cc/test_island_logger.py +++ b/monkey/monkey_island/cc/test_island_logger.py @@ -4,7 +4,7 @@ import os from monkey_island.cc.consts import MONKEY_ISLAND_ABS_PATH from monkey_island.cc.island_logger import json_setup_logging -TEST_LOGGING_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing", +TEST_LOGGER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing", "logger_config.json") @@ -18,7 +18,7 @@ def test_expanduser_filename(monkeypatch, tmpdir): set_home_env(monkeypatch, tmpdir) - json_setup_logging(TEST_LOGGING_CONFIG_PATH) + json_setup_logging(TEST_LOGGER_CONFIG_PATH) logger = logging.getLogger("TestLogger") logger.info(TEST_STRING) From e6bf085d12936e57b94e348d4a3e47397fced546 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 10 Feb 2021 14:13:39 -0500 Subject: [PATCH 17/52] address some flake8 errors --- monkey/monkey_island.py | 3 +-- monkey/monkey_island/cc/test_island_logger.py | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 8bc887d89..ec0545433 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -4,7 +4,6 @@ gevent_monkey.patch_all() import json # noqa: E402 -from monkey_island.cc.main import main # noqa: E402 from monkey_island.cc.consts import ( DEFAULT_SERVER_CONFIG_PATH, DEFAULT_LOGGER_CONFIG_PATH, @@ -55,6 +54,6 @@ if "__main__" == __name__: print(f"Error loading logging config: {ex}") exit(1) - from monkey_island.cc.main import main + from monkey_island.cc.main import main # noqa: E402 main(is_setup_only, server_config) diff --git a/monkey/monkey_island/cc/test_island_logger.py b/monkey/monkey_island/cc/test_island_logger.py index b5618a95c..050d5c7c5 100644 --- a/monkey/monkey_island/cc/test_island_logger.py +++ b/monkey/monkey_island/cc/test_island_logger.py @@ -4,8 +4,9 @@ import os from monkey_island.cc.consts import MONKEY_ISLAND_ABS_PATH from monkey_island.cc.island_logger import json_setup_logging -TEST_LOGGER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing", - "logger_config.json") +TEST_LOGGER_CONFIG_PATH = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", "testing", "logger_config.json" +) def set_home_env(monkeypatch, tmpdir): From 1f57610005d824183f351e80b25e3073f70f39a4 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 12 Feb 2021 10:24:23 -0500 Subject: [PATCH 18/52] monkey_island.py: Add TODO to refactor argument handling --- monkey/monkey_island.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index ec0545433..837f6f714 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -44,6 +44,8 @@ def parse_cli_args(): if "__main__" == __name__: + # TODO: Address https://github.com/guardicore/monkey/pull/963#discussion_r575022748 + # before merging appimage PR (is_setup_only, server_config, logger_config) = parse_cli_args() # This is here in order to catch EVERYTHING, some functions are being called on From 21e0b5170b49a694657b907f13301a1d4d6a5498 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 12 Feb 2021 11:57:16 -0500 Subject: [PATCH 19/52] cc: explicitly cast tmpdir to str in test_island_logger.py --- monkey/monkey_island/cc/test_island_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/test_island_logger.py b/monkey/monkey_island/cc/test_island_logger.py index 050d5c7c5..ca7dfd704 100644 --- a/monkey/monkey_island/cc/test_island_logger.py +++ b/monkey/monkey_island/cc/test_island_logger.py @@ -10,7 +10,7 @@ TEST_LOGGER_CONFIG_PATH = os.path.join( def set_home_env(monkeypatch, tmpdir): - monkeypatch.setenv("HOME", tmpdir) + monkeypatch.setenv("HOME", str(tmpdir)) def test_expanduser_filename(monkeypatch, tmpdir): From ef1ef3475ba8ec721ec04faea4566bb2e50f6167 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 24 Feb 2021 09:52:34 +0200 Subject: [PATCH 20/52] Extracted island argument parsing into a separate file --- monkey/monkey_island.py | 44 ++++----------------------- monkey/monkey_island/cc/arg_parser.py | 42 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 38 deletions(-) create mode 100644 monkey/monkey_island/cc/arg_parser.py diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 837f6f714..e2323a7c3 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -1,5 +1,7 @@ from gevent import monkey as gevent_monkey +from monkey_island.cc.arg_parser import parse_cli_args + gevent_monkey.patch_all() import json # noqa: E402 @@ -11,51 +13,17 @@ from monkey_island.cc.consts import ( from monkey_island.cc.island_logger import json_setup_logging # noqa: E402 -def parse_cli_args(): - import argparse - - parser = argparse.ArgumentParser( - description="Infection Monkey Island CnC Server. See https://infectionmonkey.com", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - "-s", - "--setup-only", - action="store_true", - help="Pass this flag to cause the Island to setup and exit without actually starting. " - "This is useful for preparing Island to boot faster later-on, so for " - "compiling/packaging Islands.", - ) - parser.add_argument( - "--server-config", - action="store", - help="The path to the server configuration file.", - default=DEFAULT_SERVER_CONFIG_PATH, - ) - parser.add_argument( - "--logger-config", - action="store", - help="The path to the logging configuration file.", - default=DEFAULT_LOGGER_CONFIG_PATH, - ) - args = parser.parse_args() - - return (args.setup_only, args.server_config, args.logger_config) - - if "__main__" == __name__: - # TODO: Address https://github.com/guardicore/monkey/pull/963#discussion_r575022748 - # before merging appimage PR - (is_setup_only, server_config, logger_config) = parse_cli_args() + island_args = parse_cli_args() # This is here in order to catch EVERYTHING, some functions are being called on # imports, so the log init needs to be first. try: - json_setup_logging(logger_config) - except (json.JSONDecodeError) as ex: + json_setup_logging(island_args.logger_config) + except json.JSONDecodeError as ex: print(f"Error loading logging config: {ex}") exit(1) from monkey_island.cc.main import main # noqa: E402 - main(is_setup_only, server_config) + main(island_args.setup_only, island_args.server_config) diff --git a/monkey/monkey_island/cc/arg_parser.py b/monkey/monkey_island/cc/arg_parser.py new file mode 100644 index 000000000..8716a9b4a --- /dev/null +++ b/monkey/monkey_island/cc/arg_parser.py @@ -0,0 +1,42 @@ +from dataclasses import dataclass + +from monkey_island.cc.consts import DEFAULT_SERVER_CONFIG_PATH, DEFAULT_LOGGER_CONFIG_PATH + + +@dataclass +class IslandArgs: + setup_only: bool + server_config: str + logger_config: str + + +def parse_cli_args() -> IslandArgs: + import argparse + + parser = argparse.ArgumentParser( + description="Infection Monkey Island CnC Server. See https://infectionmonkey.com", + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "-s", + "--setup-only", + action="store_true", + help="Pass this flag to cause the Island to setup and exit without actually starting. " + "This is useful for preparing Island to boot faster later-on, so for " + "compiling/packaging Islands.", + ) + parser.add_argument( + "--server-config", + action="store", + help="The path to the server configuration file.", + default=DEFAULT_SERVER_CONFIG_PATH, + ) + parser.add_argument( + "--logger-config", + action="store", + help="The path to the logging configuration file.", + default=DEFAULT_LOGGER_CONFIG_PATH, + ) + args = parser.parse_args() + + return IslandArgs(args.setup_only, args.server_config, args.logger_config) From 64018eb37305cf69ef4b1bacc49cc857f0e29a27 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 24 Feb 2021 10:07:26 +0200 Subject: [PATCH 21/52] Extracted home environment mocking into a reusable fixture and added a todo, to move it to our fixture list --- monkey/monkey_island/cc/test_island_logger.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/test_island_logger.py b/monkey/monkey_island/cc/test_island_logger.py index ca7dfd704..a1cc27aa7 100644 --- a/monkey/monkey_island/cc/test_island_logger.py +++ b/monkey/monkey_island/cc/test_island_logger.py @@ -1,6 +1,8 @@ import logging import os +import pytest + from monkey_island.cc.consts import MONKEY_ISLAND_ABS_PATH from monkey_island.cc.island_logger import json_setup_logging @@ -9,16 +11,16 @@ TEST_LOGGER_CONFIG_PATH = os.path.join( ) -def set_home_env(monkeypatch, tmpdir): +# TODO move into monkey/monkey_island/cc/test_common/fixtures after rebase/backmerge +@pytest.fixture +def mock_home_env(monkeypatch, tmpdir): monkeypatch.setenv("HOME", str(tmpdir)) -def test_expanduser_filename(monkeypatch, tmpdir): +def test_expanduser_filename(mock_home_env, tmpdir): INFO_LOG = os.path.join(tmpdir, "info.log") TEST_STRING = "Hello, Monkey!" - set_home_env(monkeypatch, tmpdir) - json_setup_logging(TEST_LOGGER_CONFIG_PATH) logger = logging.getLogger("TestLogger") From 2d971d95fc3b81ceecd1ca2864c41b8893b756b5 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 11 Feb 2021 07:27:56 -0500 Subject: [PATCH 22/52] build: select server and logger config at runtime in appimage --- deployment_scripts/appimage/build_appimage.sh | 11 +++++-- .../appimage/island_logger_config.json | 33 +++++++++++++++++++ .../appimage/monkey_island_builder.yml | 2 +- .../appimage/{run.sh => run_appimage.sh} | 16 +++++++-- .../cc/environment/environment_singleton.py | 2 ++ 5 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 deployment_scripts/appimage/island_logger_config.json rename deployment_scripts/appimage/{run.sh => run_appimage.sh} (51%) diff --git a/deployment_scripts/appimage/build_appimage.sh b/deployment_scripts/appimage/build_appimage.sh index 407df9bb7..d3a0fab47 100755 --- a/deployment_scripts/appimage/build_appimage.sh +++ b/deployment_scripts/appimage/build_appimage.sh @@ -94,7 +94,7 @@ clone_monkey_repo() { fi log_message "Cloning files from git" - branch=${2:-"develop"} + branch=${2:-"select-logger-config-at-runtime"} git clone --single-branch --recurse-submodules -b "$branch" "${MONKEY_GIT_URL}" "${REPO_MONKEY_HOME}" 2>&1 || handle_error chmod 774 -R "${MONKEY_HOME}" @@ -105,7 +105,12 @@ copy_monkey_island_to_appdir() { cp $REPO_MONKEY_SRC/monkey_island.py $INSTALL_DIR cp -r $REPO_MONKEY_SRC/common $INSTALL_DIR cp -r $REPO_MONKEY_SRC/monkey_island $INSTALL_DIR - cp ./run.sh $INSTALL_DIR/monkey_island/linux/ + cp ./run_appimage.sh $INSTALL_DIR/monkey_island/linux/ + cp ./island_logger_config.json $INSTALL_DIR/ + + # TODO: This is a workaround that may be able to be removed after PR #848 is + # merged. See monkey_island/cc/environment_singleton.py for more information. + cp $INSTALL_DIR/monkey_island/cc/server_config.json.standard $INSTALL_DIR/monkey_island/cc/server_config.json } install_monkey_island_python_dependencies() { @@ -231,7 +236,7 @@ cp $REPO_MONKEY_SRC/monkey_island/cc/ui/src/images/monkey-icon.svg $APPDIR/usr/s #cp ./monkey_island.desktop $APPDIR log_message "Building AppImage" -appimage-builder --recipe monkey_island_builder.yml --skip-appimage +appimage-builder --recipe monkey_island_builder.yml log_message "Deployment script finished." diff --git a/deployment_scripts/appimage/island_logger_config.json b/deployment_scripts/appimage/island_logger_config.json new file mode 100644 index 000000000..4acf875e3 --- /dev/null +++ b/deployment_scripts/appimage/island_logger_config.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(asctime)s - %(filename)s:%(lineno)s - %(funcName)10s() - %(levelname)s - %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "simple", + "stream": "ext://sys.stdout" + }, + "info_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "INFO", + "formatter": "simple", + "filename": "~/.monkey_island/info.log", + "maxBytes": 10485760, + "backupCount": 20, + "encoding": "utf8" + } + }, + "root": { + "level": "DEBUG", + "handlers": [ + "console", + "info_file_handler" + ] + } +} diff --git a/deployment_scripts/appimage/monkey_island_builder.yml b/deployment_scripts/appimage/monkey_island_builder.yml index f0848211a..6b77c7437 100644 --- a/deployment_scripts/appimage/monkey_island_builder.yml +++ b/deployment_scripts/appimage/monkey_island_builder.yml @@ -9,7 +9,7 @@ AppDir: icon: monkey-icon version: 1.10.0 exec: bin/bash - exec_args: "$APPDIR/usr/src/monkey_island/linux/run.sh $HOME/.monkey_island/db" + exec_args: "$APPDIR/usr/src/monkey_island/linux/run_appimage.sh" apt: diff --git a/deployment_scripts/appimage/run.sh b/deployment_scripts/appimage/run_appimage.sh similarity index 51% rename from deployment_scripts/appimage/run.sh rename to deployment_scripts/appimage/run_appimage.sh index 328535e78..17c8802b2 100644 --- a/deployment_scripts/appimage/run.sh +++ b/deployment_scripts/appimage/run_appimage.sh @@ -1,5 +1,14 @@ #!/bin/bash +DOT_MONKEY=$HOME/.monkey_island/ + +configure_default_logging() { + if [ ! -f $DOT_MONKEY/island_logger_config.json ]; then + cp $APPDIR/usr/src/island_logger_config.json $DOT_MONKEY + fi +} + + # Detecting command that calls python 3.7 python_cmd="" if [[ $(python --version 2>&1) == *"Python 3.7"* ]]; then @@ -12,10 +21,13 @@ if [[ $(python3.7 --version 2>&1) == *"Python 3.7"* ]]; then python_cmd="python3.7" fi -DB_DIR=${1:-"./monkey_island/bin/mongodb/db"} +mkdir -p $DOT_MONKEY +DB_DIR=$DOT_MONKEY/db mkdir -p $DB_DIR +configure_default_logging + cd $APPDIR/usr/src ./monkey_island/bin/mongodb/bin/mongod --dbpath $DB_DIR & -${python_cmd} ./monkey_island.py +${python_cmd} ./monkey_island.py --server-config $DOT_MONKEY/server_config.json --logger-config $DOT_MONKEY/island_logger_config.json diff --git a/monkey/monkey_island/cc/environment/environment_singleton.py b/monkey/monkey_island/cc/environment/environment_singleton.py index accb104ce..01e83096d 100644 --- a/monkey/monkey_island/cc/environment/environment_singleton.py +++ b/monkey/monkey_island/cc/environment/environment_singleton.py @@ -51,4 +51,6 @@ def initialize_from_file(file_path): raise +# TODO: This is only needed so that unit tests pass. After PR #848 is merged, we may be +# able to remove this line. initialize_from_file(DEFAULT_SERVER_CONFIG_PATH) From 438a2701d4768333e22c6772375bc44ac0aa4546 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 11 Feb 2021 13:48:22 -0500 Subject: [PATCH 23/52] cc: add `data_dir` property to EnvironmentConfig --- .../cc/environment/environment_config.py | 25 +++++++++++++------ .../cc/environment/environment_singleton.py | 3 +++ .../cc/environment/test_environment_config.py | 23 ++++++++++++----- .../monkey_island/cc/server_utils/consts.py | 2 ++ .../cc/server_utils/island_logger.py | 2 +- .../{ => server_utils}/test_island_logger.py | 4 +-- .../environment/server_config_data_dir.json | 7 ++++++ 7 files changed, 49 insertions(+), 17 deletions(-) rename monkey/monkey_island/cc/{ => server_utils}/test_island_logger.py (83%) create mode 100644 monkey/monkey_island/cc/testing/environment/server_config_data_dir.json diff --git a/monkey/monkey_island/cc/environment/environment_config.py b/monkey/monkey_island/cc/environment/environment_config.py index aec1a777f..3bc9327b4 100644 --- a/monkey/monkey_island/cc/environment/environment_config.py +++ b/monkey/monkey_island/cc/environment/environment_config.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Dict, List import monkey_island.cc.environment.server_config_generator as server_config_generator +from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR from monkey_island.cc.environment.user_creds import UserCreds from monkey_island.cc.resources.auth.auth_user import User from monkey_island.cc.resources.auth.user_store import UserStore @@ -18,6 +19,7 @@ class EnvironmentConfig: self.deployment = None self.user_creds = None self.aws = None + self.data_dir = None self._load_from_file(self._server_config_path) @@ -26,7 +28,7 @@ class EnvironmentConfig: if not Path(file_path).is_file(): server_config_generator.create_default_config_file(file_path) - with open(file_path, 'r') as f: + with open(file_path, "r") as f: config_content = f.read() self._load_from_json(config_content) @@ -37,22 +39,29 @@ class EnvironmentConfig: def _load_from_dict(self, dict_data: Dict): user_creds = UserCreds.get_from_dict(dict_data) - aws = dict_data['aws'] if 'aws' in dict_data else None + aws = dict_data["aws"] if "aws" in dict_data else None + data_dir = ( + dict_data["data_dir"] if "data_dir" in dict_data else DEFAULT_DATA_DIR + ) - self.server_config = dict_data['server_config'] - self.deployment = dict_data['deployment'] + self.server_config = dict_data["server_config"] + self.deployment = dict_data["deployment"] self.user_creds = user_creds self.aws = aws + self.data_dir = data_dir def save_to_file(self): - with open(self._server_config_path, 'w') as f: + with open(self._server_config_path, "w") as f: f.write(json.dumps(self.to_dict(), indent=2)) def to_dict(self) -> Dict: - config_dict = {'server_config': self.server_config, - 'deployment': self.deployment} + config_dict = { + "server_config": self.server_config, + "deployment": self.deployment, + "data_dir": self.data_dir, + } if self.aws: - config_dict.update({'aws': self.aws}) + config_dict.update({"aws": self.aws}) config_dict.update(self.user_creds.to_dict()) return config_dict diff --git a/monkey/monkey_island/cc/environment/environment_singleton.py b/monkey/monkey_island/cc/environment/environment_singleton.py index 01e83096d..aa2dc32d6 100644 --- a/monkey/monkey_island/cc/environment/environment_singleton.py +++ b/monkey/monkey_island/cc/environment/environment_singleton.py @@ -20,6 +20,7 @@ ENV_DICT = { } env = None +config = None def set_env(env_type: str, env_config: EnvironmentConfig): @@ -39,6 +40,8 @@ def set_to_standard(): def initialize_from_file(file_path): + global config + try: config = EnvironmentConfig(file_path) diff --git a/monkey/monkey_island/cc/environment/test_environment_config.py b/monkey/monkey_island/cc/environment/test_environment_config.py index b06e26301..a8e8a1beb 100644 --- a/monkey/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/monkey_island/cc/environment/test_environment_config.py @@ -4,7 +4,7 @@ import shutil import pytest -from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH +from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, MONKEY_ISLAND_ABS_PATH from monkey_island.cc.environment.environment_config import EnvironmentConfig from monkey_island.cc.environment.user_creds import UserCreds @@ -23,6 +23,7 @@ PARTIAL_CREDENTIALS = os.path.join( STANDARD_WITH_CREDENTIALS = os.path.join( TEST_RESOURCES_DIR, "server_config_standard_with_credentials.json" ) +DATA_DIR = os.path.join(TEST_RESOURCES_DIR, "server_config_data_dir.json") @pytest.fixture @@ -33,28 +34,31 @@ def config_file(tmpdir): def test_get_with_credentials(): config_dict = EnvironmentConfig(WITH_CREDENTIALS).to_dict() - assert len(config_dict.keys()) == 4 + assert len(config_dict.keys()) == 5 assert config_dict["server_config"] == "password" assert config_dict["deployment"] == "develop" assert config_dict["user"] == "test" assert config_dict["password_hash"] == "abcdef" + assert config_dict["data_dir"] == DEFAULT_DATA_DIR def test_get_with_no_credentials(): config_dict = EnvironmentConfig(NO_CREDENTIALS).to_dict() - assert len(config_dict.keys()) == 2 + assert len(config_dict.keys()) == 3 assert config_dict["server_config"] == "password" assert config_dict["deployment"] == "develop" + assert config_dict["data_dir"] == DEFAULT_DATA_DIR def test_get_with_partial_credentials(): config_dict = EnvironmentConfig(PARTIAL_CREDENTIALS).to_dict() - assert len(config_dict.keys()) == 3 + assert len(config_dict.keys()) == 4 assert config_dict["server_config"] == "password" assert config_dict["deployment"] == "develop" assert config_dict["user"] == "test" + assert config_dict["data_dir"] == DEFAULT_DATA_DIR def test_save_to_file(config_file): @@ -67,12 +71,13 @@ def test_save_to_file(config_file): with open(config_file, "r") as f: from_file = json.load(f) - assert len(from_file.keys()) == 5 + assert len(from_file.keys()) == 6 assert from_file["server_config"] == "standard" assert from_file["deployment"] == "develop" assert from_file["user"] == "test" assert from_file["password_hash"] == "abcdef" assert from_file["aws"] == "test_aws" + assert from_file["data_dir"] == DEFAULT_DATA_DIR def test_add_user(config_file): @@ -88,7 +93,7 @@ def test_add_user(config_file): with open(config_file, "r") as f: from_file = json.load(f) - assert len(from_file.keys()) == 4 + assert len(from_file.keys()) == 5 assert from_file["user"] == new_user assert from_file["password_hash"] == new_password_hash @@ -113,3 +118,9 @@ def test_generate_default_file(config_file): assert environment_config.user_creds.username == "" assert environment_config.user_creds.password_hash == "" assert environment_config.aws is None + assert environment_config.data_dir == DEFAULT_DATA_DIR + + +def test_data_dir(): + environment_config = EnvironmentConfig(DATA_DIR) + assert environment_config.data_dir == "/test/data/dir" diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index c12a42731..5a0e69581 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -16,3 +16,5 @@ DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( DEFAULT_LOGGER_CONFIG_PATH = os.path.join( MONKEY_ISLAND_ABS_PATH, "cc", "island_logger_default_config.json" ) + +DEFAULT_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc") diff --git a/monkey/monkey_island/cc/server_utils/island_logger.py b/monkey/monkey_island/cc/server_utils/island_logger.py index 224e85364..1efbb7734 100644 --- a/monkey/monkey_island/cc/server_utils/island_logger.py +++ b/monkey/monkey_island/cc/server_utils/island_logger.py @@ -3,7 +3,7 @@ import logging.config import os from typing import Dict -from monkey_island.cc.consts import DEFAULT_LOGGER_CONFIG_PATH +from monkey_island.cc.server_utils.consts import DEFAULT_LOGGER_CONFIG_PATH __author__ = "Maor.Rayzin" diff --git a/monkey/monkey_island/cc/test_island_logger.py b/monkey/monkey_island/cc/server_utils/test_island_logger.py similarity index 83% rename from monkey/monkey_island/cc/test_island_logger.py rename to monkey/monkey_island/cc/server_utils/test_island_logger.py index a1cc27aa7..af58f4b75 100644 --- a/monkey/monkey_island/cc/test_island_logger.py +++ b/monkey/monkey_island/cc/server_utils/test_island_logger.py @@ -3,8 +3,8 @@ import os import pytest -from monkey_island.cc.consts import MONKEY_ISLAND_ABS_PATH -from monkey_island.cc.island_logger import json_setup_logging +from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH +from monkey_island.cc.server_utils.island_logger import json_setup_logging TEST_LOGGER_CONFIG_PATH = os.path.join( MONKEY_ISLAND_ABS_PATH, "cc", "testing", "logger_config.json" diff --git a/monkey/monkey_island/cc/testing/environment/server_config_data_dir.json b/monkey/monkey_island/cc/testing/environment/server_config_data_dir.json new file mode 100644 index 000000000..b9d6845f3 --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_data_dir.json @@ -0,0 +1,7 @@ +{ + "server_config": "password", + "deployment": "develop", + "user": "test", + "password_hash": "abcdef", + "data_dir": "/test/data/dir" +} From 3f6c268f40170450803b749e44999092dda78eda Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 11 Feb 2021 20:16:31 -0500 Subject: [PATCH 24/52] cc: allow encryptor to store key file in variable locations --- monkey/monkey_island/cc/main.py | 2 + .../cc/server_utils/encryptor.py | 38 +++++++++++-------- .../technique_report_tools.py | 4 +- monkey/monkey_island/cc/services/config.py | 22 +++++------ .../services/telemetry/processing/exploit.py | 2 +- .../telemetry/processing/system_info.py | 2 +- .../scoutsuite/scoutsuite_auth_service.py | 2 +- .../test_scoutsuite_auth_service.py | 7 ++-- monkey/monkey_island/cc/test_encryptor.py | 29 ++++++++++++++ monkey/monkey_island/cc/testing/mongo_key.bin | 2 + 10 files changed, 75 insertions(+), 35 deletions(-) create mode 100644 monkey/monkey_island/cc/test_encryptor.py create mode 100644 monkey/monkey_island/cc/testing/mongo_key.bin diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index 2018da89c..4bb704aee 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -23,6 +23,7 @@ from monkey_island.cc.app import init_app # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 from monkey_island.cc.database import get_db_version # noqa: E402 from monkey_island.cc.database import is_db_server_up # noqa: E402 +from monkey_island.cc.encryptor import initialize_encryptor # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 @@ -34,6 +35,7 @@ MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" def main(should_setup_only=False, server_config_filename=DEFAULT_SERVER_CONFIG_PATH): logger.info("Starting bootloader server") env_singleton.initialize_from_file(server_config_filename) + initialize_encryptor(env_singleton.config.data_dir) mongo_url = os.environ.get('MONGO_URL', env_singleton.env.get_mongo_url()) bootloader_server_thread = Thread(target=BootloaderHttpServer(mongo_url).serve_forever, daemon=True) diff --git a/monkey/monkey_island/cc/server_utils/encryptor.py b/monkey/monkey_island/cc/server_utils/encryptor.py index cce7d464a..2a9ce8fea 100644 --- a/monkey/monkey_island/cc/server_utils/encryptor.py +++ b/monkey/monkey_island/cc/server_utils/encryptor.py @@ -6,33 +6,32 @@ import os from Crypto import Random # noqa: DUO133 # nosec: B413 from Crypto.Cipher import AES # noqa: DUO133 # nosec: B413 -from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH - __author__ = "itay.mizeretz" +_encryptor = None + class Encryptor: _BLOCK_SIZE = 32 - _DB_PASSWORD_FILENAME = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc/mongo_key.bin') + _PASSWORD_FILENAME = "mongo_key.bin" - def __init__(self): - self._load_key() + def __init__(self, data_dir): + password_file = os.path.join(data_dir, self._PASSWORD_FILENAME) - def _init_key(self): + if os.path.exists(password_file): + self._load_existing_key(password_file) + else: + self._init_key(password_file) + + def _init_key(self, password_file): self._cipher_key = Random.new().read(self._BLOCK_SIZE) - with open(self._DB_PASSWORD_FILENAME, 'wb') as f: + with open(password_file, 'wb') as f: f.write(self._cipher_key) - def _load_existing_key(self): - with open(self._DB_PASSWORD_FILENAME, 'rb') as f: + def _load_existing_key(self, password_file): + with open(password_file, 'rb') as f: self._cipher_key = f.read() - def _load_key(self): - if os.path.exists(self._DB_PASSWORD_FILENAME): - self._load_existing_key() - else: - self._init_key() - def _pad(self, message): return message + (self._BLOCK_SIZE - (len(message) % self._BLOCK_SIZE)) * chr( self._BLOCK_SIZE - (len(message) % self._BLOCK_SIZE)) @@ -52,4 +51,11 @@ class Encryptor: return self._unpad(cipher.decrypt(enc_message[AES.block_size:]).decode()) -encryptor = Encryptor() +def initialize_encryptor(data_dir): + global _encryptor + + _encryptor = Encryptor(data_dir) + + +def encryptor(): + return _encryptor diff --git a/monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py b/monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py index 34be687a4..35356cad4 100644 --- a/monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py +++ b/monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py @@ -29,7 +29,7 @@ def censor_password(password, plain_chars=3, secret_chars=5): """ if not password: return "" - password = encryptor.dec(password) + password = encryptor().dec(password) return password[0:plain_chars] + '*' * secret_chars @@ -42,5 +42,5 @@ def censor_hash(hash_, plain_chars=5): """ if not hash_: return "" - hash_ = encryptor.dec(hash_) + hash_ = encryptor().dec(hash_) return hash_[0: plain_chars] + ' ...' diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 390380131..edf44b967 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -75,9 +75,9 @@ class ConfigService: if should_decrypt: if config_key_as_arr in ENCRYPTED_CONFIG_VALUES: if isinstance(config, str): - config = encryptor.dec(config) + config = encryptor().dec(config) elif isinstance(config, list): - config = [encryptor.dec(x) for x in config] + config = [encryptor().dec(x) for x in config] return config @staticmethod @@ -112,7 +112,7 @@ class ConfigService: if item_value in items_from_config: return if should_encrypt: - item_value = encryptor.enc(item_value) + item_value = encryptor().enc(item_value) mongo.db.config.update( {'name': 'newconfig'}, {'$addToSet': {item_key: item_value}}, @@ -297,9 +297,9 @@ class ConfigService: if flat_config[key] and isinstance(flat_config[key][0], dict) and 'public_key' in flat_config[key][0]: flat_config[key] = [ConfigService.decrypt_ssh_key_pair(item) for item in flat_config[key]] else: - flat_config[key] = [encryptor.dec(item) for item in flat_config[key]] + flat_config[key] = [encryptor().dec(item) for item in flat_config[key]] else: - flat_config[key] = encryptor.dec(flat_config[key]) + flat_config[key] = encryptor().dec(flat_config[key]) return flat_config @staticmethod @@ -320,19 +320,19 @@ class ConfigService: config_arr[i] = ConfigService.decrypt_ssh_key_pair(config_arr[i]) if is_decrypt else \ ConfigService.decrypt_ssh_key_pair(config_arr[i], True) else: - config_arr[i] = encryptor.dec(config_arr[i]) if is_decrypt else encryptor.enc(config_arr[i]) + config_arr[i] = encryptor().dec(config_arr[i]) if is_decrypt else encryptor().enc(config_arr[i]) else: parent_config_arr[config_arr_as_array[-1]] = \ - encryptor.dec(config_arr) if is_decrypt else encryptor.enc(config_arr) + encryptor().dec(config_arr) if is_decrypt else encryptor().enc(config_arr) @staticmethod def decrypt_ssh_key_pair(pair, encrypt=False): if encrypt: - pair['public_key'] = encryptor.enc(pair['public_key']) - pair['private_key'] = encryptor.enc(pair['private_key']) + pair['public_key'] = encryptor().enc(pair['public_key']) + pair['private_key'] = encryptor().enc(pair['private_key']) else: - pair['public_key'] = encryptor.dec(pair['public_key']) - pair['private_key'] = encryptor.dec(pair['private_key']) + pair['public_key'] = encryptor().dec(pair['public_key']) + pair['private_key'] = encryptor().dec(pair['private_key']) return pair @staticmethod diff --git a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py index 3d8588663..b2baff562 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py @@ -66,4 +66,4 @@ def encrypt_exploit_creds(telemetry_json): for field in ['password', 'lm_hash', 'ntlm_hash']: credential = attempts[i][field] if len(credential) > 0: - attempts[i][field] = encryptor.enc(credential) + attempts[i][field] = encryptor().enc(credential) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index d3e7cfb54..5af4d1dde 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -63,7 +63,7 @@ def encrypt_system_info_ssh_keys(ssh_info): for idx, user in enumerate(ssh_info): for field in ['public_key', 'private_key', 'known_hosts']: if ssh_info[idx][field]: - ssh_info[idx][field] = encryptor.enc(ssh_info[idx][field]) + ssh_info[idx][field] = encryptor().enc(ssh_info[idx][field]) def process_credential_info(telemetry_json): diff --git a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py index 701598168..2bc7b2323 100644 --- a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py +++ b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py @@ -37,7 +37,7 @@ def set_aws_keys(access_key_id: str, secret_access_key: str, session_token: str) def _set_aws_key(key_type: str, key_value: str): path_to_keys = AWS_KEYS_PATH - encrypted_key = encryptor.enc(key_value) + encrypted_key = encryptor().enc(key_value) ConfigService.set_config_value(path_to_keys + [key_type], encrypted_key) diff --git a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/test_scoutsuite_auth_service.py b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/test_scoutsuite_auth_service.py index c35e55a8f..72a1c711a 100644 --- a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/test_scoutsuite_auth_service.py +++ b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/test_scoutsuite_auth_service.py @@ -4,7 +4,7 @@ import pytest import dpath.util from monkey_island.cc.database import mongo -from monkey_island.cc.server_utils import encryptor +from monkey_island.cc.server_utils.encryptor import initialize_encryptor, encryptor from monkey_island.cc.services.config import ConfigService from common.config_value_paths import AWS_KEYS_PATH from monkey_island.cc.services.zero_trust.scoutsuite.scoutsuite_auth_service import is_aws_keys_setup @@ -16,7 +16,7 @@ class MockObject: @pytest.mark.usefixtures(FixtureEnum.USES_DATABASE) -def test_is_aws_keys_setup(): +def test_is_aws_keys_setup(tmp_path): # Mock default configuration ConfigService.init_default_config() mongo.db = MockObject() @@ -26,7 +26,8 @@ def test_is_aws_keys_setup(): assert not is_aws_keys_setup() # Make sure noone changed config path and broke this function - bogus_key_value = encryptor.encryptor.enc('bogus_aws_key') + initialize_encryptor(tmp_path) + bogus_key_value = encryptor().enc('bogus_aws_key') dpath.util.set(ConfigService.default_config, AWS_KEYS_PATH+['aws_secret_access_key'], bogus_key_value) dpath.util.set(ConfigService.default_config, AWS_KEYS_PATH+['aws_access_key_id'], bogus_key_value) diff --git a/monkey/monkey_island/cc/test_encryptor.py b/monkey/monkey_island/cc/test_encryptor.py new file mode 100644 index 000000000..9a424cc01 --- /dev/null +++ b/monkey/monkey_island/cc/test_encryptor.py @@ -0,0 +1,29 @@ +import os + +from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH +from monkey_island.cc.server_utils.encryptor import initialize_encryptor, encryptor + + +TEST_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing") +PASSWORD_FILENAME = "mongo_key.bin" + +PLAINTEXT = "Hello, Monkey!" +CYPHERTEXT = "vKgvD6SjRyIh1dh2AM/rnTa0NI/vjfwnbZLbMocWtE4e42WJmSUz2ordtbQrH1Fq" + + +def test_aes_cbc_encryption(): + initialize_encryptor(TEST_DATA_DIR) + + assert encryptor().enc(PLAINTEXT) != PLAINTEXT + + +def test_aes_cbc_decryption(): + initialize_encryptor(TEST_DATA_DIR) + + assert encryptor().dec(CYPHERTEXT) == PLAINTEXT + + +def test_create_new_password_file(tmpdir): + initialize_encryptor(tmpdir) + + assert os.path.isfile(os.path.join(tmpdir, PASSWORD_FILENAME)) diff --git a/monkey/monkey_island/cc/testing/mongo_key.bin b/monkey/monkey_island/cc/testing/mongo_key.bin new file mode 100644 index 000000000..6b8091efb --- /dev/null +++ b/monkey/monkey_island/cc/testing/mongo_key.bin @@ -0,0 +1,2 @@ ++ù­Æõ RO +ý)êž¾T“|ÄRSíÞ&Cá™â \ No newline at end of file From d2652381078fd0894b9d812292e691657eafda88 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 11 Feb 2021 20:17:22 -0500 Subject: [PATCH 25/52] cc: format encryptor.py with black --- .../monkey_island/cc/server_utils/encryptor.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/monkey/monkey_island/cc/server_utils/encryptor.py b/monkey/monkey_island/cc/server_utils/encryptor.py index 2a9ce8fea..dd088530f 100644 --- a/monkey/monkey_island/cc/server_utils/encryptor.py +++ b/monkey/monkey_island/cc/server_utils/encryptor.py @@ -25,30 +25,33 @@ class Encryptor: def _init_key(self, password_file): self._cipher_key = Random.new().read(self._BLOCK_SIZE) - with open(password_file, 'wb') as f: + with open(password_file, "wb") as f: f.write(self._cipher_key) def _load_existing_key(self, password_file): - with open(password_file, 'rb') as f: + with open(password_file, "rb") as f: self._cipher_key = f.read() def _pad(self, message): return message + (self._BLOCK_SIZE - (len(message) % self._BLOCK_SIZE)) * chr( - self._BLOCK_SIZE - (len(message) % self._BLOCK_SIZE)) + self._BLOCK_SIZE - (len(message) % self._BLOCK_SIZE) + ) def _unpad(self, message: str): - return message[0:-ord(message[len(message) - 1])] + return message[0 : -ord(message[len(message) - 1])] def enc(self, message: str): cipher_iv = Random.new().read(AES.block_size) cipher = AES.new(self._cipher_key, AES.MODE_CBC, cipher_iv) - return base64.b64encode(cipher_iv + cipher.encrypt(self._pad(message).encode())).decode() + return base64.b64encode( + cipher_iv + cipher.encrypt(self._pad(message).encode()) + ).decode() def dec(self, enc_message): enc_message = base64.b64decode(enc_message) - cipher_iv = enc_message[0:AES.block_size] + cipher_iv = enc_message[0 : AES.block_size] cipher = AES.new(self._cipher_key, AES.MODE_CBC, cipher_iv) - return self._unpad(cipher.decrypt(enc_message[AES.block_size:]).decode()) + return self._unpad(cipher.decrypt(enc_message[AES.block_size :]).decode()) def initialize_encryptor(data_dir): From a09cd8f49743dec1bc18ed27a9d360229f60f3ac Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 16 Feb 2021 10:59:35 -0500 Subject: [PATCH 26/52] cc: expanduser in data_dir path in Encryptor --- monkey/monkey_island/cc/server_utils/encryptor.py | 4 +++- monkey/monkey_island/cc/test_encryptor.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/server_utils/encryptor.py b/monkey/monkey_island/cc/server_utils/encryptor.py index dd088530f..36fff735c 100644 --- a/monkey/monkey_island/cc/server_utils/encryptor.py +++ b/monkey/monkey_island/cc/server_utils/encryptor.py @@ -16,7 +16,9 @@ class Encryptor: _PASSWORD_FILENAME = "mongo_key.bin" def __init__(self, data_dir): - password_file = os.path.join(data_dir, self._PASSWORD_FILENAME) + password_file = os.path.expanduser( + os.path.join(data_dir, self._PASSWORD_FILENAME) + ) if os.path.exists(password_file): self._load_existing_key(password_file) diff --git a/monkey/monkey_island/cc/test_encryptor.py b/monkey/monkey_island/cc/test_encryptor.py index 9a424cc01..3380d1d43 100644 --- a/monkey/monkey_island/cc/test_encryptor.py +++ b/monkey/monkey_island/cc/test_encryptor.py @@ -27,3 +27,11 @@ def test_create_new_password_file(tmpdir): initialize_encryptor(tmpdir) assert os.path.isfile(os.path.join(tmpdir, PASSWORD_FILENAME)) + + +def test_expand_home(monkeypatch, tmpdir): + monkeypatch.setenv("HOME", str(tmpdir)) + + initialize_encryptor("~/") + + assert os.path.isfile(os.path.join(tmpdir, "mongo_key.bin")) From 044c656543cd5d351b9103ca62b41f05f3a7ac4a Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 25 Feb 2021 06:36:20 -0500 Subject: [PATCH 27/52] cc: rename encryptor() -> get_encryptor() --- monkey/monkey_island/cc/main.py | 2 +- .../cc/server_utils/encryptor.py | 2 +- .../technique_report_tools.py | 6 ++--- monkey/monkey_island/cc/services/config.py | 24 +++++++++---------- .../services/telemetry/processing/exploit.py | 4 ++-- .../telemetry/processing/system_info.py | 4 ++-- .../scoutsuite/scoutsuite_auth_service.py | 4 ++-- .../test_scoutsuite_auth_service.py | 4 ++-- monkey/monkey_island/cc/test_encryptor.py | 6 ++--- 9 files changed, 28 insertions(+), 28 deletions(-) diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index 4bb704aee..3e6f43d8c 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -23,7 +23,7 @@ from monkey_island.cc.app import init_app # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 from monkey_island.cc.database import get_db_version # noqa: E402 from monkey_island.cc.database import is_db_server_up # noqa: E402 -from monkey_island.cc.encryptor import initialize_encryptor # noqa: E402 +from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 diff --git a/monkey/monkey_island/cc/server_utils/encryptor.py b/monkey/monkey_island/cc/server_utils/encryptor.py index 36fff735c..19744840d 100644 --- a/monkey/monkey_island/cc/server_utils/encryptor.py +++ b/monkey/monkey_island/cc/server_utils/encryptor.py @@ -62,5 +62,5 @@ def initialize_encryptor(data_dir): _encryptor = Encryptor(data_dir) -def encryptor(): +def get_encryptor(): return _encryptor diff --git a/monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py b/monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py index 35356cad4..6921b0129 100644 --- a/monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py +++ b/monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py @@ -1,4 +1,4 @@ -from monkey_island.cc.server_utils.encryptor import encryptor +from monkey_island.cc.server_utils.encryptor import get_encryptor def parse_creds(attempt): @@ -29,7 +29,7 @@ def censor_password(password, plain_chars=3, secret_chars=5): """ if not password: return "" - password = encryptor().dec(password) + password = get_encryptor().dec(password) return password[0:plain_chars] + '*' * secret_chars @@ -42,5 +42,5 @@ def censor_hash(hash_, plain_chars=5): """ if not hash_: return "" - hash_ = encryptor().dec(hash_) + hash_ = get_encryptor().dec(hash_) return hash_[0: plain_chars] + ' ...' diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index edf44b967..9fd8e3417 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -8,7 +8,7 @@ from jsonschema import Draft4Validator, validators import monkey_island.cc.environment.environment_singleton as env_singleton import monkey_island.cc.services.post_breach_files from monkey_island.cc.database import mongo -from monkey_island.cc.server_utils.encryptor import encryptor +from monkey_island.cc.server_utils.encryptor import get_encryptor from monkey_island.cc.services.utils.network_utils import local_ip_addresses from monkey_island.cc.services.config_schema.config_schema import SCHEMA @@ -75,9 +75,9 @@ class ConfigService: if should_decrypt: if config_key_as_arr in ENCRYPTED_CONFIG_VALUES: if isinstance(config, str): - config = encryptor().dec(config) + config = get_encryptor().dec(config) elif isinstance(config, list): - config = [encryptor().dec(x) for x in config] + config = [get_encryptor().dec(x) for x in config] return config @staticmethod @@ -112,7 +112,7 @@ class ConfigService: if item_value in items_from_config: return if should_encrypt: - item_value = encryptor().enc(item_value) + item_value = get_encryptor().enc(item_value) mongo.db.config.update( {'name': 'newconfig'}, {'$addToSet': {item_key: item_value}}, @@ -297,9 +297,9 @@ class ConfigService: if flat_config[key] and isinstance(flat_config[key][0], dict) and 'public_key' in flat_config[key][0]: flat_config[key] = [ConfigService.decrypt_ssh_key_pair(item) for item in flat_config[key]] else: - flat_config[key] = [encryptor().dec(item) for item in flat_config[key]] + flat_config[key] = [get_encryptor().dec(item) for item in flat_config[key]] else: - flat_config[key] = encryptor().dec(flat_config[key]) + flat_config[key] = get_encryptor().dec(flat_config[key]) return flat_config @staticmethod @@ -320,19 +320,19 @@ class ConfigService: config_arr[i] = ConfigService.decrypt_ssh_key_pair(config_arr[i]) if is_decrypt else \ ConfigService.decrypt_ssh_key_pair(config_arr[i], True) else: - config_arr[i] = encryptor().dec(config_arr[i]) if is_decrypt else encryptor().enc(config_arr[i]) + config_arr[i] = get_encryptor().dec(config_arr[i]) if is_decrypt else get_encryptor().enc(config_arr[i]) else: parent_config_arr[config_arr_as_array[-1]] = \ - encryptor().dec(config_arr) if is_decrypt else encryptor().enc(config_arr) + get_encryptor().dec(config_arr) if is_decrypt else get_encryptor().enc(config_arr) @staticmethod def decrypt_ssh_key_pair(pair, encrypt=False): if encrypt: - pair['public_key'] = encryptor().enc(pair['public_key']) - pair['private_key'] = encryptor().enc(pair['private_key']) + pair['public_key'] = get_encryptor().enc(pair['public_key']) + pair['private_key'] = get_encryptor().enc(pair['private_key']) else: - pair['public_key'] = encryptor().dec(pair['public_key']) - pair['private_key'] = encryptor().dec(pair['private_key']) + pair['public_key'] = get_encryptor().dec(pair['public_key']) + pair['private_key'] = get_encryptor().dec(pair['private_key']) return pair @staticmethod diff --git a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py index b2baff562..9b06b028d 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py @@ -2,7 +2,7 @@ import copy import dateutil -from monkey_island.cc.server_utils.encryptor import encryptor +from monkey_island.cc.server_utils.encryptor import get_encryptor from monkey_island.cc.models import Monkey from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.edge.displayed_edge import EdgeService @@ -66,4 +66,4 @@ def encrypt_exploit_creds(telemetry_json): for field in ['password', 'lm_hash', 'ntlm_hash']: credential = attempts[i][field] if len(credential) > 0: - attempts[i][field] = encryptor().enc(credential) + attempts[i][field] = get_encryptor().enc(credential) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index 5af4d1dde..250080697 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -1,6 +1,6 @@ import logging -from monkey_island.cc.server_utils.encryptor import encryptor +from monkey_island.cc.server_utils.encryptor import get_encryptor from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.system_info_collectors.system_info_telemetry_dispatcher import \ @@ -63,7 +63,7 @@ def encrypt_system_info_ssh_keys(ssh_info): for idx, user in enumerate(ssh_info): for field in ['public_key', 'private_key', 'known_hosts']: if ssh_info[idx][field]: - ssh_info[idx][field] = encryptor().enc(ssh_info[idx][field]) + ssh_info[idx][field] = get_encryptor().enc(ssh_info[idx][field]) def process_credential_info(telemetry_json): diff --git a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py index 2bc7b2323..1f0ee180e 100644 --- a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py +++ b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py @@ -4,7 +4,7 @@ from ScoutSuite.providers.base.authentication_strategy import AuthenticationExce from common.cloud.scoutsuite_consts import CloudProviders from common.utils.exceptions import InvalidAWSKeys -from monkey_island.cc.server_utils.encryptor import encryptor +from monkey_island.cc.server_utils.encryptor import get_encryptor from monkey_island.cc.services.config import ConfigService from common.config_value_paths import AWS_KEYS_PATH @@ -37,7 +37,7 @@ def set_aws_keys(access_key_id: str, secret_access_key: str, session_token: str) def _set_aws_key(key_type: str, key_value: str): path_to_keys = AWS_KEYS_PATH - encrypted_key = encryptor().enc(key_value) + encrypted_key = get_encryptor().enc(key_value) ConfigService.set_config_value(path_to_keys + [key_type], encrypted_key) diff --git a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/test_scoutsuite_auth_service.py b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/test_scoutsuite_auth_service.py index 72a1c711a..1ac9afdfe 100644 --- a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/test_scoutsuite_auth_service.py +++ b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/test_scoutsuite_auth_service.py @@ -4,7 +4,7 @@ import pytest import dpath.util from monkey_island.cc.database import mongo -from monkey_island.cc.server_utils.encryptor import initialize_encryptor, encryptor +from monkey_island.cc.server_utils.encryptor import initialize_encryptor, get_encryptor from monkey_island.cc.services.config import ConfigService from common.config_value_paths import AWS_KEYS_PATH from monkey_island.cc.services.zero_trust.scoutsuite.scoutsuite_auth_service import is_aws_keys_setup @@ -27,7 +27,7 @@ def test_is_aws_keys_setup(tmp_path): # Make sure noone changed config path and broke this function initialize_encryptor(tmp_path) - bogus_key_value = encryptor().enc('bogus_aws_key') + bogus_key_value = get_encryptor().enc('bogus_aws_key') dpath.util.set(ConfigService.default_config, AWS_KEYS_PATH+['aws_secret_access_key'], bogus_key_value) dpath.util.set(ConfigService.default_config, AWS_KEYS_PATH+['aws_access_key_id'], bogus_key_value) diff --git a/monkey/monkey_island/cc/test_encryptor.py b/monkey/monkey_island/cc/test_encryptor.py index 3380d1d43..630ae22ee 100644 --- a/monkey/monkey_island/cc/test_encryptor.py +++ b/monkey/monkey_island/cc/test_encryptor.py @@ -1,7 +1,7 @@ import os from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH -from monkey_island.cc.server_utils.encryptor import initialize_encryptor, encryptor +from monkey_island.cc.server_utils.encryptor import initialize_encryptor, get_encryptor TEST_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing") @@ -14,13 +14,13 @@ CYPHERTEXT = "vKgvD6SjRyIh1dh2AM/rnTa0NI/vjfwnbZLbMocWtE4e42WJmSUz2ordtbQrH1Fq" def test_aes_cbc_encryption(): initialize_encryptor(TEST_DATA_DIR) - assert encryptor().enc(PLAINTEXT) != PLAINTEXT + assert get_encryptor().enc(PLAINTEXT) != PLAINTEXT def test_aes_cbc_decryption(): initialize_encryptor(TEST_DATA_DIR) - assert encryptor().dec(CYPHERTEXT) == PLAINTEXT + assert get_encryptor().dec(CYPHERTEXT) == PLAINTEXT def test_create_new_password_file(tmpdir): From 115368f83d4ea888de35a30f7cacf624097aadb9 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 25 Feb 2021 06:41:12 -0500 Subject: [PATCH 28/52] cc: rename DATA_DIR constant in test to WITH_DATA_DIR --- .../monkey_island/cc/environment/test_environment_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/environment/test_environment_config.py b/monkey/monkey_island/cc/environment/test_environment_config.py index a8e8a1beb..6ae765d80 100644 --- a/monkey/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/monkey_island/cc/environment/test_environment_config.py @@ -23,7 +23,7 @@ PARTIAL_CREDENTIALS = os.path.join( STANDARD_WITH_CREDENTIALS = os.path.join( TEST_RESOURCES_DIR, "server_config_standard_with_credentials.json" ) -DATA_DIR = os.path.join(TEST_RESOURCES_DIR, "server_config_data_dir.json") +WITH_DATA_DIR = os.path.join(TEST_RESOURCES_DIR, "server_config_data_dir.json") @pytest.fixture @@ -122,5 +122,5 @@ def test_generate_default_file(config_file): def test_data_dir(): - environment_config = EnvironmentConfig(DATA_DIR) + environment_config = EnvironmentConfig(WITH_DATA_DIR) assert environment_config.data_dir == "/test/data/dir" From fdeec3a6343885757cdb7e58146faad9b18b1a66 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 25 Feb 2021 06:44:34 -0500 Subject: [PATCH 29/52] cc: rename data_dir parameter in encryptor to password_file_dir --- monkey/monkey_island/cc/server_utils/encryptor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/server_utils/encryptor.py b/monkey/monkey_island/cc/server_utils/encryptor.py index 19744840d..5b1328551 100644 --- a/monkey/monkey_island/cc/server_utils/encryptor.py +++ b/monkey/monkey_island/cc/server_utils/encryptor.py @@ -15,9 +15,9 @@ class Encryptor: _BLOCK_SIZE = 32 _PASSWORD_FILENAME = "mongo_key.bin" - def __init__(self, data_dir): + def __init__(self, password_file_dir): password_file = os.path.expanduser( - os.path.join(data_dir, self._PASSWORD_FILENAME) + os.path.join(password_file_dir, self._PASSWORD_FILENAME) ) if os.path.exists(password_file): @@ -56,10 +56,10 @@ class Encryptor: return self._unpad(cipher.decrypt(enc_message[AES.block_size :]).decode()) -def initialize_encryptor(data_dir): +def initialize_encryptor(password_file_dir): global _encryptor - _encryptor = Encryptor(data_dir) + _encryptor = Encryptor(password_file_dir) def get_encryptor(): From 45367bb0517366f587b69c6f4f605f0fc3bddccb Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 25 Feb 2021 06:51:03 -0500 Subject: [PATCH 30/52] cc: add encrypt/decrypt test --- monkey/monkey_island/cc/test_encryptor.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/monkey/monkey_island/cc/test_encryptor.py b/monkey/monkey_island/cc/test_encryptor.py index 630ae22ee..d147ed3f5 100644 --- a/monkey/monkey_island/cc/test_encryptor.py +++ b/monkey/monkey_island/cc/test_encryptor.py @@ -23,6 +23,12 @@ def test_aes_cbc_decryption(): assert get_encryptor().dec(CYPHERTEXT) == PLAINTEXT +def test_aes_cbc_enc_dec(): + initialize_encryptor(TEST_DATA_DIR) + + assert get_encryptor().dec(get_encryptor().enc(PLAINTEXT)) == PLAINTEXT + + def test_create_new_password_file(tmpdir): initialize_encryptor(tmpdir) From 29c9c72ef36da97a2cd23975e5b9e835b3ebb4b9 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 25 Feb 2021 07:02:00 -0500 Subject: [PATCH 31/52] cc: rename server_config_data_dir.json -> server_config_with_data_dir.json --- monkey/monkey_island/cc/environment/test_environment_config.py | 2 +- ...er_config_data_dir.json => server_config_with_data_dir.json} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename monkey/monkey_island/cc/testing/environment/{server_config_data_dir.json => server_config_with_data_dir.json} (100%) diff --git a/monkey/monkey_island/cc/environment/test_environment_config.py b/monkey/monkey_island/cc/environment/test_environment_config.py index 6ae765d80..b7712d480 100644 --- a/monkey/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/monkey_island/cc/environment/test_environment_config.py @@ -23,7 +23,7 @@ PARTIAL_CREDENTIALS = os.path.join( STANDARD_WITH_CREDENTIALS = os.path.join( TEST_RESOURCES_DIR, "server_config_standard_with_credentials.json" ) -WITH_DATA_DIR = os.path.join(TEST_RESOURCES_DIR, "server_config_data_dir.json") +WITH_DATA_DIR = os.path.join(TEST_RESOURCES_DIR, "server_config_with_data_dir.json") @pytest.fixture diff --git a/monkey/monkey_island/cc/testing/environment/server_config_data_dir.json b/monkey/monkey_island/cc/testing/environment/server_config_with_data_dir.json similarity index 100% rename from monkey/monkey_island/cc/testing/environment/server_config_data_dir.json rename to monkey/monkey_island/cc/testing/environment/server_config_with_data_dir.json From 20a3d31852c03177adc759b2f5fce7b578c6c2fd Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 16 Feb 2021 11:24:10 -0500 Subject: [PATCH 32/52] build: add data_dir to standard server config in appimage build --- deployment_scripts/appimage/build_appimage.sh | 5 +++-- deployment_scripts/appimage/server_config.json.standard | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 deployment_scripts/appimage/server_config.json.standard diff --git a/deployment_scripts/appimage/build_appimage.sh b/deployment_scripts/appimage/build_appimage.sh index d3a0fab47..b880a06d8 100755 --- a/deployment_scripts/appimage/build_appimage.sh +++ b/deployment_scripts/appimage/build_appimage.sh @@ -94,7 +94,7 @@ clone_monkey_repo() { fi log_message "Cloning files from git" - branch=${2:-"select-logger-config-at-runtime"} + branch=${2:-"data-dir"} git clone --single-branch --recurse-submodules -b "$branch" "${MONKEY_GIT_URL}" "${REPO_MONKEY_HOME}" 2>&1 || handle_error chmod 774 -R "${MONKEY_HOME}" @@ -107,10 +107,11 @@ copy_monkey_island_to_appdir() { cp -r $REPO_MONKEY_SRC/monkey_island $INSTALL_DIR cp ./run_appimage.sh $INSTALL_DIR/monkey_island/linux/ cp ./island_logger_config.json $INSTALL_DIR/ + cp ./server_config.json.standard $INSTALL_DIR/monkey_island/cc/ # TODO: This is a workaround that may be able to be removed after PR #848 is # merged. See monkey_island/cc/environment_singleton.py for more information. - cp $INSTALL_DIR/monkey_island/cc/server_config.json.standard $INSTALL_DIR/monkey_island/cc/server_config.json + cp ./server_config.json.standard $INSTALL_DIR/monkey_island/cc/server_config.json } install_monkey_island_python_dependencies() { diff --git a/deployment_scripts/appimage/server_config.json.standard b/deployment_scripts/appimage/server_config.json.standard new file mode 100644 index 000000000..99848f945 --- /dev/null +++ b/deployment_scripts/appimage/server_config.json.standard @@ -0,0 +1,5 @@ +{ + "server_config": "password", + "deployment": "standard", + "data_dir": "~/.monkey_island" +} From e1209dcb4c7c54fcb864f2652c5fadf63bdf00ab Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 16 Feb 2021 11:59:27 -0500 Subject: [PATCH 33/52] cc: add data_dir_abs_path property to EnvironmentConfig EnvironmentConfig needs to handle environment variables and '~' in its `data_dir` property. Other components that consume `data_dir` need environment variables and '~' resolved to an absolute path. Add a property called `data_dir_abs_path` that calculates the absolute path from `data_dir`. Since `data_dir` remains unchanged, the EnvironmentConfig can be saved to file without modifying the `data_dir` option in the file. --- .../cc/environment/environment_config.py | 4 ++++ .../cc/environment/test_environment_config.py | 13 ++++++++++++- monkey/monkey_island/cc/main.py | 2 +- monkey/monkey_island/cc/server_utils/encryptor.py | 4 +--- monkey/monkey_island/cc/test_encryptor.py | 8 -------- .../server_config_with_data_dir_home.json | 7 +++++++ 6 files changed, 25 insertions(+), 13 deletions(-) create mode 100644 monkey/monkey_island/cc/testing/environment/server_config_with_data_dir_home.json diff --git a/monkey/monkey_island/cc/environment/environment_config.py b/monkey/monkey_island/cc/environment/environment_config.py index 3bc9327b4..f390d8186 100644 --- a/monkey/monkey_island/cc/environment/environment_config.py +++ b/monkey/monkey_island/cc/environment/environment_config.py @@ -50,6 +50,10 @@ class EnvironmentConfig: self.aws = aws self.data_dir = data_dir + @property + def data_dir_abs_path(self): + return os.path.abspath(os.path.expanduser(os.path.expandvars(self.data_dir))) + def save_to_file(self): with open(self._server_config_path, "w") as f: f.write(json.dumps(self.to_dict(), indent=2)) diff --git a/monkey/monkey_island/cc/environment/test_environment_config.py b/monkey/monkey_island/cc/environment/test_environment_config.py index b7712d480..cee36645b 100644 --- a/monkey/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/monkey_island/cc/environment/test_environment_config.py @@ -24,7 +24,7 @@ STANDARD_WITH_CREDENTIALS = os.path.join( TEST_RESOURCES_DIR, "server_config_standard_with_credentials.json" ) WITH_DATA_DIR = os.path.join(TEST_RESOURCES_DIR, "server_config_with_data_dir.json") - +WITH_DATA_DIR_HOME = os.path.join(TEST_RESOURCES_DIR, "server_config_with_data_dir_home.json") @pytest.fixture def config_file(tmpdir): @@ -124,3 +124,14 @@ def test_generate_default_file(config_file): def test_data_dir(): environment_config = EnvironmentConfig(WITH_DATA_DIR) assert environment_config.data_dir == "/test/data/dir" + + +def set_home_env(monkeypatch, tmpdir): + monkeypatch.setenv("HOME", str(tmpdir)) + + +def test_data_dir_abs_path_from_file(monkeypatch, tmpdir): + set_home_env(monkeypatch, tmpdir) + + config = EnvironmentConfig(WITH_DATA_DIR_HOME) + assert config.data_dir_abs_path == os.path.join(tmpdir, "data_dir") diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index 3e6f43d8c..303f5997d 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -35,7 +35,7 @@ MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" def main(should_setup_only=False, server_config_filename=DEFAULT_SERVER_CONFIG_PATH): logger.info("Starting bootloader server") env_singleton.initialize_from_file(server_config_filename) - initialize_encryptor(env_singleton.config.data_dir) + initialize_encryptor(env_singleton.config.data_dir_abs_path) mongo_url = os.environ.get('MONGO_URL', env_singleton.env.get_mongo_url()) bootloader_server_thread = Thread(target=BootloaderHttpServer(mongo_url).serve_forever, daemon=True) diff --git a/monkey/monkey_island/cc/server_utils/encryptor.py b/monkey/monkey_island/cc/server_utils/encryptor.py index 5b1328551..162efd15e 100644 --- a/monkey/monkey_island/cc/server_utils/encryptor.py +++ b/monkey/monkey_island/cc/server_utils/encryptor.py @@ -16,9 +16,7 @@ class Encryptor: _PASSWORD_FILENAME = "mongo_key.bin" def __init__(self, password_file_dir): - password_file = os.path.expanduser( - os.path.join(password_file_dir, self._PASSWORD_FILENAME) - ) + password_file = os.path.join(password_file_dir, self._PASSWORD_FILENAME) if os.path.exists(password_file): self._load_existing_key(password_file) diff --git a/monkey/monkey_island/cc/test_encryptor.py b/monkey/monkey_island/cc/test_encryptor.py index d147ed3f5..b2564b16c 100644 --- a/monkey/monkey_island/cc/test_encryptor.py +++ b/monkey/monkey_island/cc/test_encryptor.py @@ -33,11 +33,3 @@ def test_create_new_password_file(tmpdir): initialize_encryptor(tmpdir) assert os.path.isfile(os.path.join(tmpdir, PASSWORD_FILENAME)) - - -def test_expand_home(monkeypatch, tmpdir): - monkeypatch.setenv("HOME", str(tmpdir)) - - initialize_encryptor("~/") - - assert os.path.isfile(os.path.join(tmpdir, "mongo_key.bin")) diff --git a/monkey/monkey_island/cc/testing/environment/server_config_with_data_dir_home.json b/monkey/monkey_island/cc/testing/environment/server_config_with_data_dir_home.json new file mode 100644 index 000000000..e6e4a0a1f --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_with_data_dir_home.json @@ -0,0 +1,7 @@ +{ + "server_config": "password", + "deployment": "develop", + "user": "test", + "password_hash": "abcdef", + "data_dir": "~/data_dir" +} From b5e8d895c80990808b793bce20a55534022c1dd4 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 16 Feb 2021 15:04:14 -0500 Subject: [PATCH 34/52] cc: use data_dir when running monkey agent locally from island --- monkey/monkey_island/cc/resources/local_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/resources/local_run.py b/monkey/monkey_island/cc/resources/local_run.py index 1a388db0a..f76548a9a 100644 --- a/monkey/monkey_island/cc/resources/local_run.py +++ b/monkey/monkey_island/cc/resources/local_run.py @@ -32,7 +32,7 @@ def run_local_monkey(): return False, "OS Type not found" monkey_path = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', 'binaries', result['filename']) - target_path = os.path.join(MONKEY_ISLAND_ABS_PATH, result['filename']) + target_path = os.path.join(env_singleton.config.data_dir_abs_path, result['filename']) # copy the executable to temp path (don't run the monkey from its current location as it may delete itself) try: From 1fad6b4666c40792b54eda7841c438786e0f195f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 16 Feb 2021 15:12:57 -0500 Subject: [PATCH 35/52] cc: remove unnecessary `config` property from environment_singleton Introduced in b0d478473fe --- monkey/monkey_island/cc/environment/environment_singleton.py | 3 --- monkey/monkey_island/cc/main.py | 2 +- monkey/monkey_island/cc/resources/local_run.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/monkey/monkey_island/cc/environment/environment_singleton.py b/monkey/monkey_island/cc/environment/environment_singleton.py index aa2dc32d6..01e83096d 100644 --- a/monkey/monkey_island/cc/environment/environment_singleton.py +++ b/monkey/monkey_island/cc/environment/environment_singleton.py @@ -20,7 +20,6 @@ ENV_DICT = { } env = None -config = None def set_env(env_type: str, env_config: EnvironmentConfig): @@ -40,8 +39,6 @@ def set_to_standard(): def initialize_from_file(file_path): - global config - try: config = EnvironmentConfig(file_path) diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index 303f5997d..211084565 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -35,7 +35,7 @@ MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" def main(should_setup_only=False, server_config_filename=DEFAULT_SERVER_CONFIG_PATH): logger.info("Starting bootloader server") env_singleton.initialize_from_file(server_config_filename) - initialize_encryptor(env_singleton.config.data_dir_abs_path) + initialize_encryptor(env_singleton.env.get_config().data_dir_abs_path) mongo_url = os.environ.get('MONGO_URL', env_singleton.env.get_mongo_url()) bootloader_server_thread = Thread(target=BootloaderHttpServer(mongo_url).serve_forever, daemon=True) diff --git a/monkey/monkey_island/cc/resources/local_run.py b/monkey/monkey_island/cc/resources/local_run.py index f76548a9a..0758d40c2 100644 --- a/monkey/monkey_island/cc/resources/local_run.py +++ b/monkey/monkey_island/cc/resources/local_run.py @@ -32,7 +32,7 @@ def run_local_monkey(): return False, "OS Type not found" monkey_path = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', 'binaries', result['filename']) - target_path = os.path.join(env_singleton.config.data_dir_abs_path, result['filename']) + target_path = os.path.join(env_singleton.env.get_config().data_dir_abs_path, result['filename']) # copy the executable to temp path (don't run the monkey from its current location as it may delete itself) try: From 921c4d01ca586751cc62f909f457081164b6ee48 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 24 Mar 2021 07:27:09 -0400 Subject: [PATCH 36/52] cc: resolve some flake8 warnings --- monkey/monkey_island.py | 4 ---- .../monkey_island/cc/environment/test_environment_config.py | 1 + monkey/monkey_island/cc/server_utils/encryptor.py | 6 +++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index e2323a7c3..1261af888 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -6,10 +6,6 @@ gevent_monkey.patch_all() import json # noqa: E402 -from monkey_island.cc.consts import ( - DEFAULT_SERVER_CONFIG_PATH, - DEFAULT_LOGGER_CONFIG_PATH, -) # noqa: E402 from monkey_island.cc.island_logger import json_setup_logging # noqa: E402 diff --git a/monkey/monkey_island/cc/environment/test_environment_config.py b/monkey/monkey_island/cc/environment/test_environment_config.py index cee36645b..de941a6f3 100644 --- a/monkey/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/monkey_island/cc/environment/test_environment_config.py @@ -26,6 +26,7 @@ STANDARD_WITH_CREDENTIALS = os.path.join( WITH_DATA_DIR = os.path.join(TEST_RESOURCES_DIR, "server_config_with_data_dir.json") WITH_DATA_DIR_HOME = os.path.join(TEST_RESOURCES_DIR, "server_config_with_data_dir_home.json") + @pytest.fixture def config_file(tmpdir): return os.path.join(tmpdir, "test_config.json") diff --git a/monkey/monkey_island/cc/server_utils/encryptor.py b/monkey/monkey_island/cc/server_utils/encryptor.py index 162efd15e..161032c52 100644 --- a/monkey/monkey_island/cc/server_utils/encryptor.py +++ b/monkey/monkey_island/cc/server_utils/encryptor.py @@ -38,7 +38,7 @@ class Encryptor: ) def _unpad(self, message: str): - return message[0 : -ord(message[len(message) - 1])] + return message[0:-ord(message[len(message) - 1])] def enc(self, message: str): cipher_iv = Random.new().read(AES.block_size) @@ -49,9 +49,9 @@ class Encryptor: def dec(self, enc_message): enc_message = base64.b64decode(enc_message) - cipher_iv = enc_message[0 : AES.block_size] + cipher_iv = enc_message[0:AES.block_size] cipher = AES.new(self._cipher_key, AES.MODE_CBC, cipher_iv) - return self._unpad(cipher.decrypt(enc_message[AES.block_size :]).decode()) + return self._unpad(cipher.decrypt(enc_message[AES.block_size:]).decode()) def initialize_encryptor(password_file_dir): From 1ac67cfe548673b8cf3dcaaadee716dbb9849e38 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 17 Feb 2021 15:38:43 -0500 Subject: [PATCH 37/52] build: create ~/.monkey_island with 0700 permissions --- deployment_scripts/appimage/run_appimage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment_scripts/appimage/run_appimage.sh b/deployment_scripts/appimage/run_appimage.sh index 17c8802b2..f33a95699 100644 --- a/deployment_scripts/appimage/run_appimage.sh +++ b/deployment_scripts/appimage/run_appimage.sh @@ -21,7 +21,7 @@ if [[ $(python3.7 --version 2>&1) == *"Python 3.7"* ]]; then python_cmd="python3.7" fi -mkdir -p $DOT_MONKEY +mkdir --mode=0700 --parents $DOT_MONKEY DB_DIR=$DOT_MONKEY/db mkdir -p $DB_DIR From eae5881b5f7c78d9cb5064a90569d8d5423b3c2f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 17 Feb 2021 15:40:33 -0500 Subject: [PATCH 38/52] build: prevent PyInstaller from being installed into the appimage --- deployment_scripts/appimage/build_appimage.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/deployment_scripts/appimage/build_appimage.sh b/deployment_scripts/appimage/build_appimage.sh index b880a06d8..a064cb739 100755 --- a/deployment_scripts/appimage/build_appimage.sh +++ b/deployment_scripts/appimage/build_appimage.sh @@ -58,7 +58,7 @@ install_build_prereqs() { sudo apt install -y python3 python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace #monkey island prereqs - sudo apt install -y curl libcurl4 python3.7 python3.7-dev openssl git build-essential + sudo apt install -y curl libcurl4 python3.7 python3.7-dev openssl git build-essential moreutils install_pip_37 install_nodejs } @@ -94,7 +94,7 @@ clone_monkey_repo() { fi log_message "Cloning files from git" - branch=${2:-"data-dir"} + branch=${2:-"local-run-data-dir"} git clone --single-branch --recurse-submodules -b "$branch" "${MONKEY_GIT_URL}" "${REPO_MONKEY_HOME}" 2>&1 || handle_error chmod 774 -R "${MONKEY_HOME}" @@ -116,9 +116,15 @@ copy_monkey_island_to_appdir() { install_monkey_island_python_dependencies() { log_message "Installing island requirements" + requirements_island="$ISLAND_PATH/requirements.txt" + # TODO: This is an ugly hack. PyInstaller is a build-time dependency and should + # not be installed as a runtime requirement. + sed '4d' $requirements_island | sponge $requirements_island + ${python_cmd} -m pip install -r "${requirements_island}" --ignore-installed --prefix /usr --root=$APPDIR || handle_error ${python_cmd} -m pip install pyjwt==1.7 --ignore-installed -U --prefix /usr --root=$APPDIR || handle_error + #${python_cmd} -m pip install PyNacl --user --upgrade || handle_error } From 3c113f7a89743edccfff2388a67db6f0ac46bf9d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 18 Feb 2021 11:33:59 -0500 Subject: [PATCH 39/52] build: work around limitations in appimage-builder See https://github.com/AppImageCrafters/appimage-builder/issues/93 for more information. --- deployment_scripts/appimage/build_appimage.sh | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/deployment_scripts/appimage/build_appimage.sh b/deployment_scripts/appimage/build_appimage.sh index a064cb739..a88e814ed 100755 --- a/deployment_scripts/appimage/build_appimage.sh +++ b/deployment_scripts/appimage/build_appimage.sh @@ -169,6 +169,26 @@ build_frontend() { popd || handle_error } +build_appimage() { + log_message "Building AppImage" + appimage-builder --recipe monkey_island_builder.yml --log DEBUG --skip-appimage + + # There is a bug or unwanted behavior in appimage-builder that causes issues + # if 32-bit binaries are present in the appimage. To work around this, we: + # 1. Build the AppDir with appimage-builder and skip building the appimage + # 2. Add the 32-bit binaries to the AppDir + # 3. Build the AppImage with appimage-builder from the already-built AppDir + # + # Note that appimage-builder replaces the interpreter on the monkey agent binaries + # when building the AppDir. This is unwanted as the monkey agents may execute in + # environments where the AppImage isn't loaded. + # + # See https://github.com/AppImageCrafters/appimage-builder/issues/93 for more info. + download_monkey_agent_binaries + + appimage-builder --recipe monkey_island_builder.yml --log DEBUG --skip-build +} + download_monkey_helper_binaries() { # Making dir for binaries mkdir "${MONKEY_BIN_DIR}" @@ -228,7 +248,6 @@ install_monkey_island_python_dependencies #requirements_monkey="$INFECTION_MONKEY_DIR/requirements.txt" #${python_cmd} -m pip install -r "${requirements_monkey}" --user --upgrade || handle_error -download_monkey_agent_binaries install_mongodb @@ -242,9 +261,7 @@ mkdir -p $APPDIR/usr/share/icons cp $REPO_MONKEY_SRC/monkey_island/cc/ui/src/images/monkey-icon.svg $APPDIR/usr/share/icons/monkey-icon.svg #cp ./monkey_island.desktop $APPDIR -log_message "Building AppImage" -appimage-builder --recipe monkey_island_builder.yml - +build_appimage log_message "Deployment script finished." exit 0 From a97bd1935847e562b1fe20035e3e4e7e36adda1c Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 18 Feb 2021 11:36:49 -0500 Subject: [PATCH 40/52] build: remove unnecessary comments from appimage build --- deployment_scripts/appimage/build_appimage.sh | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/deployment_scripts/appimage/build_appimage.sh b/deployment_scripts/appimage/build_appimage.sh index a88e814ed..c81281242 100755 --- a/deployment_scripts/appimage/build_appimage.sh +++ b/deployment_scripts/appimage/build_appimage.sh @@ -124,8 +124,6 @@ install_monkey_island_python_dependencies() { ${python_cmd} -m pip install -r "${requirements_island}" --ignore-installed --prefix /usr --root=$APPDIR || handle_error ${python_cmd} -m pip install pyjwt==1.7 --ignore-installed -U --prefix /usr --root=$APPDIR || handle_error - - #${python_cmd} -m pip install PyNacl --user --upgrade || handle_error } download_monkey_agent_binaries() { @@ -142,9 +140,6 @@ log_message "Downloading monkey agent binaries to ${ISLAND_BINARIES_PATH}" } install_mongodb() { - #log_message "Installing libcurl4" - #sudo apt-get install -y libcurl4 - log_message "Installing MongoDB" mkdir -p $MONGO_PATH @@ -152,7 +147,6 @@ install_mongodb() { } generate_ssl_cert() { - # Generate SSL certificate log_message "Generating certificate" chmod u+x "${ISLAND_PATH}"/linux/create_certificate.sh @@ -235,31 +229,21 @@ clone_monkey_repo download_monkey_helper_binaries copy_monkey_island_to_appdir + # Create folders log_message "Creating island dirs under $ISLAND_PATH" mkdir -p "${MONGO_PATH}" || handle_error -#log_message "Installing python3-distutils" -#sudo apt-get install -y python3-distutils install_monkey_island_python_dependencies -#log_message "Installing monkey requirements" -#sudo apt-get install -y libffi-dev upx libssl-dev libc++1 -#requirements_monkey="$INFECTION_MONKEY_DIR/requirements.txt" -#${python_cmd} -m pip install -r "${requirements_monkey}" --user --upgrade || handle_error - - install_mongodb generate_ssl_cert build_frontend -#cp ./AppRun $APPDIR - mkdir -p $APPDIR/usr/share/icons cp $REPO_MONKEY_SRC/monkey_island/cc/ui/src/images/monkey-icon.svg $APPDIR/usr/share/icons/monkey-icon.svg -#cp ./monkey_island.desktop $APPDIR build_appimage From 7910d9be52a9a93dbb804ca5d465c732322c050a Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 18 Feb 2021 12:23:13 -0500 Subject: [PATCH 41/52] build: install nodejs 12 in appimage --- deployment_scripts/appimage/monkey_island_builder.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deployment_scripts/appimage/monkey_island_builder.yml b/deployment_scripts/appimage/monkey_island_builder.yml index 6b77c7437..faa3c2876 100644 --- a/deployment_scripts/appimage/monkey_island_builder.yml +++ b/deployment_scripts/appimage/monkey_island_builder.yml @@ -22,6 +22,9 @@ AppDir: - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-security universe - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-updates universe + - sourceline: deb [arch=amd64] http://deb.nodesource.com/node_12.x bionic main + key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x1655A0AB68576280 + include: - libc6 From 5e56257051c91e94ca2acd4192409e8432561333 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 18 Feb 2021 14:49:08 -0500 Subject: [PATCH 42/52] build: do not pull agent helper binaries during appimage build --- deployment_scripts/appimage/build_appimage.sh | 27 +------------------ 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/deployment_scripts/appimage/build_appimage.sh b/deployment_scripts/appimage/build_appimage.sh index c81281242..e611a3778 100755 --- a/deployment_scripts/appimage/build_appimage.sh +++ b/deployment_scripts/appimage/build_appimage.sh @@ -12,8 +12,6 @@ REPO_MONKEY_SRC=$REPO_MONKEY_HOME/monkey ISLAND_PATH="$INSTALL_DIR/monkey_island" MONGO_PATH="$ISLAND_PATH/bin/mongodb" ISLAND_BINARIES_PATH="$ISLAND_PATH/cc/binaries" -INFECTION_MONKEY_DIR="$MONKEY_HOME/monkey/infection_monkey" -MONKEY_BIN_DIR="$INFECTION_MONKEY_DIR/bin" is_root() { return $(id -u) @@ -169,7 +167,7 @@ build_appimage() { # There is a bug or unwanted behavior in appimage-builder that causes issues # if 32-bit binaries are present in the appimage. To work around this, we: - # 1. Build the AppDir with appimage-builder and skip building the appimage + # 1. Build the AppDir with appimage-builder and skip building the appimage # 2. Add the 32-bit binaries to the AppDir # 3. Build the AppImage with appimage-builder from the already-built AppDir # @@ -183,28 +181,6 @@ build_appimage() { appimage-builder --recipe monkey_island_builder.yml --log DEBUG --skip-build } -download_monkey_helper_binaries() { - # Making dir for binaries - mkdir "${MONKEY_BIN_DIR}" - - download_sambacry_binaries - download_tracerout_binaries -} - -download_sambacry_binaries() { - # Download sambacry binaries - log_message "Downloading sambacry binaries" - curl -L -o ${MONKEY_BIN_DIR}/sc_monkey_runner64.so ${SAMBACRY_64_BINARY_URL} - curl -L -o ${MONKEY_BIN_DIR}/sc_monkey_runner32.so ${SAMBACRY_32_BINARY_URL} -} - -download_tracerout_binaries() { - # Download traceroute binaries - log_message "Downloading traceroute binaries" - curl -L -o ${MONKEY_BIN_DIR}/traceroute64 ${TRACEROUTE_64_BINARY_URL} - curl -L -o ${MONKEY_BIN_DIR}/traceroute32 ${TRACEROUTE_32_BINARY_URL} -} - if is_root; then log_message "Please don't run this script as root" exit 1 @@ -226,7 +202,6 @@ install_appimage_builder load_monkey_binary_config clone_monkey_repo -download_monkey_helper_binaries copy_monkey_island_to_appdir From 412aa2ba350665ca09661ae2bdaea6e50646eb4f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 19 Feb 2021 10:07:46 -0500 Subject: [PATCH 43/52] build: remove unnecessary includes from monkey_island_builder.yml --- deployment_scripts/appimage/monkey_island_builder.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/deployment_scripts/appimage/monkey_island_builder.yml b/deployment_scripts/appimage/monkey_island_builder.yml index faa3c2876..2c85c41d3 100644 --- a/deployment_scripts/appimage/monkey_island_builder.yml +++ b/deployment_scripts/appimage/monkey_island_builder.yml @@ -22,18 +22,11 @@ AppDir: - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-security universe - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-updates universe - - sourceline: deb [arch=amd64] http://deb.nodesource.com/node_12.x bionic main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x1655A0AB68576280 include: - - libc6 - bash - - curl - - libcurl4 - - coreutils - python3.7 - - nodejs runtime: env: From b0af8b1b978ab4c3f859406fd71fc5f5028324b1 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 25 Mar 2021 08:45:08 -0400 Subject: [PATCH 44/52] build: create appimage-comaptible server_config.json on start --- deployment_scripts/appimage/run_appimage.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deployment_scripts/appimage/run_appimage.sh b/deployment_scripts/appimage/run_appimage.sh index f33a95699..7738301cd 100644 --- a/deployment_scripts/appimage/run_appimage.sh +++ b/deployment_scripts/appimage/run_appimage.sh @@ -8,6 +8,13 @@ configure_default_logging() { fi } +configure_default_server() { + if [ ! -f $DOT_MONKEY/server_config.json ]; then + cp $APPDIR/usr/src/monkey_island/cc/server_config.json.standard $DOT_MONKEY/server_config.json + fi +} + + # Detecting command that calls python 3.7 python_cmd="" @@ -27,6 +34,7 @@ DB_DIR=$DOT_MONKEY/db mkdir -p $DB_DIR configure_default_logging +configure_default_server cd $APPDIR/usr/src ./monkey_island/bin/mongodb/bin/mongod --dbpath $DB_DIR & From 2c75eab4675ddf34774f55549363d1a7a645b85e Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 31 Mar 2021 11:30:55 -0400 Subject: [PATCH 45/52] build: remove separate pyjwt `pip install` This line was a workaround for an issue resolved by ed589bd. --- deployment_scripts/appimage/build_appimage.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/deployment_scripts/appimage/build_appimage.sh b/deployment_scripts/appimage/build_appimage.sh index e611a3778..ceda1dc63 100755 --- a/deployment_scripts/appimage/build_appimage.sh +++ b/deployment_scripts/appimage/build_appimage.sh @@ -121,7 +121,6 @@ install_monkey_island_python_dependencies() { sed '4d' $requirements_island | sponge $requirements_island ${python_cmd} -m pip install -r "${requirements_island}" --ignore-installed --prefix /usr --root=$APPDIR || handle_error - ${python_cmd} -m pip install pyjwt==1.7 --ignore-installed -U --prefix /usr --root=$APPDIR || handle_error } download_monkey_agent_binaries() { From 3d938f253627ee5fd6764b422481e90e97c03051 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 31 Mar 2021 11:55:21 -0400 Subject: [PATCH 46/52] cc: Fix come incorrect import paths --- monkey/monkey_island.py | 2 +- monkey/monkey_island/cc/arg_parser.py | 2 +- monkey/monkey_island/cc/environment/set_server_config.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 1261af888..88e41b6e2 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -6,7 +6,7 @@ gevent_monkey.patch_all() import json # noqa: E402 -from monkey_island.cc.island_logger import json_setup_logging # noqa: E402 +from monkey_island.cc.server_utils.island_logger import json_setup_logging # noqa: E402 if "__main__" == __name__: diff --git a/monkey/monkey_island/cc/arg_parser.py b/monkey/monkey_island/cc/arg_parser.py index 8716a9b4a..5ea12aec4 100644 --- a/monkey/monkey_island/cc/arg_parser.py +++ b/monkey/monkey_island/cc/arg_parser.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from monkey_island.cc.consts import DEFAULT_SERVER_CONFIG_PATH, DEFAULT_LOGGER_CONFIG_PATH +from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH, DEFAULT_LOGGER_CONFIG_PATH @dataclass diff --git a/monkey/monkey_island/cc/environment/set_server_config.py b/monkey/monkey_island/cc/environment/set_server_config.py index 2bb4336ae..f3fbd66ff 100644 --- a/monkey/monkey_island/cc/environment/set_server_config.py +++ b/monkey/monkey_island/cc/environment/set_server_config.py @@ -14,7 +14,7 @@ def add_monkey_dir_to_sys_path(): add_monkey_dir_to_sys_path() -from monkey_island.cc.consts import DEFAULT_SERVER_CONFIG_PATH # noqa: E402 isort:skip +from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH # noqa: E402 isort:skip SERVER_CONFIG = "server_config" BACKUP_CONFIG_FILENAME = "./server_config.backup" From 05a368e53456111402472ca4f4fdadf283d2f873 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 31 Mar 2021 12:07:20 -0400 Subject: [PATCH 47/52] Update CHANGELOG.md to include AppImage changes --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcec5d8e9..264e65efe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,3 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Added +- A runtime-configurable option to specify a data directory where runtime + configuration and other artifacts can be stored. #994 +- Scripts to build a prototype AppImage for Monkey Island. #1069 + +### Changed +- server_config.json can be selected at runtime. #963 +- Logger configuration can be selected at runtime. #971 +- `mongo_key.bin` file location can be selected at runtime. #994 +- Monkey agents are stored in the configurable data_dir when monkey is "run + from the island". #997 From 8278e0eb6b388058fd0801505a137c916d14fcff Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 1 Apr 2021 12:25:06 -0400 Subject: [PATCH 48/52] build: improve appimage build script quality by addressing review comments --- deployment_scripts/appimage/AppRun | 5 -- deployment_scripts/appimage/build_appimage.sh | 50 +++++++++++-------- 2 files changed, 28 insertions(+), 27 deletions(-) delete mode 100644 deployment_scripts/appimage/AppRun diff --git a/deployment_scripts/appimage/AppRun b/deployment_scripts/appimage/AppRun deleted file mode 100644 index 29d0197c4..000000000 --- a/deployment_scripts/appimage/AppRun +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -cd ./usr/src - -bash monkey_island/linux/run.sh $HOME/.monkey_island/db diff --git a/deployment_scripts/appimage/build_appimage.sh b/deployment_scripts/appimage/build_appimage.sh index ceda1dc63..50173794e 100755 --- a/deployment_scripts/appimage/build_appimage.sh +++ b/deployment_scripts/appimage/build_appimage.sh @@ -45,13 +45,17 @@ install_pip_37() { } install_nodejs() { + NODE_SRC=https://deb.nodesource.com/setup_12.x + log_message "Installing nodejs" - node_src=https://deb.nodesource.com/setup_12.x - curl -sL $node_src | sudo -E bash - + + curl -sL $NODE_SRC | sudo -E bash - sudo apt-get install -y nodejs } install_build_prereqs() { + sudo apt update + # appimage-builder prereqs sudo apt install -y python3 python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace @@ -69,8 +73,10 @@ install_appimage_builder() { install_appimage_tool() { APP_TOOL_BIN=$HOME/bin/appimagetool + APP_TOOL_URL=https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage + mkdir $HOME/bin - curl -L -o $APP_TOOL_BIN https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage + curl -L -o $APP_TOOL_BIN $APP_TOOL_URL chmod u+x $APP_TOOL_BIN PATH=$PATH:$HOME/bin @@ -92,7 +98,7 @@ clone_monkey_repo() { fi log_message "Cloning files from git" - branch=${2:-"local-run-data-dir"} + branch=${2:-"develop"} git clone --single-branch --recurse-submodules -b "$branch" "${MONKEY_GIT_URL}" "${REPO_MONKEY_HOME}" 2>&1 || handle_error chmod 774 -R "${MONKEY_HOME}" @@ -116,9 +122,9 @@ install_monkey_island_python_dependencies() { log_message "Installing island requirements" requirements_island="$ISLAND_PATH/requirements.txt" - # TODO: This is an ugly hack. PyInstaller is a build-time dependency and should - # not be installed as a runtime requirement. - sed '4d' $requirements_island | sponge $requirements_island + # TODO: This is an ugly hack. PyInstaller and VirtualEnv are build-time + # dependencies and should not be installed as a runtime requirement. + cat $requirements_island | grep -Piv "virtualenv|pyinstaller" | sponge $requirements_island ${python_cmd} -m pip install -r "${requirements_island}" --ignore-installed --prefix /usr --root=$APPDIR || handle_error } @@ -161,23 +167,23 @@ build_frontend() { } build_appimage() { - log_message "Building AppImage" - appimage-builder --recipe monkey_island_builder.yml --log DEBUG --skip-appimage + log_message "Building AppImage" + appimage-builder --recipe monkey_island_builder.yml --log DEBUG --skip-appimage - # There is a bug or unwanted behavior in appimage-builder that causes issues - # if 32-bit binaries are present in the appimage. To work around this, we: - # 1. Build the AppDir with appimage-builder and skip building the appimage - # 2. Add the 32-bit binaries to the AppDir - # 3. Build the AppImage with appimage-builder from the already-built AppDir - # - # Note that appimage-builder replaces the interpreter on the monkey agent binaries - # when building the AppDir. This is unwanted as the monkey agents may execute in - # environments where the AppImage isn't loaded. - # - # See https://github.com/AppImageCrafters/appimage-builder/issues/93 for more info. - download_monkey_agent_binaries + # There is a bug or unwanted behavior in appimage-builder that causes issues + # if 32-bit binaries are present in the appimage. To work around this, we: + # 1. Build the AppDir with appimage-builder and skip building the appimage + # 2. Add the 32-bit binaries to the AppDir + # 3. Build the AppImage with appimage-builder from the already-built AppDir + # + # Note that appimage-builder replaces the interpreter on the monkey agent binaries + # when building the AppDir. This is unwanted as the monkey agents may execute in + # environments where the AppImage isn't loaded. + # + # See https://github.com/AppImageCrafters/appimage-builder/issues/93 for more info. + download_monkey_agent_binaries - appimage-builder --recipe monkey_island_builder.yml --log DEBUG --skip-build + appimage-builder --recipe monkey_island_builder.yml --log DEBUG --skip-build } if is_root; then From 5b1296e05d33b5264b02129efea1c6c37f43012f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 1 Apr 2021 12:25:34 -0400 Subject: [PATCH 49/52] build: Add README with instructions for appimage builder --- deployment_scripts/appimage/README.md | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 deployment_scripts/appimage/README.md diff --git a/deployment_scripts/appimage/README.md b/deployment_scripts/appimage/README.md new file mode 100644 index 000000000..a17be007e --- /dev/null +++ b/deployment_scripts/appimage/README.md @@ -0,0 +1,37 @@ +# Monkey Island AppImage + +## About + +This directory contains the necessary artifacts for building a prototype +monkey_island AppImage using appimage-builder. + +## Building an AppImage + +1. Create a clean VM or LXC (not docker!) based on Ubuntu 18.04. +1. Update the OS with `sudo apt update && sudo apt upgrade` +1. Copy the `deployment_scripts/appimage` directory to `$HOME/` in the VM. +1. Run `sudo -v`. +1. On the VM, `cd $HOME/appimage` +1. Execute `./build_appimage.sh`. This will pull all necessary dependencies + and build the AppImage. + +NOTE: This script is intended to be run from a clean VM. You can also manually +remove build artifacts by removing the following files and directories. + +- $HOME/.monkey_island (optional) +- $HOME/monkey-appdir +- $HOME/git/monkey +- $HOME/appimage/appimage-builder-cache +- $HOME/appimage/"Monkey\ Island-\*-x86-64.Appimage" + +After removing the above files and directories, you can again execute `bash +build_appimage.sh`. + +## Running the AppImage + +The build script will produce an AppImage executible named something like +`Monkey Island-VERSION-x86-64.AppImage`. Simply execute this file and you're +off to the races. + +A new directory, `$HOME/.monkey_island` will be created to store runtime +artifacts. From bbe075bca553d46650db68e6cf04ac2bbd753376 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 1 Apr 2021 12:28:39 -0400 Subject: [PATCH 50/52] build: remove unused monkey_island.desktop --- deployment_scripts/appimage/monkey_island.desktop | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 deployment_scripts/appimage/monkey_island.desktop diff --git a/deployment_scripts/appimage/monkey_island.desktop b/deployment_scripts/appimage/monkey_island.desktop deleted file mode 100644 index bfa2b8955..000000000 --- a/deployment_scripts/appimage/monkey_island.desktop +++ /dev/null @@ -1,10 +0,0 @@ -[Desktop Entry] -Version=1.10 -Type=Application -Name=Monkey Island -Comment=FILL ME IN -TryExec=fooview -Exec=AppRun -Icon=monkey_island.svg -Terminal=true; -Category=Security; From ed3d55c8aa90ff5d0896ebf2e0021f442be0fc1f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Sun, 4 Apr 2021 21:08:48 -0400 Subject: [PATCH 51/52] build: run `apt upgrade` in appimage script --- deployment_scripts/appimage/README.md | 1 - deployment_scripts/appimage/build_appimage.sh | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment_scripts/appimage/README.md b/deployment_scripts/appimage/README.md index a17be007e..37321378f 100644 --- a/deployment_scripts/appimage/README.md +++ b/deployment_scripts/appimage/README.md @@ -8,7 +8,6 @@ monkey_island AppImage using appimage-builder. ## Building an AppImage 1. Create a clean VM or LXC (not docker!) based on Ubuntu 18.04. -1. Update the OS with `sudo apt update && sudo apt upgrade` 1. Copy the `deployment_scripts/appimage` directory to `$HOME/` in the VM. 1. Run `sudo -v`. 1. On the VM, `cd $HOME/appimage` diff --git a/deployment_scripts/appimage/build_appimage.sh b/deployment_scripts/appimage/build_appimage.sh index 50173794e..3f5398d35 100755 --- a/deployment_scripts/appimage/build_appimage.sh +++ b/deployment_scripts/appimage/build_appimage.sh @@ -55,6 +55,7 @@ install_nodejs() { install_build_prereqs() { sudo apt update + sudo apt upgrade # appimage-builder prereqs sudo apt install -y python3 python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace From f7cc01811c1fdcc9e15b979c2e78197f06afab45 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Sun, 4 Apr 2021 21:23:35 -0400 Subject: [PATCH 52/52] build: address shellcheck findings in build_appimage.sh --- deployment_scripts/appimage/build_appimage.sh | 59 ++++++++++--------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/deployment_scripts/appimage/build_appimage.sh b/deployment_scripts/appimage/build_appimage.sh index 3f5398d35..c85f6f81c 100755 --- a/deployment_scripts/appimage/build_appimage.sh +++ b/deployment_scripts/appimage/build_appimage.sh @@ -14,12 +14,13 @@ MONGO_PATH="$ISLAND_PATH/bin/mongodb" ISLAND_BINARIES_PATH="$ISLAND_PATH/cc/binaries" is_root() { - return $(id -u) + return "$(id -u)" } has_sudo() { # 0 true, 1 false - return $(sudo -nv > /dev/null 2>&1) + sudo -nv > /dev/null 2>&1 + return $? } handle_error() { @@ -33,8 +34,8 @@ log_message() { } setup_appdir() { - rm -rf $APPDIR | true - mkdir -p $INSTALL_DIR + rm -rf "$APPDIR" || true + mkdir -p "$INSTALL_DIR" } install_pip_37() { @@ -76,9 +77,9 @@ install_appimage_tool() { APP_TOOL_BIN=$HOME/bin/appimagetool APP_TOOL_URL=https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage - mkdir $HOME/bin - curl -L -o $APP_TOOL_BIN $APP_TOOL_URL - chmod u+x $APP_TOOL_BIN + mkdir "$HOME"/bin + curl -L -o "$APP_TOOL_BIN" "$APP_TOOL_URL" + chmod u+x "$APP_TOOL_BIN" PATH=$PATH:$HOME/bin } @@ -87,10 +88,10 @@ load_monkey_binary_config() { tmpfile=$(mktemp) log_message "downloading configuration" - curl -L -s -o $tmpfile "$config_url" + curl -L -s -o "$tmpfile" "$config_url" log_message "loading configuration" - source $tmpfile + source "$tmpfile" } clone_monkey_repo() { @@ -106,17 +107,17 @@ clone_monkey_repo() { } copy_monkey_island_to_appdir() { - cp $REPO_MONKEY_SRC/__init__.py $INSTALL_DIR - cp $REPO_MONKEY_SRC/monkey_island.py $INSTALL_DIR - cp -r $REPO_MONKEY_SRC/common $INSTALL_DIR - cp -r $REPO_MONKEY_SRC/monkey_island $INSTALL_DIR - cp ./run_appimage.sh $INSTALL_DIR/monkey_island/linux/ - cp ./island_logger_config.json $INSTALL_DIR/ - cp ./server_config.json.standard $INSTALL_DIR/monkey_island/cc/ + cp "$REPO_MONKEY_SRC"/__init__.py "$INSTALL_DIR" + cp "$REPO_MONKEY_SRC"/monkey_island.py "$INSTALL_DIR" + cp -r "$REPO_MONKEY_SRC"/common "$INSTALL_DIR" + cp -r "$REPO_MONKEY_SRC"/monkey_island "$INSTALL_DIR" + cp ./run_appimage.sh "$INSTALL_DIR"/monkey_island/linux/ + cp ./island_logger_config.json "$INSTALL_DIR"/ + cp ./server_config.json.standard "$INSTALL_DIR"/monkey_island/cc/ # TODO: This is a workaround that may be able to be removed after PR #848 is # merged. See monkey_island/cc/environment_singleton.py for more information. - cp ./server_config.json.standard $INSTALL_DIR/monkey_island/cc/server_config.json + cp ./server_config.json.standard "$INSTALL_DIR"/monkey_island/cc/server_config.json } install_monkey_island_python_dependencies() { @@ -125,18 +126,18 @@ install_monkey_island_python_dependencies() { requirements_island="$ISLAND_PATH/requirements.txt" # TODO: This is an ugly hack. PyInstaller and VirtualEnv are build-time # dependencies and should not be installed as a runtime requirement. - cat $requirements_island | grep -Piv "virtualenv|pyinstaller" | sponge $requirements_island + cat "$requirements_island" | grep -Piv "virtualenv|pyinstaller" | sponge "$requirements_island" - ${python_cmd} -m pip install -r "${requirements_island}" --ignore-installed --prefix /usr --root=$APPDIR || handle_error + ${python_cmd} -m pip install -r "${requirements_island}" --ignore-installed --prefix /usr --root="$APPDIR" || handle_error } download_monkey_agent_binaries() { log_message "Downloading monkey agent binaries to ${ISLAND_BINARIES_PATH}" mkdir -p "${ISLAND_BINARIES_PATH}" || handle_error - curl -L -o ${ISLAND_BINARIES_PATH}/${LINUX_32_BINARY_NAME} ${LINUX_32_BINARY_URL} - curl -L -o ${ISLAND_BINARIES_PATH}/${LINUX_64_BINARY_NAME} ${LINUX_64_BINARY_URL} - curl -L -o ${ISLAND_BINARIES_PATH}/${WINDOWS_32_BINARY_NAME} ${WINDOWS_32_BINARY_URL} - curl -L -o ${ISLAND_BINARIES_PATH}/${WINDOWS_64_BINARY_NAME} ${WINDOWS_64_BINARY_URL} + curl -L -o "${ISLAND_BINARIES_PATH}/${LINUX_32_BINARY_NAME}" "${LINUX_32_BINARY_URL}" + curl -L -o "${ISLAND_BINARIES_PATH}/${LINUX_64_BINARY_NAME}" "${LINUX_64_BINARY_URL}" + curl -L -o "${ISLAND_BINARIES_PATH}/${WINDOWS_32_BINARY_NAME}" "${WINDOWS_32_BINARY_URL}" + curl -L -o "${ISLAND_BINARIES_PATH}/${WINDOWS_64_BINARY_NAME}" "${WINDOWS_64_BINARY_URL}" # Allow them to be executed chmod a+x "$ISLAND_BINARIES_PATH/$LINUX_32_BINARY_NAME" @@ -146,15 +147,15 @@ log_message "Downloading monkey agent binaries to ${ISLAND_BINARIES_PATH}" install_mongodb() { log_message "Installing MongoDB" - mkdir -p $MONGO_PATH - "${ISLAND_PATH}"/linux/install_mongo.sh ${MONGO_PATH} || handle_error + mkdir -p "$MONGO_PATH" + "${ISLAND_PATH}"/linux/install_mongo.sh "${MONGO_PATH}" || handle_error } generate_ssl_cert() { log_message "Generating certificate" chmod u+x "${ISLAND_PATH}"/linux/create_certificate.sh - "${ISLAND_PATH}"/linux/create_certificate.sh ${ISLAND_PATH}/cc + "${ISLAND_PATH}"/linux/create_certificate.sh "${ISLAND_PATH}"/cc } build_frontend() { @@ -207,7 +208,7 @@ install_appimage_builder load_monkey_binary_config -clone_monkey_repo +clone_monkey_repo "$@" copy_monkey_island_to_appdir @@ -223,8 +224,8 @@ generate_ssl_cert build_frontend -mkdir -p $APPDIR/usr/share/icons -cp $REPO_MONKEY_SRC/monkey_island/cc/ui/src/images/monkey-icon.svg $APPDIR/usr/share/icons/monkey-icon.svg +mkdir -p "$APPDIR"/usr/share/icons +cp "$REPO_MONKEY_SRC"/monkey_island/cc/ui/src/images/monkey-icon.svg "$APPDIR"/usr/share/icons/monkey-icon.svg build_appimage