diff --git a/.travis.yml b/.travis.yml index 8ac8db204..509da86ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ os: linux before_install: # Init server_config.json to default -- cp monkey/monkey_island/cc/server_config.json.default monkey/monkey_island/cc/server_config.json +- cp monkey/monkey_island/cc/server_config.json.develop monkey/monkey_island/cc/server_config.json install: # Python diff --git a/CHANGELOG.md b/CHANGELOG.md index c1ad6af12..d44e13480 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,3 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - PostgreSQL fingerprinter. #892 +- A runtime-configurable option to specify a data directory where runtime + configuration and other artifacts can be stored. #994 +- Scripts to build a prototype AppImage for Monkey Island. #1069 + +### Changed +- server_config.json can be selected at runtime. #963 +- Logger configuration can be selected at runtime. #971 +- `mongo_key.bin` file location can be selected at runtime. #994 +- Monkey agents are stored in the configurable data_dir when monkey is "run + from the island". #997 diff --git a/deployment_scripts/appimage/README.md b/deployment_scripts/appimage/README.md new file mode 100644 index 000000000..37321378f --- /dev/null +++ b/deployment_scripts/appimage/README.md @@ -0,0 +1,36 @@ +# Monkey Island AppImage + +## About + +This directory contains the necessary artifacts for building a prototype +monkey_island AppImage using appimage-builder. + +## Building an AppImage + +1. Create a clean VM or LXC (not docker!) based on Ubuntu 18.04. +1. Copy the `deployment_scripts/appimage` directory to `$HOME/` in the VM. +1. Run `sudo -v`. +1. On the VM, `cd $HOME/appimage` +1. Execute `./build_appimage.sh`. This will pull all necessary dependencies + and build the AppImage. + +NOTE: This script is intended to be run from a clean VM. You can also manually +remove build artifacts by removing the following files and directories. + +- $HOME/.monkey_island (optional) +- $HOME/monkey-appdir +- $HOME/git/monkey +- $HOME/appimage/appimage-builder-cache +- $HOME/appimage/"Monkey\ Island-\*-x86-64.Appimage" + +After removing the above files and directories, you can again execute `bash +build_appimage.sh`. + +## Running the AppImage + +The build script will produce an AppImage executible named something like +`Monkey Island-VERSION-x86-64.AppImage`. Simply execute this file and you're +off to the races. + +A new directory, `$HOME/.monkey_island` will be created to store runtime +artifacts. diff --git a/deployment_scripts/appimage/build_appimage.sh b/deployment_scripts/appimage/build_appimage.sh new file mode 100755 index 000000000..c85f6f81c --- /dev/null +++ b/deployment_scripts/appimage/build_appimage.sh @@ -0,0 +1,233 @@ +#!/bin/bash + +python_cmd="python3.7" +APPDIR="$HOME/monkey-appdir" +INSTALL_DIR="$APPDIR/usr/src" + +GIT=$HOME/git + +REPO_MONKEY_HOME=$GIT/monkey +REPO_MONKEY_SRC=$REPO_MONKEY_HOME/monkey + +ISLAND_PATH="$INSTALL_DIR/monkey_island" +MONGO_PATH="$ISLAND_PATH/bin/mongodb" +ISLAND_BINARIES_PATH="$ISLAND_PATH/cc/binaries" + +is_root() { + return "$(id -u)" +} + +has_sudo() { + # 0 true, 1 false + sudo -nv > /dev/null 2>&1 + return $? +} + +handle_error() { + echo "Fix the errors above and rerun the script" + exit 1 +} + +log_message() { + echo -e "\n\n" + echo -e "DEPLOYMENT SCRIPT: $1" +} + +setup_appdir() { + rm -rf "$APPDIR" || true + mkdir -p "$INSTALL_DIR" +} + +install_pip_37() { + pip_url=https://bootstrap.pypa.io/get-pip.py + curl $pip_url -o get-pip.py + ${python_cmd} get-pip.py + rm get-pip.py +} + +install_nodejs() { + NODE_SRC=https://deb.nodesource.com/setup_12.x + + log_message "Installing nodejs" + + curl -sL $NODE_SRC | sudo -E bash - + sudo apt-get install -y nodejs +} + +install_build_prereqs() { + sudo apt update + sudo apt upgrade + + # appimage-builder prereqs + sudo apt install -y python3 python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace + + #monkey island prereqs + sudo apt install -y curl libcurl4 python3.7 python3.7-dev openssl git build-essential moreutils + install_pip_37 + install_nodejs +} + +install_appimage_builder() { + sudo pip3 install appimage-builder + + install_appimage_tool +} + +install_appimage_tool() { + APP_TOOL_BIN=$HOME/bin/appimagetool + APP_TOOL_URL=https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage + + mkdir "$HOME"/bin + curl -L -o "$APP_TOOL_BIN" "$APP_TOOL_URL" + chmod u+x "$APP_TOOL_BIN" + + PATH=$PATH:$HOME/bin +} + +load_monkey_binary_config() { + tmpfile=$(mktemp) + + log_message "downloading configuration" + curl -L -s -o "$tmpfile" "$config_url" + + log_message "loading configuration" + source "$tmpfile" +} + +clone_monkey_repo() { + if [[ ! -d ${GIT} ]]; then + mkdir -p "${GIT}" + fi + + log_message "Cloning files from git" + branch=${2:-"develop"} + git clone --single-branch --recurse-submodules -b "$branch" "${MONKEY_GIT_URL}" "${REPO_MONKEY_HOME}" 2>&1 || handle_error + + chmod 774 -R "${MONKEY_HOME}" +} + +copy_monkey_island_to_appdir() { + cp "$REPO_MONKEY_SRC"/__init__.py "$INSTALL_DIR" + cp "$REPO_MONKEY_SRC"/monkey_island.py "$INSTALL_DIR" + cp -r "$REPO_MONKEY_SRC"/common "$INSTALL_DIR" + cp -r "$REPO_MONKEY_SRC"/monkey_island "$INSTALL_DIR" + cp ./run_appimage.sh "$INSTALL_DIR"/monkey_island/linux/ + cp ./island_logger_config.json "$INSTALL_DIR"/ + cp ./server_config.json.standard "$INSTALL_DIR"/monkey_island/cc/ + + # TODO: This is a workaround that may be able to be removed after PR #848 is + # merged. See monkey_island/cc/environment_singleton.py for more information. + cp ./server_config.json.standard "$INSTALL_DIR"/monkey_island/cc/server_config.json +} + +install_monkey_island_python_dependencies() { + log_message "Installing island requirements" + + requirements_island="$ISLAND_PATH/requirements.txt" + # TODO: This is an ugly hack. PyInstaller and VirtualEnv are build-time + # dependencies and should not be installed as a runtime requirement. + cat "$requirements_island" | grep -Piv "virtualenv|pyinstaller" | sponge "$requirements_island" + + ${python_cmd} -m pip install -r "${requirements_island}" --ignore-installed --prefix /usr --root="$APPDIR" || handle_error +} + +download_monkey_agent_binaries() { +log_message "Downloading monkey agent binaries to ${ISLAND_BINARIES_PATH}" + mkdir -p "${ISLAND_BINARIES_PATH}" || handle_error + curl -L -o "${ISLAND_BINARIES_PATH}/${LINUX_32_BINARY_NAME}" "${LINUX_32_BINARY_URL}" + curl -L -o "${ISLAND_BINARIES_PATH}/${LINUX_64_BINARY_NAME}" "${LINUX_64_BINARY_URL}" + curl -L -o "${ISLAND_BINARIES_PATH}/${WINDOWS_32_BINARY_NAME}" "${WINDOWS_32_BINARY_URL}" + curl -L -o "${ISLAND_BINARIES_PATH}/${WINDOWS_64_BINARY_NAME}" "${WINDOWS_64_BINARY_URL}" + + # Allow them to be executed + chmod a+x "$ISLAND_BINARIES_PATH/$LINUX_32_BINARY_NAME" + chmod a+x "$ISLAND_BINARIES_PATH/$LINUX_64_BINARY_NAME" +} + +install_mongodb() { + log_message "Installing MongoDB" + + mkdir -p "$MONGO_PATH" + "${ISLAND_PATH}"/linux/install_mongo.sh "${MONGO_PATH}" || handle_error +} + +generate_ssl_cert() { + log_message "Generating certificate" + + chmod u+x "${ISLAND_PATH}"/linux/create_certificate.sh + "${ISLAND_PATH}"/linux/create_certificate.sh "${ISLAND_PATH}"/cc +} + +build_frontend() { + pushd "$ISLAND_PATH/cc/ui" || handle_error + npm install sass-loader node-sass webpack --save-dev + npm update + + log_message "Generating front end" + npm run dist + popd || handle_error +} + +build_appimage() { + log_message "Building AppImage" + appimage-builder --recipe monkey_island_builder.yml --log DEBUG --skip-appimage + + # There is a bug or unwanted behavior in appimage-builder that causes issues + # if 32-bit binaries are present in the appimage. To work around this, we: + # 1. Build the AppDir with appimage-builder and skip building the appimage + # 2. Add the 32-bit binaries to the AppDir + # 3. Build the AppImage with appimage-builder from the already-built AppDir + # + # Note that appimage-builder replaces the interpreter on the monkey agent binaries + # when building the AppDir. This is unwanted as the monkey agents may execute in + # environments where the AppImage isn't loaded. + # + # See https://github.com/AppImageCrafters/appimage-builder/issues/93 for more info. + download_monkey_agent_binaries + + appimage-builder --recipe monkey_island_builder.yml --log DEBUG --skip-build +} + +if is_root; then + log_message "Please don't run this script as root" + exit 1 +fi + +if ! has_sudo; then + log_message "You need root permissions for some of this script operations. \ +Run \`sudo -v\`, enter your password, and then re-run this script." + exit 1 +fi + +config_url="https://raw.githubusercontent.com/mssalvatore/monkey/linux-deploy-binaries/deployment_scripts/config" + +setup_appdir + +install_build_prereqs +install_appimage_builder + + +load_monkey_binary_config +clone_monkey_repo "$@" + +copy_monkey_island_to_appdir + +# Create folders +log_message "Creating island dirs under $ISLAND_PATH" +mkdir -p "${MONGO_PATH}" || handle_error + +install_monkey_island_python_dependencies + +install_mongodb + +generate_ssl_cert + +build_frontend + +mkdir -p "$APPDIR"/usr/share/icons +cp "$REPO_MONKEY_SRC"/monkey_island/cc/ui/src/images/monkey-icon.svg "$APPDIR"/usr/share/icons/monkey-icon.svg + +build_appimage + +log_message "Deployment script finished." +exit 0 diff --git a/deployment_scripts/appimage/island_logger_config.json b/deployment_scripts/appimage/island_logger_config.json new file mode 100644 index 000000000..4acf875e3 --- /dev/null +++ b/deployment_scripts/appimage/island_logger_config.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(asctime)s - %(filename)s:%(lineno)s - %(funcName)10s() - %(levelname)s - %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "simple", + "stream": "ext://sys.stdout" + }, + "info_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "INFO", + "formatter": "simple", + "filename": "~/.monkey_island/info.log", + "maxBytes": 10485760, + "backupCount": 20, + "encoding": "utf8" + } + }, + "root": { + "level": "DEBUG", + "handlers": [ + "console", + "info_file_handler" + ] + } +} diff --git a/deployment_scripts/appimage/monkey_island_builder.yml b/deployment_scripts/appimage/monkey_island_builder.yml new file mode 100644 index 000000000..2c85c41d3 --- /dev/null +++ b/deployment_scripts/appimage/monkey_island_builder.yml @@ -0,0 +1,40 @@ +version: 1 + +AppDir: + path: '../monkey-appdir' + + app_info: + id: org.guardicore.monkey-island + name: Monkey Island + icon: monkey-icon + version: 1.10.0 + exec: bin/bash + exec_args: "$APPDIR/usr/src/monkey_island/linux/run_appimage.sh" + + + apt: + arch: amd64 + sources: + - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic main restricted + key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3B4FE6ACC0B21F32 + - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic universe + - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-security main restricted + - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-security universe + - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted + - sourceline: deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ bionic-updates universe + + + include: + - bash + - python3.7 + + runtime: + env: + PATH: '${APPDIR}/usr/bin:${PATH}' + PYTHONHOME: '${APPDIR}/usr' + PYTHONPATH: '${APPDIR}/usr/lib/python3.7/site-packages' + +AppImage: + update-information: None + sign-key: None + arch: x86_64 diff --git a/deployment_scripts/appimage/run_appimage.sh b/deployment_scripts/appimage/run_appimage.sh new file mode 100644 index 000000000..7738301cd --- /dev/null +++ b/deployment_scripts/appimage/run_appimage.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +DOT_MONKEY=$HOME/.monkey_island/ + +configure_default_logging() { + if [ ! -f $DOT_MONKEY/island_logger_config.json ]; then + cp $APPDIR/usr/src/island_logger_config.json $DOT_MONKEY + fi +} + +configure_default_server() { + if [ ! -f $DOT_MONKEY/server_config.json ]; then + cp $APPDIR/usr/src/monkey_island/cc/server_config.json.standard $DOT_MONKEY/server_config.json + fi +} + + + +# Detecting command that calls python 3.7 +python_cmd="" +if [[ $(python --version 2>&1) == *"Python 3.7"* ]]; then + python_cmd="python" +fi +if [[ $(python37 --version 2>&1) == *"Python 3.7"* ]]; then + python_cmd="python37" +fi +if [[ $(python3.7 --version 2>&1) == *"Python 3.7"* ]]; then + python_cmd="python3.7" +fi + +mkdir --mode=0700 --parents $DOT_MONKEY + +DB_DIR=$DOT_MONKEY/db +mkdir -p $DB_DIR + +configure_default_logging +configure_default_server + +cd $APPDIR/usr/src +./monkey_island/bin/mongodb/bin/mongod --dbpath $DB_DIR & +${python_cmd} ./monkey_island.py --server-config $DOT_MONKEY/server_config.json --logger-config $DOT_MONKEY/island_logger_config.json diff --git a/deployment_scripts/appimage/server_config.json.standard b/deployment_scripts/appimage/server_config.json.standard new file mode 100644 index 000000000..99848f945 --- /dev/null +++ b/deployment_scripts/appimage/server_config.json.standard @@ -0,0 +1,5 @@ +{ + "server_config": "password", + "deployment": "standard", + "data_dir": "~/.monkey_island" +} diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index cd452066c..88e41b6e2 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -1,21 +1,25 @@ from gevent import monkey as gevent_monkey +from monkey_island.cc.arg_parser import parse_cli_args + gevent_monkey.patch_all() -from monkey_island.cc.main import main +import json # noqa: E402 - -def parse_cli_args(): - import argparse - parser = argparse.ArgumentParser(description="Infection Monkey Island CnC Server. See https://infectionmonkey.com") - parser.add_argument("-s", "--setup-only", action="store_true", - help="Pass this flag to cause the Island to setup and exit without actually starting. " - "This is useful for preparing Island to boot faster later-on, so for " - "compiling/packaging Islands.") - args = parser.parse_args() - return args.setup_only +from monkey_island.cc.server_utils.island_logger import json_setup_logging # noqa: E402 if "__main__" == __name__: - is_setup_only = parse_cli_args() - main(is_setup_only) + island_args = parse_cli_args() + + # This is here in order to catch EVERYTHING, some functions are being called on + # imports, so the log init needs to be first. + try: + json_setup_logging(island_args.logger_config) + except json.JSONDecodeError as ex: + print(f"Error loading logging config: {ex}") + exit(1) + + from monkey_island.cc.main import main # noqa: E402 + + main(island_args.setup_only, island_args.server_config) diff --git a/monkey/monkey_island/cc/arg_parser.py b/monkey/monkey_island/cc/arg_parser.py new file mode 100644 index 000000000..5ea12aec4 --- /dev/null +++ b/monkey/monkey_island/cc/arg_parser.py @@ -0,0 +1,42 @@ +from dataclasses import dataclass + +from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH, DEFAULT_LOGGER_CONFIG_PATH + + +@dataclass +class IslandArgs: + setup_only: bool + server_config: str + logger_config: str + + +def parse_cli_args() -> IslandArgs: + import argparse + + parser = argparse.ArgumentParser( + description="Infection Monkey Island CnC Server. See https://infectionmonkey.com", + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "-s", + "--setup-only", + action="store_true", + help="Pass this flag to cause the Island to setup and exit without actually starting. " + "This is useful for preparing Island to boot faster later-on, so for " + "compiling/packaging Islands.", + ) + parser.add_argument( + "--server-config", + action="store", + help="The path to the server configuration file.", + default=DEFAULT_SERVER_CONFIG_PATH, + ) + parser.add_argument( + "--logger-config", + action="store", + help="The path to the logging configuration file.", + default=DEFAULT_LOGGER_CONFIG_PATH, + ) + args = parser.parse_args() + + return IslandArgs(args.setup_only, args.server_config, args.logger_config) diff --git a/monkey/monkey_island/cc/environment/environment_config.py b/monkey/monkey_island/cc/environment/environment_config.py index 35dbafc8e..f390d8186 100644 --- a/monkey/monkey_island/cc/environment/environment_config.py +++ b/monkey/monkey_island/cc/environment/environment_config.py @@ -6,62 +6,66 @@ from pathlib import Path from typing import Dict, List import monkey_island.cc.environment.server_config_generator as server_config_generator -from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH +from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR from monkey_island.cc.environment.user_creds import UserCreds from monkey_island.cc.resources.auth.auth_user import User from monkey_island.cc.resources.auth.user_store import UserStore -SERVER_CONFIG_FILENAME = "server_config.json" - class EnvironmentConfig: - def __init__(self, - server_config: str, - deployment: str, - user_creds: UserCreds, - aws=None): - self.server_config = server_config - self.deployment = deployment - self.user_creds = user_creds - self.aws = aws + def __init__(self, file_path): + self._server_config_path = os.path.expanduser(file_path) + self.server_config = None + self.deployment = None + self.user_creds = None + self.aws = None + self.data_dir = None - @staticmethod - def get_from_json(config_json: str) -> EnvironmentConfig: - data = json.loads(config_json) - return EnvironmentConfig.get_from_dict(data) + self._load_from_file(self._server_config_path) - @staticmethod - def get_from_dict(dict_data: Dict) -> EnvironmentConfig: - user_creds = UserCreds.get_from_dict(dict_data) - aws = dict_data['aws'] if 'aws' in dict_data else None - return EnvironmentConfig(server_config=dict_data['server_config'], - deployment=dict_data['deployment'], - user_creds=user_creds, - aws=aws) + def _load_from_file(self, file_path): + file_path = os.path.expanduser(file_path) - def save_to_file(self): - file_path = EnvironmentConfig.get_config_file_path() - with open(file_path, 'w') as f: - f.write(json.dumps(self.to_dict(), indent=2)) - - @staticmethod - def get_from_file() -> EnvironmentConfig: - file_path = EnvironmentConfig.get_config_file_path() if not Path(file_path).is_file(): server_config_generator.create_default_config_file(file_path) - with open(file_path, 'r') as f: + with open(file_path, "r") as f: config_content = f.read() - return EnvironmentConfig.get_from_json(config_content) - @staticmethod - def get_config_file_path() -> str: - return os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', SERVER_CONFIG_FILENAME) + self._load_from_json(config_content) + + def _load_from_json(self, config_json: str) -> EnvironmentConfig: + data = json.loads(config_json) + self._load_from_dict(data) + + def _load_from_dict(self, dict_data: Dict): + user_creds = UserCreds.get_from_dict(dict_data) + aws = dict_data["aws"] if "aws" in dict_data else None + data_dir = ( + dict_data["data_dir"] if "data_dir" in dict_data else DEFAULT_DATA_DIR + ) + + self.server_config = dict_data["server_config"] + self.deployment = dict_data["deployment"] + self.user_creds = user_creds + self.aws = aws + self.data_dir = data_dir + + @property + def data_dir_abs_path(self): + return os.path.abspath(os.path.expanduser(os.path.expandvars(self.data_dir))) + + def save_to_file(self): + with open(self._server_config_path, "w") as f: + f.write(json.dumps(self.to_dict(), indent=2)) def to_dict(self) -> Dict: - config_dict = {'server_config': self.server_config, - 'deployment': self.deployment} + config_dict = { + "server_config": self.server_config, + "deployment": self.deployment, + "data_dir": self.data_dir, + } if self.aws: - config_dict.update({'aws': self.aws}) + config_dict.update({"aws": self.aws}) config_dict.update(self.user_creds.to_dict()) return config_dict diff --git a/monkey/monkey_island/cc/environment/environment_singleton.py b/monkey/monkey_island/cc/environment/environment_singleton.py index 6b98d0b7c..01e83096d 100644 --- a/monkey/monkey_island/cc/environment/environment_singleton.py +++ b/monkey/monkey_island/cc/environment/environment_singleton.py @@ -1,7 +1,9 @@ import logging import monkey_island.cc.resources.auth.user_store as user_store -from monkey_island.cc.environment import EnvironmentConfig, aws, password, standard, testing +from monkey_island.cc.environment import (EnvironmentConfig, aws, password, + standard, testing) +from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH __author__ = 'itay.mizeretz' @@ -36,12 +38,19 @@ def set_to_standard(): user_store.UserStore.set_users(env.get_auth_users()) -try: - config = EnvironmentConfig.get_from_file() - __env_type = config.server_config - set_env(__env_type, config) - # noinspection PyUnresolvedReferences - logger.info('Monkey\'s env is: {0}'.format(env.__class__.__name__)) -except Exception: - logger.error('Failed initializing environment', exc_info=True) - raise +def initialize_from_file(file_path): + try: + config = EnvironmentConfig(file_path) + + __env_type = config.server_config + set_env(__env_type, config) + # noinspection PyUnresolvedReferences + logger.info('Monkey\'s env is: {0}'.format(env.__class__.__name__)) + except Exception: + logger.error('Failed initializing environment', exc_info=True) + raise + + +# TODO: This is only needed so that unit tests pass. After PR #848 is merged, we may be +# able to remove this line. +initialize_from_file(DEFAULT_SERVER_CONFIG_PATH) diff --git a/monkey/monkey_island/cc/environment/server_config_generator.py b/monkey/monkey_island/cc/environment/server_config_generator.py index d5c645564..211b745c5 100644 --- a/monkey/monkey_island/cc/environment/server_config_generator.py +++ b/monkey/monkey_island/cc/environment/server_config_generator.py @@ -1,7 +1,8 @@ from pathlib import Path +from monkey_island.cc.server_utils.consts import DEFAULT_DEVELOP_SERVER_CONFIG_PATH + def create_default_config_file(path): - default_config_path = f"{path}.default" - default_config = Path(default_config_path).read_text() + default_config = Path(DEFAULT_DEVELOP_SERVER_CONFIG_PATH).read_text() Path(path).write_text(default_config) diff --git a/monkey/monkey_island/cc/environment/set_server_config.py b/monkey/monkey_island/cc/environment/set_server_config.py index 168fe13cd..f3fbd66ff 100644 --- a/monkey/monkey_island/cc/environment/set_server_config.py +++ b/monkey/monkey_island/cc/environment/set_server_config.py @@ -14,7 +14,7 @@ def add_monkey_dir_to_sys_path(): add_monkey_dir_to_sys_path() -from monkey_island.cc.environment.environment_config import EnvironmentConfig # noqa: E402 isort:skip +from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH # noqa: E402 isort:skip SERVER_CONFIG = "server_config" BACKUP_CONFIG_FILENAME = "./server_config.backup" @@ -26,7 +26,7 @@ logger.setLevel(logging.DEBUG) def main(): args = parse_args() - file_path = EnvironmentConfig.get_config_file_path() + file_path = DEFAULT_SERVER_CONFIG_PATH if args.server_config == "restore": restore_previous_config(file_path) diff --git a/monkey/monkey_island/cc/environment/test__init__.py b/monkey/monkey_island/cc/environment/test__init__.py index c55e1b65b..fde0a8b27 100644 --- a/monkey/monkey_island/cc/environment/test__init__.py +++ b/monkey/monkey_island/cc/environment/test__init__.py @@ -1,13 +1,42 @@ -import json import os +import tempfile from typing import Dict from unittest import TestCase from unittest.mock import MagicMock, patch -import monkey_island.cc.test_common.environment.server_config_mocks as config_mocks -from common.utils.exceptions import (AlreadyRegisteredError, CredentialsNotRequiredError, - InvalidRegistrationCredentialsError, RegistrationNotNeededError) -from monkey_island.cc.environment import Environment, EnvironmentConfig, UserCreds +from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH +from common.utils.exceptions import (AlreadyRegisteredError, + CredentialsNotRequiredError, + InvalidRegistrationCredentialsError, + RegistrationNotNeededError) +from monkey_island.cc.environment import (Environment, EnvironmentConfig, + UserCreds) + +TEST_RESOURCES_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing", "environment") + +WITH_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_with_credentials.json") +NO_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_no_credentials.json") +PARTIAL_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_partial_credentials.json") +STANDARD_WITH_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, + "server_config_standard_with_credentials.json") +STANDARD_ENV = os.path.join(TEST_RESOURCES_DIR, + "server_config_standard_env.json") + + +def get_tmp_file(): + with tempfile.NamedTemporaryFile(delete=False) as f: + return f.name + + +class StubEnvironmentConfig(EnvironmentConfig): + def __init__(self, server_config, deployment, user_creds): + self.server_config = server_config + self.deployment = deployment + self.user_creds = user_creds + self.server_config_path = get_tmp_file() + + def __del__(self): + os.remove(self.server_config_path) def get_server_config_file_path_test_version(): @@ -18,7 +47,7 @@ class TestEnvironment(TestCase): class EnvironmentCredentialsNotRequired(Environment): def __init__(self): - config = EnvironmentConfig('test', 'test', UserCreds()) + config = StubEnvironmentConfig('test', 'test', UserCreds()) super().__init__(config) _credentials_required = False @@ -28,7 +57,7 @@ class TestEnvironment(TestCase): class EnvironmentCredentialsRequired(Environment): def __init__(self): - config = EnvironmentConfig('test', 'test', UserCreds()) + config = StubEnvironmentConfig('test', 'test', UserCreds()) super().__init__(config) _credentials_required = True @@ -38,7 +67,7 @@ class TestEnvironment(TestCase): class EnvironmentAlreadyRegistered(Environment): def __init__(self): - config = EnvironmentConfig('test', 'test', UserCreds('test_user', 'test_secret')) + config = StubEnvironmentConfig('test', 'test', UserCreds('test_user', 'test_secret')) super().__init__(config) _credentials_required = True @@ -75,35 +104,35 @@ class TestEnvironment(TestCase): def test_needs_registration(self): env = TestEnvironment.EnvironmentCredentialsRequired() - self._test_bool_env_method("needs_registration", env, config_mocks.CONFIG_WITH_CREDENTIALS, False) - self._test_bool_env_method("needs_registration", env, config_mocks.CONFIG_NO_CREDENTIALS, True) - self._test_bool_env_method("needs_registration", env, config_mocks.CONFIG_PARTIAL_CREDENTIALS, True) + self._test_bool_env_method("needs_registration", env, WITH_CREDENTIALS, False) + self._test_bool_env_method("needs_registration", env, NO_CREDENTIALS, True) + self._test_bool_env_method("needs_registration", env, PARTIAL_CREDENTIALS, True) env = TestEnvironment.EnvironmentCredentialsNotRequired() - self._test_bool_env_method("needs_registration", env, config_mocks.CONFIG_STANDARD_ENV, False) - self._test_bool_env_method("needs_registration", env, config_mocks.CONFIG_STANDARD_WITH_CREDENTIALS, False) + self._test_bool_env_method("needs_registration", env, STANDARD_ENV, False) + self._test_bool_env_method("needs_registration", env, STANDARD_WITH_CREDENTIALS, False) def test_is_registered(self): env = TestEnvironment.EnvironmentCredentialsRequired() - self._test_bool_env_method("_is_registered", env, config_mocks.CONFIG_WITH_CREDENTIALS, True) - self._test_bool_env_method("_is_registered", env, config_mocks.CONFIG_NO_CREDENTIALS, False) - self._test_bool_env_method("_is_registered", env, config_mocks.CONFIG_PARTIAL_CREDENTIALS, False) + self._test_bool_env_method("_is_registered", env, WITH_CREDENTIALS, True) + self._test_bool_env_method("_is_registered", env, NO_CREDENTIALS, False) + self._test_bool_env_method("_is_registered", env, PARTIAL_CREDENTIALS, False) env = TestEnvironment.EnvironmentCredentialsNotRequired() - self._test_bool_env_method("_is_registered", env, config_mocks.CONFIG_STANDARD_ENV, False) - self._test_bool_env_method("_is_registered", env, config_mocks.CONFIG_STANDARD_WITH_CREDENTIALS, False) + self._test_bool_env_method("_is_registered", env, STANDARD_ENV, False) + self._test_bool_env_method("_is_registered", env, STANDARD_WITH_CREDENTIALS, False) def test_is_credentials_set_up(self): env = TestEnvironment.EnvironmentCredentialsRequired() - self._test_bool_env_method("_is_credentials_set_up", env, config_mocks.CONFIG_NO_CREDENTIALS, False) - self._test_bool_env_method("_is_credentials_set_up", env, config_mocks.CONFIG_WITH_CREDENTIALS, True) - self._test_bool_env_method("_is_credentials_set_up", env, config_mocks.CONFIG_PARTIAL_CREDENTIALS, False) + self._test_bool_env_method("_is_credentials_set_up", env, NO_CREDENTIALS, False) + self._test_bool_env_method("_is_credentials_set_up", env, WITH_CREDENTIALS, True) + self._test_bool_env_method("_is_credentials_set_up", env, PARTIAL_CREDENTIALS, False) env = TestEnvironment.EnvironmentCredentialsNotRequired() - self._test_bool_env_method("_is_credentials_set_up", env, config_mocks.CONFIG_STANDARD_ENV, False) + self._test_bool_env_method("_is_credentials_set_up", env, STANDARD_ENV, False) def _test_bool_env_method(self, method_name: str, env: Environment, config: Dict, expected_result: bool): - env._config = EnvironmentConfig.get_from_json(json.dumps(config)) + env._config = EnvironmentConfig(config) method = getattr(env, method_name) if expected_result: self.assertTrue(method()) diff --git a/monkey/monkey_island/cc/environment/test_environment_config.py b/monkey/monkey_island/cc/environment/test_environment_config.py index ed9b0ef96..de941a6f3 100644 --- a/monkey/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/monkey_island/cc/environment/test_environment_config.py @@ -1,99 +1,138 @@ import json import os -import platform -from typing import Dict -from unittest import TestCase -from unittest.mock import MagicMock, patch +import shutil -import monkey_island.cc.test_common.environment.server_config_mocks as config_mocks -from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH +import pytest + +from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, MONKEY_ISLAND_ABS_PATH from monkey_island.cc.environment.environment_config import EnvironmentConfig from monkey_island.cc.environment.user_creds import UserCreds -def get_server_config_file_path_test_version(): - return os.path.join(os.getcwd(), 'test_config.json') +TEST_RESOURCES_DIR = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", "testing", "environment" +) + +WITH_CREDENTIALS = os.path.join( + TEST_RESOURCES_DIR, "server_config_with_credentials.json" +) +NO_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_no_credentials.json") +PARTIAL_CREDENTIALS = os.path.join( + TEST_RESOURCES_DIR, "server_config_partial_credentials.json" +) +STANDARD_WITH_CREDENTIALS = os.path.join( + TEST_RESOURCES_DIR, "server_config_standard_with_credentials.json" +) +WITH_DATA_DIR = os.path.join(TEST_RESOURCES_DIR, "server_config_with_data_dir.json") +WITH_DATA_DIR_HOME = os.path.join(TEST_RESOURCES_DIR, "server_config_with_data_dir_home.json") -class TestEnvironmentConfig(TestCase): +@pytest.fixture +def config_file(tmpdir): + return os.path.join(tmpdir, "test_config.json") - def test_get_from_json(self): - self._test_get_from_json(config_mocks.CONFIG_WITH_CREDENTIALS) - self._test_get_from_json(config_mocks.CONFIG_NO_CREDENTIALS) - self._test_get_from_json(config_mocks.CONFIG_PARTIAL_CREDENTIALS) - def _test_get_from_json(self, config: Dict): - config_json = json.dumps(config) - env_config_object = EnvironmentConfig.get_from_json(config_json) - self.assertEqual(config['server_config'], env_config_object.server_config) - self.assertEqual(config['deployment'], env_config_object.deployment) - if 'user' in config: - self.assertEqual(config['user'], env_config_object.user_creds.username) - if 'password_hash' in config: - self.assertEqual(config['password_hash'], env_config_object.user_creds.password_hash) - if 'aws' in config: - self.assertEqual(config['aws'], env_config_object.aws) +def test_get_with_credentials(): + config_dict = EnvironmentConfig(WITH_CREDENTIALS).to_dict() - def test_save_to_file(self): - self._test_save_to_file(config_mocks.CONFIG_WITH_CREDENTIALS) - self._test_save_to_file(config_mocks.CONFIG_NO_CREDENTIALS) - self._test_save_to_file(config_mocks.CONFIG_PARTIAL_CREDENTIALS) + assert len(config_dict.keys()) == 5 + assert config_dict["server_config"] == "password" + assert config_dict["deployment"] == "develop" + assert config_dict["user"] == "test" + assert config_dict["password_hash"] == "abcdef" + assert config_dict["data_dir"] == DEFAULT_DATA_DIR - @patch.object(target=EnvironmentConfig, attribute="get_config_file_path", - new=MagicMock(return_value=get_server_config_file_path_test_version())) - def _test_save_to_file(self, config: Dict): - user_creds = UserCreds.get_from_dict(config) - env_config = EnvironmentConfig(server_config=config['server_config'], - deployment=config['deployment'], - user_creds=user_creds) - env_config.save_to_file() - file_path = get_server_config_file_path_test_version() - with open(file_path, 'r') as f: - content_from_file = f.read() - os.remove(file_path) +def test_get_with_no_credentials(): + config_dict = EnvironmentConfig(NO_CREDENTIALS).to_dict() - self.assertDictEqual(config, json.loads(content_from_file)) + assert len(config_dict.keys()) == 3 + assert config_dict["server_config"] == "password" + assert config_dict["deployment"] == "develop" + assert config_dict["data_dir"] == DEFAULT_DATA_DIR - def test_get_server_config_file_path(self): - if platform.system() == "Windows": - server_file_path = MONKEY_ISLAND_ABS_PATH + r"\cc\server_config.json" - else: - server_file_path = MONKEY_ISLAND_ABS_PATH + "/cc/server_config.json" - self.assertEqual(EnvironmentConfig.get_config_file_path(), server_file_path) - def test_get_from_dict(self): - config_dict = config_mocks.CONFIG_WITH_CREDENTIALS - env_conf = EnvironmentConfig.get_from_dict(config_dict) - self.assertEqual(env_conf.server_config, config_dict['server_config']) - self.assertEqual(env_conf.deployment, config_dict['deployment']) - self.assertEqual(env_conf.user_creds.username, config_dict['user']) - self.assertEqual(env_conf.aws, None) +def test_get_with_partial_credentials(): + config_dict = EnvironmentConfig(PARTIAL_CREDENTIALS).to_dict() - config_dict = config_mocks.CONFIG_BOGUS_VALUES - env_conf = EnvironmentConfig.get_from_dict(config_dict) - self.assertEqual(env_conf.server_config, config_dict['server_config']) - self.assertEqual(env_conf.deployment, config_dict['deployment']) - self.assertEqual(env_conf.user_creds.username, config_dict['user']) - self.assertEqual(env_conf.aws, config_dict['aws']) + assert len(config_dict.keys()) == 4 + assert config_dict["server_config"] == "password" + assert config_dict["deployment"] == "develop" + assert config_dict["user"] == "test" + assert config_dict["data_dir"] == DEFAULT_DATA_DIR - def test_to_dict(self): - conf_json1 = json.dumps(config_mocks.CONFIG_WITH_CREDENTIALS) - self._test_to_dict(EnvironmentConfig.get_from_json(conf_json1)) - conf_json2 = json.dumps(config_mocks.CONFIG_NO_CREDENTIALS) - self._test_to_dict(EnvironmentConfig.get_from_json(conf_json2)) +def test_save_to_file(config_file): + shutil.copyfile(STANDARD_WITH_CREDENTIALS, config_file) - conf_json3 = json.dumps(config_mocks.CONFIG_PARTIAL_CREDENTIALS) - self._test_to_dict(EnvironmentConfig.get_from_json(conf_json3)) + environment_config = EnvironmentConfig(config_file) + environment_config.aws = "test_aws" + environment_config.save_to_file() - def _test_to_dict(self, env_config_object: EnvironmentConfig): - test_dict = {'server_config': env_config_object.server_config, - 'deployment': env_config_object.deployment} - user_creds = env_config_object.user_creds - if user_creds.username: - test_dict.update({'user': user_creds.username}) - if user_creds.password_hash: - test_dict.update({'password_hash': user_creds.password_hash}) + with open(config_file, "r") as f: + from_file = json.load(f) - self.assertDictEqual(test_dict, env_config_object.to_dict()) + assert len(from_file.keys()) == 6 + assert from_file["server_config"] == "standard" + assert from_file["deployment"] == "develop" + assert from_file["user"] == "test" + assert from_file["password_hash"] == "abcdef" + assert from_file["aws"] == "test_aws" + assert from_file["data_dir"] == DEFAULT_DATA_DIR + + +def test_add_user(config_file): + new_user = "new_user" + new_password_hash = "fedcba" + new_user_creds = UserCreds(new_user, new_password_hash) + + shutil.copyfile(STANDARD_WITH_CREDENTIALS, config_file) + + environment_config = EnvironmentConfig(config_file) + environment_config.add_user(new_user_creds) + + with open(config_file, "r") as f: + from_file = json.load(f) + + assert len(from_file.keys()) == 5 + assert from_file["user"] == new_user + assert from_file["password_hash"] == new_password_hash + + +def test_get_users(): + environment_config = EnvironmentConfig(STANDARD_WITH_CREDENTIALS) + users = environment_config.get_users() + + assert len(users) == 1 + assert users[0].id == 1 + assert users[0].username == "test" + assert users[0].secret == "abcdef" + + +def test_generate_default_file(config_file): + environment_config = EnvironmentConfig(config_file) + + assert os.path.isfile(config_file) + + assert environment_config.server_config == "password" + assert environment_config.deployment == "develop" + assert environment_config.user_creds.username == "" + assert environment_config.user_creds.password_hash == "" + assert environment_config.aws is None + assert environment_config.data_dir == DEFAULT_DATA_DIR + + +def test_data_dir(): + environment_config = EnvironmentConfig(WITH_DATA_DIR) + assert environment_config.data_dir == "/test/data/dir" + + +def set_home_env(monkeypatch, tmpdir): + monkeypatch.setenv("HOME", str(tmpdir)) + + +def test_data_dir_abs_path_from_file(monkeypatch, tmpdir): + set_home_env(monkeypatch, tmpdir) + + config = EnvironmentConfig(WITH_DATA_DIR_HOME) + assert config.data_dir_abs_path == os.path.join(tmpdir, "data_dir") diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index ce142edcc..211084565 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -13,19 +13,17 @@ if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path: sys.path.insert(0, MONKEY_ISLAND_DIR_BASE_PATH) from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH # noqa: E402 -from monkey_island.cc.server_utils.island_logger import json_setup_logging # noqa: E402 -# This is here in order to catch EVERYTHING, some functions are being called on imports the log init needs to be on top. -json_setup_logging(default_path=Path(MONKEY_ISLAND_ABS_PATH, 'cc', 'island_logger_default_config.json'), - default_level=logging.DEBUG) logger = logging.getLogger(__name__) import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 +from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH # noqa: E402 from common.version import get_version # noqa: E402 from monkey_island.cc.app import init_app # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 from monkey_island.cc.database import get_db_version # noqa: E402 from monkey_island.cc.database import is_db_server_up # noqa: E402 +from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 @@ -34,8 +32,11 @@ from monkey_island.cc.setup import setup # noqa: E402 MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" -def main(should_setup_only=False): +def main(should_setup_only=False, server_config_filename=DEFAULT_SERVER_CONFIG_PATH): logger.info("Starting bootloader server") + env_singleton.initialize_from_file(server_config_filename) + initialize_encryptor(env_singleton.env.get_config().data_dir_abs_path) + mongo_url = os.environ.get('MONGO_URL', env_singleton.env.get_mongo_url()) bootloader_server_thread = Thread(target=BootloaderHttpServer(mongo_url).serve_forever, daemon=True) diff --git a/monkey/monkey_island/cc/resources/local_run.py b/monkey/monkey_island/cc/resources/local_run.py index 1a388db0a..0758d40c2 100644 --- a/monkey/monkey_island/cc/resources/local_run.py +++ b/monkey/monkey_island/cc/resources/local_run.py @@ -32,7 +32,7 @@ def run_local_monkey(): return False, "OS Type not found" monkey_path = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc', 'binaries', result['filename']) - target_path = os.path.join(MONKEY_ISLAND_ABS_PATH, result['filename']) + target_path = os.path.join(env_singleton.env.get_config().data_dir_abs_path, result['filename']) # copy the executable to temp path (don't run the monkey from its current location as it may delete itself) try: diff --git a/monkey/monkey_island/cc/server_config.json.default b/monkey/monkey_island/cc/server_config.json.develop similarity index 100% rename from monkey/monkey_island/cc/server_config.json.default rename to monkey/monkey_island/cc/server_config.json.develop diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index c302f6fb7..5a0e69581 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -1,6 +1,20 @@ import os -__author__ = 'itay.mizeretz' +__author__ = "itay.mizeretz" -MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), 'monkey_island') +MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island") DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5 + +DEFAULT_SERVER_CONFIG_PATH = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json" +) + +DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json.develop" +) + +DEFAULT_LOGGER_CONFIG_PATH = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", "island_logger_default_config.json" +) + +DEFAULT_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc") diff --git a/monkey/monkey_island/cc/server_utils/encryptor.py b/monkey/monkey_island/cc/server_utils/encryptor.py index cce7d464a..161032c52 100644 --- a/monkey/monkey_island/cc/server_utils/encryptor.py +++ b/monkey/monkey_island/cc/server_utils/encryptor.py @@ -6,36 +6,36 @@ import os from Crypto import Random # noqa: DUO133 # nosec: B413 from Crypto.Cipher import AES # noqa: DUO133 # nosec: B413 -from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH - __author__ = "itay.mizeretz" +_encryptor = None + class Encryptor: _BLOCK_SIZE = 32 - _DB_PASSWORD_FILENAME = os.path.join(MONKEY_ISLAND_ABS_PATH, 'cc/mongo_key.bin') + _PASSWORD_FILENAME = "mongo_key.bin" - def __init__(self): - self._load_key() + def __init__(self, password_file_dir): + password_file = os.path.join(password_file_dir, self._PASSWORD_FILENAME) - def _init_key(self): + if os.path.exists(password_file): + self._load_existing_key(password_file) + else: + self._init_key(password_file) + + def _init_key(self, password_file): self._cipher_key = Random.new().read(self._BLOCK_SIZE) - with open(self._DB_PASSWORD_FILENAME, 'wb') as f: + with open(password_file, "wb") as f: f.write(self._cipher_key) - def _load_existing_key(self): - with open(self._DB_PASSWORD_FILENAME, 'rb') as f: + def _load_existing_key(self, password_file): + with open(password_file, "rb") as f: self._cipher_key = f.read() - def _load_key(self): - if os.path.exists(self._DB_PASSWORD_FILENAME): - self._load_existing_key() - else: - self._init_key() - def _pad(self, message): return message + (self._BLOCK_SIZE - (len(message) % self._BLOCK_SIZE)) * chr( - self._BLOCK_SIZE - (len(message) % self._BLOCK_SIZE)) + self._BLOCK_SIZE - (len(message) % self._BLOCK_SIZE) + ) def _unpad(self, message: str): return message[0:-ord(message[len(message) - 1])] @@ -43,7 +43,9 @@ class Encryptor: def enc(self, message: str): cipher_iv = Random.new().read(AES.block_size) cipher = AES.new(self._cipher_key, AES.MODE_CBC, cipher_iv) - return base64.b64encode(cipher_iv + cipher.encrypt(self._pad(message).encode())).decode() + return base64.b64encode( + cipher_iv + cipher.encrypt(self._pad(message).encode()) + ).decode() def dec(self, enc_message): enc_message = base64.b64decode(enc_message) @@ -52,4 +54,11 @@ class Encryptor: return self._unpad(cipher.decrypt(enc_message[AES.block_size:]).decode()) -encryptor = Encryptor() +def initialize_encryptor(password_file_dir): + global _encryptor + + _encryptor = Encryptor(password_file_dir) + + +def get_encryptor(): + return _encryptor diff --git a/monkey/monkey_island/cc/server_utils/island_logger.py b/monkey/monkey_island/cc/server_utils/island_logger.py index 2b4843876..1efbb7734 100644 --- a/monkey/monkey_island/cc/server_utils/island_logger.py +++ b/monkey/monkey_island/cc/server_utils/island_logger.py @@ -1,11 +1,18 @@ import json import logging.config import os +from typing import Dict -__author__ = 'Maor.Rayzin' +from monkey_island.cc.server_utils.consts import DEFAULT_LOGGER_CONFIG_PATH + +__author__ = "Maor.Rayzin" -def json_setup_logging(default_path='logging.json', default_level=logging.INFO, env_key='LOG_CFG'): +def json_setup_logging( + default_path=DEFAULT_LOGGER_CONFIG_PATH, + default_level=logging.INFO, + env_key="LOG_CFG", +): """ Setup the logging configuration :param default_path: the default log configuration file path @@ -13,13 +20,26 @@ def json_setup_logging(default_path='logging.json', default_level=logging.INFO, :param env_key: SYS ENV key to use for external configuration file path :return: """ - path = default_path + path = os.path.expanduser(default_path) value = os.getenv(env_key, None) + if value: path = value + if os.path.exists(path): - with open(path, 'rt') as f: + with open(path, "rt") as f: config = json.load(f) - logging.config.dictConfig(config) + _expanduser_log_file_paths(config) + logging.config.dictConfig(config) else: logging.basicConfig(level=default_level) + + +def _expanduser_log_file_paths(config: Dict): + handlers = config.get("handlers", {}) + + for handler_settings in handlers.values(): + if "filename" in handler_settings: + handler_settings["filename"] = os.path.expanduser( + handler_settings["filename"] + ) diff --git a/monkey/monkey_island/cc/server_utils/test_island_logger.py b/monkey/monkey_island/cc/server_utils/test_island_logger.py new file mode 100644 index 000000000..af58f4b75 --- /dev/null +++ b/monkey/monkey_island/cc/server_utils/test_island_logger.py @@ -0,0 +1,32 @@ +import logging +import os + +import pytest + +from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH +from monkey_island.cc.server_utils.island_logger import json_setup_logging + +TEST_LOGGER_CONFIG_PATH = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", "testing", "logger_config.json" +) + + +# TODO move into monkey/monkey_island/cc/test_common/fixtures after rebase/backmerge +@pytest.fixture +def mock_home_env(monkeypatch, tmpdir): + monkeypatch.setenv("HOME", str(tmpdir)) + + +def test_expanduser_filename(mock_home_env, tmpdir): + INFO_LOG = os.path.join(tmpdir, "info.log") + TEST_STRING = "Hello, Monkey!" + + json_setup_logging(TEST_LOGGER_CONFIG_PATH) + + logger = logging.getLogger("TestLogger") + logger.info(TEST_STRING) + + assert os.path.isfile(INFO_LOG) + with open(INFO_LOG, "r") as f: + line = f.readline() + assert TEST_STRING in line diff --git a/monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py b/monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py index 34be687a4..6921b0129 100644 --- a/monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py +++ b/monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py @@ -1,4 +1,4 @@ -from monkey_island.cc.server_utils.encryptor import encryptor +from monkey_island.cc.server_utils.encryptor import get_encryptor def parse_creds(attempt): @@ -29,7 +29,7 @@ def censor_password(password, plain_chars=3, secret_chars=5): """ if not password: return "" - password = encryptor.dec(password) + password = get_encryptor().dec(password) return password[0:plain_chars] + '*' * secret_chars @@ -42,5 +42,5 @@ def censor_hash(hash_, plain_chars=5): """ if not hash_: return "" - hash_ = encryptor.dec(hash_) + hash_ = get_encryptor().dec(hash_) return hash_[0: plain_chars] + ' ...' diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 390380131..9fd8e3417 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -8,7 +8,7 @@ from jsonschema import Draft4Validator, validators import monkey_island.cc.environment.environment_singleton as env_singleton import monkey_island.cc.services.post_breach_files from monkey_island.cc.database import mongo -from monkey_island.cc.server_utils.encryptor import encryptor +from monkey_island.cc.server_utils.encryptor import get_encryptor from monkey_island.cc.services.utils.network_utils import local_ip_addresses from monkey_island.cc.services.config_schema.config_schema import SCHEMA @@ -75,9 +75,9 @@ class ConfigService: if should_decrypt: if config_key_as_arr in ENCRYPTED_CONFIG_VALUES: if isinstance(config, str): - config = encryptor.dec(config) + config = get_encryptor().dec(config) elif isinstance(config, list): - config = [encryptor.dec(x) for x in config] + config = [get_encryptor().dec(x) for x in config] return config @staticmethod @@ -112,7 +112,7 @@ class ConfigService: if item_value in items_from_config: return if should_encrypt: - item_value = encryptor.enc(item_value) + item_value = get_encryptor().enc(item_value) mongo.db.config.update( {'name': 'newconfig'}, {'$addToSet': {item_key: item_value}}, @@ -297,9 +297,9 @@ class ConfigService: if flat_config[key] and isinstance(flat_config[key][0], dict) and 'public_key' in flat_config[key][0]: flat_config[key] = [ConfigService.decrypt_ssh_key_pair(item) for item in flat_config[key]] else: - flat_config[key] = [encryptor.dec(item) for item in flat_config[key]] + flat_config[key] = [get_encryptor().dec(item) for item in flat_config[key]] else: - flat_config[key] = encryptor.dec(flat_config[key]) + flat_config[key] = get_encryptor().dec(flat_config[key]) return flat_config @staticmethod @@ -320,19 +320,19 @@ class ConfigService: config_arr[i] = ConfigService.decrypt_ssh_key_pair(config_arr[i]) if is_decrypt else \ ConfigService.decrypt_ssh_key_pair(config_arr[i], True) else: - config_arr[i] = encryptor.dec(config_arr[i]) if is_decrypt else encryptor.enc(config_arr[i]) + config_arr[i] = get_encryptor().dec(config_arr[i]) if is_decrypt else get_encryptor().enc(config_arr[i]) else: parent_config_arr[config_arr_as_array[-1]] = \ - encryptor.dec(config_arr) if is_decrypt else encryptor.enc(config_arr) + get_encryptor().dec(config_arr) if is_decrypt else get_encryptor().enc(config_arr) @staticmethod def decrypt_ssh_key_pair(pair, encrypt=False): if encrypt: - pair['public_key'] = encryptor.enc(pair['public_key']) - pair['private_key'] = encryptor.enc(pair['private_key']) + pair['public_key'] = get_encryptor().enc(pair['public_key']) + pair['private_key'] = get_encryptor().enc(pair['private_key']) else: - pair['public_key'] = encryptor.dec(pair['public_key']) - pair['private_key'] = encryptor.dec(pair['private_key']) + pair['public_key'] = get_encryptor().dec(pair['public_key']) + pair['private_key'] = get_encryptor().dec(pair['private_key']) return pair @staticmethod diff --git a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py index 3d8588663..9b06b028d 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py @@ -2,7 +2,7 @@ import copy import dateutil -from monkey_island.cc.server_utils.encryptor import encryptor +from monkey_island.cc.server_utils.encryptor import get_encryptor from monkey_island.cc.models import Monkey from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.edge.displayed_edge import EdgeService @@ -66,4 +66,4 @@ def encrypt_exploit_creds(telemetry_json): for field in ['password', 'lm_hash', 'ntlm_hash']: credential = attempts[i][field] if len(credential) > 0: - attempts[i][field] = encryptor.enc(credential) + attempts[i][field] = get_encryptor().enc(credential) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index d3e7cfb54..250080697 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -1,6 +1,6 @@ import logging -from monkey_island.cc.server_utils.encryptor import encryptor +from monkey_island.cc.server_utils.encryptor import get_encryptor from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.system_info_collectors.system_info_telemetry_dispatcher import \ @@ -63,7 +63,7 @@ def encrypt_system_info_ssh_keys(ssh_info): for idx, user in enumerate(ssh_info): for field in ['public_key', 'private_key', 'known_hosts']: if ssh_info[idx][field]: - ssh_info[idx][field] = encryptor.enc(ssh_info[idx][field]) + ssh_info[idx][field] = get_encryptor().enc(ssh_info[idx][field]) def process_credential_info(telemetry_json): diff --git a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py index 701598168..1f0ee180e 100644 --- a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py +++ b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py @@ -4,7 +4,7 @@ from ScoutSuite.providers.base.authentication_strategy import AuthenticationExce from common.cloud.scoutsuite_consts import CloudProviders from common.utils.exceptions import InvalidAWSKeys -from monkey_island.cc.server_utils.encryptor import encryptor +from monkey_island.cc.server_utils.encryptor import get_encryptor from monkey_island.cc.services.config import ConfigService from common.config_value_paths import AWS_KEYS_PATH @@ -37,7 +37,7 @@ def set_aws_keys(access_key_id: str, secret_access_key: str, session_token: str) def _set_aws_key(key_type: str, key_value: str): path_to_keys = AWS_KEYS_PATH - encrypted_key = encryptor.enc(key_value) + encrypted_key = get_encryptor().enc(key_value) ConfigService.set_config_value(path_to_keys + [key_type], encrypted_key) diff --git a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/test_scoutsuite_auth_service.py b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/test_scoutsuite_auth_service.py index c35e55a8f..1ac9afdfe 100644 --- a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/test_scoutsuite_auth_service.py +++ b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/test_scoutsuite_auth_service.py @@ -4,7 +4,7 @@ import pytest import dpath.util from monkey_island.cc.database import mongo -from monkey_island.cc.server_utils import encryptor +from monkey_island.cc.server_utils.encryptor import initialize_encryptor, get_encryptor from monkey_island.cc.services.config import ConfigService from common.config_value_paths import AWS_KEYS_PATH from monkey_island.cc.services.zero_trust.scoutsuite.scoutsuite_auth_service import is_aws_keys_setup @@ -16,7 +16,7 @@ class MockObject: @pytest.mark.usefixtures(FixtureEnum.USES_DATABASE) -def test_is_aws_keys_setup(): +def test_is_aws_keys_setup(tmp_path): # Mock default configuration ConfigService.init_default_config() mongo.db = MockObject() @@ -26,7 +26,8 @@ def test_is_aws_keys_setup(): assert not is_aws_keys_setup() # Make sure noone changed config path and broke this function - bogus_key_value = encryptor.encryptor.enc('bogus_aws_key') + initialize_encryptor(tmp_path) + bogus_key_value = get_encryptor().enc('bogus_aws_key') dpath.util.set(ConfigService.default_config, AWS_KEYS_PATH+['aws_secret_access_key'], bogus_key_value) dpath.util.set(ConfigService.default_config, AWS_KEYS_PATH+['aws_access_key_id'], bogus_key_value) diff --git a/monkey/monkey_island/cc/test_common/environment/server_config_mocks.py b/monkey/monkey_island/cc/test_common/environment/server_config_mocks.py deleted file mode 100644 index ddbff3118..000000000 --- a/monkey/monkey_island/cc/test_common/environment/server_config_mocks.py +++ /dev/null @@ -1,41 +0,0 @@ -# Username:test Password:test -CONFIG_WITH_CREDENTIALS = { - "server_config": "password", - "deployment": "develop", - "user": "test", - "password_hash": "9ece086e9bac491fac5c1d1046ca11d737b92a2b2ebd93f005d7b710110c0a678288166e7fbe796883a" - "4f2e9b3ca9f484f521d0ce464345cc1aec96779149c14" -} - -CONFIG_NO_CREDENTIALS = { - "server_config": "password", - "deployment": "develop" -} - -CONFIG_PARTIAL_CREDENTIALS = { - "server_config": "password", - "deployment": "develop", - "user": "test" -} - -CONFIG_BOGUS_VALUES = { - "server_config": "password", - "deployment": "develop", - "user": "test", - "aws": "test", - "test": "test", - "test2": "test2" -} - -CONFIG_STANDARD_ENV = { - "server_config": "standard", - "deployment": "develop" -} - -CONFIG_STANDARD_WITH_CREDENTIALS = { - "server_config": "standard", - "deployment": "develop", - "user": "test", - "password_hash": "9ece086e9bac491fac5c1d1046ca11d737b92a2b2ebd93f005d7b710110c0a678288166e7fbe796883a" - "4f2e9b3ca9f484f521d0ce464345cc1aec96779149c14" -} diff --git a/monkey/monkey_island/cc/test_consts.py b/monkey/monkey_island/cc/test_consts.py new file mode 100644 index 000000000..76a08a258 --- /dev/null +++ b/monkey/monkey_island/cc/test_consts.py @@ -0,0 +1,11 @@ +import platform +import monkey_island.cc.server_utils.consts as consts + + +def test_default_server_config_file_path(): + if platform.system() == "Windows": + server_file_path = consts.MONKEY_ISLAND_ABS_PATH + r"\cc\server_config.json" + else: + server_file_path = consts.MONKEY_ISLAND_ABS_PATH + "/cc/server_config.json" + + assert consts.DEFAULT_SERVER_CONFIG_PATH == server_file_path diff --git a/monkey/monkey_island/cc/test_encryptor.py b/monkey/monkey_island/cc/test_encryptor.py new file mode 100644 index 000000000..b2564b16c --- /dev/null +++ b/monkey/monkey_island/cc/test_encryptor.py @@ -0,0 +1,35 @@ +import os + +from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH +from monkey_island.cc.server_utils.encryptor import initialize_encryptor, get_encryptor + + +TEST_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing") +PASSWORD_FILENAME = "mongo_key.bin" + +PLAINTEXT = "Hello, Monkey!" +CYPHERTEXT = "vKgvD6SjRyIh1dh2AM/rnTa0NI/vjfwnbZLbMocWtE4e42WJmSUz2ordtbQrH1Fq" + + +def test_aes_cbc_encryption(): + initialize_encryptor(TEST_DATA_DIR) + + assert get_encryptor().enc(PLAINTEXT) != PLAINTEXT + + +def test_aes_cbc_decryption(): + initialize_encryptor(TEST_DATA_DIR) + + assert get_encryptor().dec(CYPHERTEXT) == PLAINTEXT + + +def test_aes_cbc_enc_dec(): + initialize_encryptor(TEST_DATA_DIR) + + assert get_encryptor().dec(get_encryptor().enc(PLAINTEXT)) == PLAINTEXT + + +def test_create_new_password_file(tmpdir): + initialize_encryptor(tmpdir) + + assert os.path.isfile(os.path.join(tmpdir, PASSWORD_FILENAME)) diff --git a/monkey/monkey_island/cc/testing/environment/server_config_no_credentials.json b/monkey/monkey_island/cc/testing/environment/server_config_no_credentials.json new file mode 100644 index 000000000..ecc4c1708 --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_no_credentials.json @@ -0,0 +1,4 @@ +{ + "server_config": "password", + "deployment": "develop" +} diff --git a/monkey/monkey_island/cc/testing/environment/server_config_partial_credentials.json b/monkey/monkey_island/cc/testing/environment/server_config_partial_credentials.json new file mode 100644 index 000000000..a9e283924 --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_partial_credentials.json @@ -0,0 +1,5 @@ +{ + "server_config": "password", + "deployment": "develop", + "user": "test" +} diff --git a/monkey/monkey_island/cc/testing/environment/server_config_standard_env.json b/monkey/monkey_island/cc/testing/environment/server_config_standard_env.json new file mode 100644 index 000000000..420f1b303 --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_standard_env.json @@ -0,0 +1,4 @@ +{ + "server_config": "standard", + "deployment": "develop" +} diff --git a/monkey/monkey_island/cc/testing/environment/server_config_standard_with_credentials.json b/monkey/monkey_island/cc/testing/environment/server_config_standard_with_credentials.json new file mode 100644 index 000000000..4bff379e8 --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_standard_with_credentials.json @@ -0,0 +1,6 @@ +{ + "server_config": "standard", + "deployment": "develop", + "user": "test", + "password_hash": "abcdef" +} diff --git a/monkey/monkey_island/cc/testing/environment/server_config_with_credentials.json b/monkey/monkey_island/cc/testing/environment/server_config_with_credentials.json new file mode 100644 index 000000000..54c0fa787 --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_with_credentials.json @@ -0,0 +1,6 @@ +{ + "server_config": "password", + "deployment": "develop", + "user": "test", + "password_hash": "abcdef" +} diff --git a/monkey/monkey_island/cc/testing/environment/server_config_with_data_dir.json b/monkey/monkey_island/cc/testing/environment/server_config_with_data_dir.json new file mode 100644 index 000000000..b9d6845f3 --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_with_data_dir.json @@ -0,0 +1,7 @@ +{ + "server_config": "password", + "deployment": "develop", + "user": "test", + "password_hash": "abcdef", + "data_dir": "/test/data/dir" +} diff --git a/monkey/monkey_island/cc/testing/environment/server_config_with_data_dir_home.json b/monkey/monkey_island/cc/testing/environment/server_config_with_data_dir_home.json new file mode 100644 index 000000000..e6e4a0a1f --- /dev/null +++ b/monkey/monkey_island/cc/testing/environment/server_config_with_data_dir_home.json @@ -0,0 +1,7 @@ +{ + "server_config": "password", + "deployment": "develop", + "user": "test", + "password_hash": "abcdef", + "data_dir": "~/data_dir" +} diff --git a/monkey/monkey_island/cc/testing/logger_config.json b/monkey/monkey_island/cc/testing/logger_config.json new file mode 100644 index 000000000..b3ad82641 --- /dev/null +++ b/monkey/monkey_island/cc/testing/logger_config.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(asctime)s - %(filename)s:%(lineno)s - %(funcName)10s() - %(levelname)s - %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "simple", + "stream": "ext://sys.stdout" + }, + "info_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "INFO", + "formatter": "simple", + "filename": "~/info.log", + "maxBytes": 10485760, + "backupCount": 20, + "encoding": "utf8" + } + }, + "root": { + "level": "DEBUG", + "handlers": [ + "console", + "info_file_handler" + ] + } +} diff --git a/monkey/monkey_island/cc/testing/mongo_key.bin b/monkey/monkey_island/cc/testing/mongo_key.bin new file mode 100644 index 000000000..6b8091efb --- /dev/null +++ b/monkey/monkey_island/cc/testing/mongo_key.bin @@ -0,0 +1,2 @@ ++ RO +)ꝞT|RS&C \ No newline at end of file