From 14b68580f8feb2e848d135b49ab75d48f299befe Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 1 Jun 2021 13:47:41 -0400 Subject: [PATCH 01/31] island: Configure loggin in python code instead of JSON --- .../cc/server_utils/island_logger.py | 71 +++++++++---------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/monkey/monkey_island/cc/server_utils/island_logger.py b/monkey/monkey_island/cc/server_utils/island_logger.py index c05915bc8..beb6df94f 100644 --- a/monkey/monkey_island/cc/server_utils/island_logger.py +++ b/monkey/monkey_island/cc/server_utils/island_logger.py @@ -1,38 +1,15 @@ -import logging.config +import logging +import logging.handlers import os -from copy import deepcopy +import sys ISLAND_LOG_FILENAME = "monkey_island.log" - -LOGGER_CONFIG_DICT = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "simple": { - "format": "%(asctime)s - %(filename)s:%(lineno)s - " - + "%(funcName)10s() - %(levelname)s - %(message)s" - } - }, - "handlers": { - "console_handler": { - "class": "logging.StreamHandler", - "formatter": "simple", - "stream": "ext://sys.stdout", - }, - "file_handler": { - "class": "logging.handlers.RotatingFileHandler", - "formatter": "simple", - "filename": None, # set in setup_logging() - "maxBytes": 10485760, - "backupCount": 20, - "encoding": "utf8", - }, - }, - "root": { - "level": None, # set in setup_logging() - "handlers": ["console_handler", "file_handler"], - }, -} +LOG_FORMAT = ( + "%(asctime)s - %(filename)s:%(lineno)s - %(funcName)10s() - %(levelname)s - %(message)s" +) +FILE_MAX_BYTES = 10485760 +FILE_BACKUP_COUNT = 20 +FILE_ENCODING = "utf8" def setup_logging(data_dir_path, log_level): @@ -42,12 +19,32 @@ def setup_logging(data_dir_path, log_level): :param log_level: level to log from :return: """ + logger = logging.getLogger() + logger.setLevel(log_level.upper()) - logger_configuration = deepcopy(LOGGER_CONFIG_DICT) + formatter = _get_log_formatter() - logger_configuration["handlers"]["file_handler"]["filename"] = os.path.join( - data_dir_path, ISLAND_LOG_FILENAME + log_file_path = os.path.join(data_dir_path, ISLAND_LOG_FILENAME) + _add_file_handler(logger, formatter, log_file_path) + + _add_console_handler(logger, formatter) + + +def _get_log_formatter(): + return logging.Formatter(LOG_FORMAT) + + +def _add_file_handler(logger, formatter, file_path): + fh = logging.handlers.RotatingFileHandler( + file_path, maxBytes=FILE_MAX_BYTES, backupCount=FILE_BACKUP_COUNT, encoding=FILE_ENCODING ) - logger_configuration["root"]["level"] = log_level.upper() + fh.setFormatter(formatter) - logging.config.dictConfig(logger_configuration) + logger.addHandler(fh) + + +def _add_console_handler(logger, formatter): + ch = logging.StreamHandler(stream=sys.stdout) + ch.setFormatter(formatter) + + logger.addHandler(ch) From 583115c419148c90e80200c5b11cade6fc5793e1 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 08:38:57 -0400 Subject: [PATCH 02/31] island: Reset logger handlers after each test The root logger does not get reset for each test that is run. Add an autouse fixture that resets the logger handlers after each test run so that handlers do not accumulate in the root logger. --- .../cc/server_utils/island_logger.py | 7 ++++++ .../cc/server_utils/test_island_logger.py | 25 +++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/monkey/monkey_island/cc/server_utils/island_logger.py b/monkey/monkey_island/cc/server_utils/island_logger.py index beb6df94f..9a8929e3a 100644 --- a/monkey/monkey_island/cc/server_utils/island_logger.py +++ b/monkey/monkey_island/cc/server_utils/island_logger.py @@ -48,3 +48,10 @@ def _add_console_handler(logger, formatter): ch.setFormatter(formatter) logger.addHandler(ch) + + +def reset_logger(): + logger = logging.getLogger() + + for handler in logger.handlers: + logger.removeHandler(handler) diff --git a/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_island_logger.py b/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_island_logger.py index 9f4e59af8..6061109d1 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_island_logger.py +++ b/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_island_logger.py @@ -1,16 +1,25 @@ import logging import os -from monkey_island.cc.server_utils.island_logger import ISLAND_LOG_FILENAME, setup_logging +import pytest + +import monkey_island.cc.server_utils.island_logger as island_logger + + +@pytest.fixture(autouse=True) +def reset_logger(): + yield + + island_logger.reset_logger() def test_setup_logging_file_log_level_debug(tmpdir): DATA_DIR = tmpdir - LOG_FILE = os.path.join(DATA_DIR, ISLAND_LOG_FILENAME) + LOG_FILE = os.path.join(DATA_DIR, island_logger.ISLAND_LOG_FILENAME) LOG_LEVEL = "DEBUG" TEST_STRING = "Hello, Monkey! (File; Log level: debug)" - setup_logging(DATA_DIR, LOG_LEVEL) + island_logger.setup_logging(DATA_DIR, LOG_LEVEL) logger = logging.getLogger("TestLogger") logger.debug(TEST_STRING) @@ -23,11 +32,11 @@ def test_setup_logging_file_log_level_debug(tmpdir): def test_setup_logging_file_log_level_info(tmpdir): DATA_DIR = tmpdir - LOG_FILE = os.path.join(DATA_DIR, ISLAND_LOG_FILENAME) + LOG_FILE = os.path.join(DATA_DIR, island_logger.ISLAND_LOG_FILENAME) LOG_LEVEL = "INFO" TEST_STRING = "Hello, Monkey! (File; Log level: info)" - setup_logging(DATA_DIR, LOG_LEVEL) + island_logger.setup_logging(DATA_DIR, LOG_LEVEL) logger = logging.getLogger("TestLogger") logger.debug(TEST_STRING) @@ -43,7 +52,7 @@ def test_setup_logging_console_log_level_debug(capsys, tmpdir): LOG_LEVEL = "DEBUG" TEST_STRING = "Hello, Monkey! (Console; Log level: debug)" - setup_logging(DATA_DIR, LOG_LEVEL) + island_logger.setup_logging(DATA_DIR, LOG_LEVEL) logger = logging.getLogger("TestLogger") logger.debug(TEST_STRING) @@ -57,7 +66,7 @@ def test_setup_logging_console_log_level_info(capsys, tmpdir): LOG_LEVEL = "INFO" TEST_STRING = "Hello, Monkey! (Console; Log level: info)" - setup_logging(DATA_DIR, LOG_LEVEL) + island_logger.setup_logging(DATA_DIR, LOG_LEVEL) logger = logging.getLogger("TestLogger") logger.debug(TEST_STRING) @@ -71,7 +80,7 @@ def test_setup_logging_console_log_level_lower_case(capsys, tmpdir): LOG_LEVEL = "debug" TEST_STRING = "Hello, Monkey! (Console; Log level: debug)" - setup_logging(DATA_DIR, LOG_LEVEL) + island_logger.setup_logging(DATA_DIR, LOG_LEVEL) logger = logging.getLogger("TestLogger") logger.debug(TEST_STRING) From 5e78666f91cebebc9e0ff9f6462d78107234ecce Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 08:40:49 -0400 Subject: [PATCH 03/31] island: Add function to setup default failsafe logger --- .../monkey_island/cc/server_utils/island_logger.py | 9 +++++++++ .../cc/server_utils/test_island_logger.py | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/monkey/monkey_island/cc/server_utils/island_logger.py b/monkey/monkey_island/cc/server_utils/island_logger.py index 9a8929e3a..06547d84d 100644 --- a/monkey/monkey_island/cc/server_utils/island_logger.py +++ b/monkey/monkey_island/cc/server_utils/island_logger.py @@ -30,6 +30,15 @@ def setup_logging(data_dir_path, log_level): _add_console_handler(logger, formatter) +def setup_default_failsafe_logging(): + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + + formatter = _get_log_formatter() + + _add_console_handler(logger, formatter) + + def _get_log_formatter(): return logging.Formatter(LOG_FORMAT) diff --git a/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_island_logger.py b/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_island_logger.py index 6061109d1..c4256252f 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_island_logger.py +++ b/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_island_logger.py @@ -87,3 +87,15 @@ def test_setup_logging_console_log_level_lower_case(capsys, tmpdir): captured = capsys.readouterr() assert TEST_STRING in captured.out + + +def test_setup_defailt_failsafe_logging(capsys): + TEST_STRING = "Hello, Monkey! (Console; Log level: debug)" + + island_logger.setup_default_failsafe_logging() + logger = logging.getLogger("TestLogger") + logger.debug(TEST_STRING) + + captured = capsys.readouterr() + assert TEST_STRING in captured.out + assert "DEBUG" in captured.out From 184594f5092bda55c4c3b58085c3b1de7de10d92 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 09:16:00 -0400 Subject: [PATCH 04/31] island: Refactor setup/start logic separate concerns Most configuration/setup logic is removed from monkey_island.py. The only responsibility of startup.py is now to setup a default failsafe logger and call run_monkey_island(). setup_island() has been renamed to run_monkey_island(), since that's what it does. Some of the logic used to setup/configure/initialize monkey island has been wrapped in different functions with specific responsibilities. setup_mongodb() has been renamed to start_mongodb(), since that's what it does. --- monkey/monkey_island/cc/server_setup.py | 36 ++++++++++++++----- .../cc/setup/mongo/mongo_setup.py | 2 +- monkey/monkey_island/setup/config_setup.py | 12 +++++-- monkey/monkey_island/startup.py | 27 ++++---------- 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index 208055173..a989f6387 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -15,39 +15,59 @@ if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path: import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 from common.version import get_version # noqa: E402 from monkey_island.cc.app import init_app # noqa: E402 +from monkey_island.cc.arg_parser import parse_cli_args # noqa: E402 from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH # noqa: E402 from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402 +from monkey_island.cc.server_utils.island_logger import reset_logger, setup_logging # noqa: E402 from monkey_island.cc.services.initialize import initialize_services # noqa: E402 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 from monkey_island.cc.setup.mongo.database_initializer import init_collections # noqa: E402 -from monkey_island.cc.setup.mongo.mongo_setup import MONGO_URL, setup_mongodb # noqa: E402 +from monkey_island.cc.setup.mongo.mongo_setup import MONGO_URL, start_mongodb # noqa: E402 +from monkey_island.setup.config_setup import setup_data_dir # noqa: E402 from monkey_island.setup.island_config_options import IslandConfigOptions # noqa: E402 logger = logging.getLogger(__name__) -def setup_island(setup_only: bool, config_options: IslandConfigOptions, server_config_path: str): +def run_monkey_island(): + island_args = parse_cli_args() + config_options, server_config_path = setup_data_dir(island_args) + + _configure_logging(config_options) + _initialize_global_resources(config_options, server_config_path) + + start_mongodb(config_options) + bootloader_server_thread = _start_bootloader_server(MONGO_URL) + _start_island_server(island_args.setup_only, config_options) + bootloader_server_thread.join() + + +def _configure_logging(config_options): + reset_logger() + setup_logging(config_options.data_dir, config_options.log_level) + + +def _initialize_global_resources(config_options: IslandConfigOptions, server_config_path: str): env_singleton.initialize_from_file(server_config_path) initialize_encryptor(config_options.data_dir) initialize_services(config_options.data_dir) + +def _start_bootloader_server(mongo_url) -> Thread: bootloader_server_thread = Thread( - target=BootloaderHttpServer(MONGO_URL).serve_forever, daemon=True + target=BootloaderHttpServer(mongo_url).serve_forever, daemon=True ) bootloader_server_thread.start() - _start_island_server(setup_only, config_options) - bootloader_server_thread.join() + + return bootloader_server_thread def _start_island_server(should_setup_only, config_options: IslandConfigOptions): - - setup_mongodb(config_options) - populate_exporter_list() app = init_app(MONGO_URL) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py index ad523ce98..d5001dd1b 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py @@ -18,7 +18,7 @@ MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" logger = logging.getLogger(__name__) -def setup_mongodb(config_options: IslandConfigOptions): +def start_mongodb(config_options: IslandConfigOptions): if config_options.start_mongodb: MongoDbRunner( db_dir_parent_path=config_options.data_dir, logging_dir_path=config_options.data_dir diff --git a/monkey/monkey_island/setup/config_setup.py b/monkey/monkey_island/setup/config_setup.py index 50330aea3..95ebbc194 100644 --- a/monkey/monkey_island/setup/config_setup.py +++ b/monkey/monkey_island/setup/config_setup.py @@ -1,20 +1,28 @@ import os from typing import Tuple +from monkey_island.cc.arg_parser import IslandCmdArgs from monkey_island.cc.environment import server_config_handler from monkey_island.cc.environment.utils import create_secure_directory from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH from monkey_island.setup.island_config_options import IslandConfigOptions -def setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, str]: +def setup_data_dir(island_args: IslandCmdArgs): + if island_args.server_config_path: + return _setup_config_by_cmd_arg(island_args.server_config_path) + + return _setup_default_config() + + +def _setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, str]: server_config_path = os.path.expandvars(os.path.expanduser(server_config_path)) config = server_config_handler.load_server_config_from_file(server_config_path) create_secure_directory(config.data_dir, create_parent_dirs=True) return config, server_config_path -def setup_default_config() -> Tuple[IslandConfigOptions, str]: +def _setup_default_config() -> Tuple[IslandConfigOptions, str]: server_config_path = DEFAULT_SERVER_CONFIG_PATH create_secure_directory(DEFAULT_DATA_DIR, create_parent_dirs=False) server_config_handler.create_default_server_config_file() diff --git a/monkey/monkey_island/startup.py b/monkey/monkey_island/startup.py index f53c679ab..90dcd577e 100644 --- a/monkey/monkey_island/startup.py +++ b/monkey/monkey_island/startup.py @@ -1,36 +1,21 @@ # This import patches other imports and needs to be first import monkey_island.setup.gevent_setup # noqa: F401 isort:skip -import json - -from monkey_island.cc.arg_parser import parse_cli_args -from monkey_island.cc.server_setup import setup_island -from monkey_island.cc.server_utils.island_logger import setup_logging -from monkey_island.setup.config_setup import setup_config_by_cmd_arg, setup_default_config +from monkey_island.cc.server_utils.island_logger import setup_default_failsafe_logging def start_island(): - island_args = parse_cli_args() - # This is here in order to catch EVERYTHING, some functions are being called on # imports, so the log init needs to be first. try: - if island_args.server_config_path: - config, server_config_path = setup_config_by_cmd_arg(island_args.server_config_path) - else: - config, server_config_path = setup_default_config() - - setup_logging(config.data_dir, config.log_level) - - except OSError as ex: - print(f"Error opening server config file: {ex}") + setup_default_failsafe_logging() + except Exception as ex: + print(f"Error configuring logging: {ex}") exit(1) - except json.JSONDecodeError as ex: - print(f"Error loading server config: {ex}") - exit(1) + from monkey_island.cc.server_setup import run_monkey_island # noqa: E402 - setup_island(island_args.setup_only, config, server_config_path) + run_monkey_island() if "__main__" == __name__: From a8570987a6133ce4eee5be13dd1f7d2a75943adb Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 1 Jun 2021 19:40:11 -0400 Subject: [PATCH 05/31] island: Find MONKEY_ISLAND_ABS_PATH by __file__ instead of os.getcwd() We can't be sure what $PWD is when Monkey Island is started. --- monkey/monkey_island/cc/server_utils/consts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 5cc9a0dd1..6f4382b87 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -1,4 +1,5 @@ import os +from pathlib import Path from monkey_island.cc.environment.utils import is_windows_os @@ -14,7 +15,7 @@ def get_default_data_dir() -> str: SERVER_CONFIG_FILENAME = "server_config.json" -MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island") +MONKEY_ISLAND_ABS_PATH = str(Path(__file__).resolve().parent.parent.parent) DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir()) From 28a34a4ec98a210c7508ad4801c0559c32a57e34 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 09:54:46 -0400 Subject: [PATCH 06/31] island: Use MONKEY_ISLAND_ABS_PATH to locate STIX attack data --- .../cc/services/attack/mitre_api_interface.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/services/attack/mitre_api_interface.py b/monkey/monkey_island/cc/services/attack/mitre_api_interface.py index 1a3025233..596f4d498 100644 --- a/monkey/monkey_island/cc/services/attack/mitre_api_interface.py +++ b/monkey/monkey_island/cc/services/attack/mitre_api_interface.py @@ -1,10 +1,15 @@ +import os from typing import Dict, List from stix2 import AttackPattern, CourseOfAction, FileSystemSource, Filter +from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH + class MitreApiInterface: - ATTACK_DATA_PATH = "monkey_island/cc/services/attack/attack_data/enterprise-attack" + ATTACK_DATA_PATH = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", "services", "attack", "attack_data", "enterprise-attack" + ) @staticmethod def get_all_mitigations() -> Dict[str, CourseOfAction]: From d35099fa9b72c553df6d75c771e32a87b988eed2 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 13:24:18 -0400 Subject: [PATCH 07/31] island: Rename MongoDbRunner -> MongoDbProcess --- .../mongo/{mongo_process_runner.py => mongo_db_process.py} | 4 ++-- monkey/monkey_island/cc/setup/mongo/mongo_setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename monkey/monkey_island/cc/setup/mongo/{mongo_process_runner.py => mongo_db_process.py} (93%) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_process_runner.py b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py similarity index 93% rename from monkey/monkey_island/cc/setup/mongo/mongo_process_runner.py rename to monkey/monkey_island/cc/setup/mongo/mongo_db_process.py index d03b62913..4b0150d99 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_process_runner.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py @@ -13,7 +13,7 @@ DB_DIR_PARAM = "--dbpath" MONGO_LOG_FILENAME = "mongo_log.txt" -class MongoDbRunner: +class MongoDbProcess: def __init__(self, db_dir_parent_path: str, logging_dir_path: str): """ @param db_dir_parent_path: Path where a folder for database contents will be created @@ -35,7 +35,7 @@ class MongoDbRunner: def _start_mongodb_process(self, db_dir_path: str): logger.info("Starting MongoDb process.") - mongo_run_cmd = MongoDbRunner._build_mongo_launch_cmd(MONGO_EXECUTABLE_PATH, db_dir_path) + mongo_run_cmd = MongoDbProcess._build_mongo_launch_cmd(MONGO_EXECUTABLE_PATH, db_dir_path) logger.info(f"Mongodb will be launched with command: {' '.join(mongo_run_cmd)}.") mongo_log_path = os.path.join(self.logging_dir_path, MONGO_LOG_FILENAME) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py index d5001dd1b..5accd6f65 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py @@ -6,7 +6,7 @@ import time from monkey_island.cc.database import get_db_version, is_db_server_up from monkey_island.cc.setup.mongo import mongo_connector from monkey_island.cc.setup.mongo.mongo_connector import MONGO_DB_HOST, MONGO_DB_NAME, MONGO_DB_PORT -from monkey_island.cc.setup.mongo.mongo_process_runner import MongoDbRunner +from monkey_island.cc.setup.mongo.mongo_db_process import MongoDbProcess from monkey_island.setup.island_config_options import IslandConfigOptions MONGO_URL = os.environ.get( @@ -20,7 +20,7 @@ logger = logging.getLogger(__name__) def start_mongodb(config_options: IslandConfigOptions): if config_options.start_mongodb: - MongoDbRunner( + MongoDbProcess( db_dir_parent_path=config_options.data_dir, logging_dir_path=config_options.data_dir ).launch_mongodb() wait_for_mongo_db_server(MONGO_URL) From dc40713683dd76419d680fd9eafe685650dd0cb4 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 13:24:58 -0400 Subject: [PATCH 08/31] island: Rename launch_mongodb() -> start() --- monkey/monkey_island/cc/setup/mongo/mongo_db_process.py | 2 +- monkey/monkey_island/cc/setup/mongo/mongo_setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py index 4b0150d99..c8b043249 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py @@ -22,7 +22,7 @@ class MongoDbProcess: self.db_dir_parent_path = db_dir_parent_path self.logging_dir_path = logging_dir_path - def launch_mongodb(self): + def start(self): db_path = self._create_db_dir() self._start_mongodb_process(db_path) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py index 5accd6f65..b6e86a3e3 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py @@ -22,7 +22,7 @@ def start_mongodb(config_options: IslandConfigOptions): if config_options.start_mongodb: MongoDbProcess( db_dir_parent_path=config_options.data_dir, logging_dir_path=config_options.data_dir - ).launch_mongodb() + ).start() wait_for_mongo_db_server(MONGO_URL) assert_mongo_db_version(MONGO_URL) mongo_connector.connect_dal_to_mongodb() From d1a2501a5bbfdd391e9b1d7edd303a05118b9c4d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 14:05:40 -0400 Subject: [PATCH 09/31] island: Add connect_to_mongo() function --- monkey/monkey_island/cc/server_setup.py | 8 +++++++- monkey/monkey_island/cc/setup/mongo/mongo_setup.py | 11 +++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index a989f6387..e6895ac59 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -25,7 +25,11 @@ from monkey_island.cc.services.initialize import initialize_services # noqa: E4 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 from monkey_island.cc.setup.mongo.database_initializer import init_collections # noqa: E402 -from monkey_island.cc.setup.mongo.mongo_setup import MONGO_URL, start_mongodb # noqa: E402 +from monkey_island.cc.setup.mongo.mongo_setup import ( # noqa: E402 + MONGO_URL, + connect_to_mongodb, + start_mongodb, +) from monkey_island.setup.config_setup import setup_data_dir # noqa: E402 from monkey_island.setup.island_config_options import IslandConfigOptions # noqa: E402 @@ -40,6 +44,8 @@ def run_monkey_island(): _initialize_global_resources(config_options, server_config_path) start_mongodb(config_options) + connect_to_mongodb() + bootloader_server_thread = _start_bootloader_server(MONGO_URL) _start_island_server(island_args.setup_only, config_options) bootloader_server_thread.join() diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py index b6e86a3e3..7d83ee5f4 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py @@ -23,18 +23,21 @@ def start_mongodb(config_options: IslandConfigOptions): MongoDbProcess( db_dir_parent_path=config_options.data_dir, logging_dir_path=config_options.data_dir ).start() - wait_for_mongo_db_server(MONGO_URL) - assert_mongo_db_version(MONGO_URL) + + +def connect_to_mongodb(): + _wait_for_mongo_db_server(MONGO_URL) + _assert_mongo_db_version(MONGO_URL) mongo_connector.connect_dal_to_mongodb() -def wait_for_mongo_db_server(mongo_url): +def _wait_for_mongo_db_server(mongo_url): while not is_db_server_up(mongo_url): logger.info("Waiting for MongoDB server on {0}".format(mongo_url)) time.sleep(1) -def assert_mongo_db_version(mongo_url): +def _assert_mongo_db_version(mongo_url): """ Checks if the mongodb version is new enough for running the app. If the DB is too old, quits. From 559af47928e0edce27d194cf9e300c78d4ee4e00 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 14:20:21 -0400 Subject: [PATCH 10/31] island: Clean up MongoDB subprocess when Monkey Island shuts down --- monkey/monkey_island/cc/server_setup.py | 6 +++++- .../cc/setup/mongo/mongo_db_process.py | 17 ++++++++++++++++- .../cc/setup/mongo/mongo_setup.py | 18 +++++++++++++----- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index e6895ac59..ba0ed1411 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -28,6 +28,7 @@ from monkey_island.cc.setup.mongo.database_initializer import init_collections from monkey_island.cc.setup.mongo.mongo_setup import ( # noqa: E402 MONGO_URL, connect_to_mongodb, + register_mongo_shutdown_callback, start_mongodb, ) from monkey_island.setup.config_setup import setup_data_dir # noqa: E402 @@ -43,7 +44,10 @@ def run_monkey_island(): _configure_logging(config_options) _initialize_global_resources(config_options, server_config_path) - start_mongodb(config_options) + if config_options.start_mongodb: + mongo_db_process = start_mongodb(config_options) + register_mongo_shutdown_callback(mongo_db_process) + connect_to_mongodb() bootloader_server_thread = _start_bootloader_server(MONGO_URL) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py index c8b043249..f24406630 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py @@ -11,6 +11,7 @@ logger = logging.getLogger(__name__) DB_DIR_NAME = "db" DB_DIR_PARAM = "--dbpath" MONGO_LOG_FILENAME = "mongo_log.txt" +TERMINATE_TIMEOUT = 10 class MongoDbProcess: @@ -21,11 +22,25 @@ class MongoDbProcess: """ self.db_dir_parent_path = db_dir_parent_path self.logging_dir_path = logging_dir_path + self._process = None def start(self): db_path = self._create_db_dir() self._start_mongodb_process(db_path) + def stop(self): + if self._process: + logger.info("Terminating MongoDB process") + self._process.terminate() + try: + self._process.wait(timeout=TERMINATE_TIMEOUT) + logger.info("MongoDB process terminated successfully") + except subprocess.TimeoutExpired as te: + logger.warning( + f"MongoDB did not terminate gracefully and will be forcefully killed: {te}" + ) + self._process.kill() + def _create_db_dir(self) -> str: db_path = os.path.join(self.db_dir_parent_path, DB_DIR_NAME) logger.info(f"Database content directory: {db_path}.") @@ -42,7 +57,7 @@ class MongoDbProcess: logger.info(f"Mongodb log will be available at {mongo_log_path}.") with open(mongo_log_path, "w") as log: - subprocess.Popen(mongo_run_cmd, stderr=subprocess.STDOUT, stdout=log) + self._process = subprocess.Popen(mongo_run_cmd, stderr=subprocess.STDOUT, stdout=log) logger.info("MongoDb launched successfully!") @staticmethod diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py index 7d83ee5f4..412ff49d6 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py @@ -1,3 +1,4 @@ +import atexit import logging import os import sys @@ -18,11 +19,18 @@ MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" logger = logging.getLogger(__name__) -def start_mongodb(config_options: IslandConfigOptions): - if config_options.start_mongodb: - MongoDbProcess( - db_dir_parent_path=config_options.data_dir, logging_dir_path=config_options.data_dir - ).start() +def start_mongodb(config_options: IslandConfigOptions) -> MongoDbProcess: + mongo_db_process = MongoDbProcess( + db_dir_parent_path=config_options.data_dir, logging_dir_path=config_options.data_dir + ) + + mongo_db_process.start() + + return mongo_db_process + + +def register_mongo_shutdown_callback(mongo_db_process: MongoDbProcess): + atexit.register(mongo_db_process.stop) def connect_to_mongodb(): From 7d85ce06112cd8cb213cc0d66aaf1dbed22b887d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 14:27:01 -0400 Subject: [PATCH 11/31] island: Store MongoDB logs in mondogb.log instead of mongo_log.txt ".log" is the standard extension for log files. --- monkey/monkey_island/cc/setup/mongo/mongo_db_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py index f24406630..c283710fc 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) DB_DIR_NAME = "db" DB_DIR_PARAM = "--dbpath" -MONGO_LOG_FILENAME = "mongo_log.txt" +MONGO_LOG_FILENAME = "mongodb.log" TERMINATE_TIMEOUT = 10 From a3bd4325389b4e7bfcf6c42e9c29c29eaa450bab Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 15:12:30 -0400 Subject: [PATCH 12/31] island: Remove directory creation logic from MongoDbProcess --- .../cc/setup/mongo/mongo_db_process.py | 25 ++++++------------- .../cc/setup/mongo/mongo_setup.py | 15 ++++++++--- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py index c283710fc..9a514ab9a 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py @@ -3,30 +3,27 @@ import os import subprocess from typing import List -from monkey_island.cc.environment.utils import create_secure_directory from monkey_island.cc.server_utils.consts import MONGO_EXECUTABLE_PATH logger = logging.getLogger(__name__) -DB_DIR_NAME = "db" DB_DIR_PARAM = "--dbpath" MONGO_LOG_FILENAME = "mongodb.log" TERMINATE_TIMEOUT = 10 class MongoDbProcess: - def __init__(self, db_dir_parent_path: str, logging_dir_path: str): + def __init__(self, db_dir: str, logging_dir_path: str): """ - @param db_dir_parent_path: Path where a folder for database contents will be created + @param db_dir: Path where a folder for database contents will be created @param logging_dir_path: Path to a folder where mongodb logs will be created """ - self.db_dir_parent_path = db_dir_parent_path + self._db_dir = db_dir self.logging_dir_path = logging_dir_path self._process = None def start(self): - db_path = self._create_db_dir() - self._start_mongodb_process(db_path) + self._start_mongodb_process() def stop(self): if self._process: @@ -41,16 +38,10 @@ class MongoDbProcess: ) self._process.kill() - def _create_db_dir(self) -> str: - db_path = os.path.join(self.db_dir_parent_path, DB_DIR_NAME) - logger.info(f"Database content directory: {db_path}.") - create_secure_directory(db_path, create_parent_dirs=False) - return db_path - - def _start_mongodb_process(self, db_dir_path: str): + def _start_mongodb_process(self): logger.info("Starting MongoDb process.") - mongo_run_cmd = MongoDbProcess._build_mongo_launch_cmd(MONGO_EXECUTABLE_PATH, db_dir_path) + mongo_run_cmd = MongoDbProcess._build_mongo_launch_cmd(MONGO_EXECUTABLE_PATH, self._db_dir) logger.info(f"Mongodb will be launched with command: {' '.join(mongo_run_cmd)}.") mongo_log_path = os.path.join(self.logging_dir_path, MONGO_LOG_FILENAME) @@ -61,5 +52,5 @@ class MongoDbProcess: logger.info("MongoDb launched successfully!") @staticmethod - def _build_mongo_launch_cmd(exec_path: str, db_path: str) -> List[str]: - return [exec_path, DB_DIR_PARAM, db_path] + def _build_mongo_launch_cmd(exec_path: str, db_dir: str) -> List[str]: + return [exec_path, DB_DIR_PARAM, db_dir] diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py index 412ff49d6..73ed9d096 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py @@ -5,11 +5,13 @@ import sys import time from monkey_island.cc.database import get_db_version, is_db_server_up +from monkey_island.cc.environment.utils import create_secure_directory from monkey_island.cc.setup.mongo import mongo_connector from monkey_island.cc.setup.mongo.mongo_connector import MONGO_DB_HOST, MONGO_DB_NAME, MONGO_DB_PORT from monkey_island.cc.setup.mongo.mongo_db_process import MongoDbProcess from monkey_island.setup.island_config_options import IslandConfigOptions +DB_DIR_NAME = "db" MONGO_URL = os.environ.get( "MONKEY_MONGO_URL", "mongodb://{0}:{1}/{2}".format(MONGO_DB_HOST, MONGO_DB_PORT, MONGO_DB_NAME), @@ -20,15 +22,22 @@ logger = logging.getLogger(__name__) def start_mongodb(config_options: IslandConfigOptions) -> MongoDbProcess: - mongo_db_process = MongoDbProcess( - db_dir_parent_path=config_options.data_dir, logging_dir_path=config_options.data_dir - ) + db_dir = _create_db_dir(config_options.data_dir) + mongo_db_process = MongoDbProcess(db_dir=db_dir, logging_dir_path=config_options.data_dir) mongo_db_process.start() return mongo_db_process +def _create_db_dir(db_dir_parent_path) -> str: + db_dir = os.path.join(db_dir_parent_path, DB_DIR_NAME) + logger.info(f"Database content directory: {db_dir}.") + + create_secure_directory(db_dir, create_parent_dirs=False) + return db_dir + + def register_mongo_shutdown_callback(mongo_db_process: MongoDbProcess): atexit.register(mongo_db_process.stop) From 1e21446bb8565bf18d5a089a2902f59bbcd1c082 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 15:16:18 -0400 Subject: [PATCH 13/31] island: Rename logging_dir_path -> _logging_dir in MongoDbProcess --- monkey/monkey_island/cc/setup/mongo/mongo_db_process.py | 8 ++++---- monkey/monkey_island/cc/setup/mongo/mongo_setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py index 9a514ab9a..96341b7d8 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py @@ -13,13 +13,13 @@ TERMINATE_TIMEOUT = 10 class MongoDbProcess: - def __init__(self, db_dir: str, logging_dir_path: str): + def __init__(self, db_dir: str, logging_dir: str): """ @param db_dir: Path where a folder for database contents will be created - @param logging_dir_path: Path to a folder where mongodb logs will be created + @param logging_dir: Path to a folder where mongodb logs will be created """ self._db_dir = db_dir - self.logging_dir_path = logging_dir_path + self._logging_dir = logging_dir self._process = None def start(self): @@ -44,7 +44,7 @@ class MongoDbProcess: mongo_run_cmd = MongoDbProcess._build_mongo_launch_cmd(MONGO_EXECUTABLE_PATH, self._db_dir) logger.info(f"Mongodb will be launched with command: {' '.join(mongo_run_cmd)}.") - mongo_log_path = os.path.join(self.logging_dir_path, MONGO_LOG_FILENAME) + mongo_log_path = os.path.join(self._logging_dir, MONGO_LOG_FILENAME) logger.info(f"Mongodb log will be available at {mongo_log_path}.") with open(mongo_log_path, "w") as log: diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py index 73ed9d096..2920fc8bb 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) def start_mongodb(config_options: IslandConfigOptions) -> MongoDbProcess: db_dir = _create_db_dir(config_options.data_dir) - mongo_db_process = MongoDbProcess(db_dir=db_dir, logging_dir_path=config_options.data_dir) + mongo_db_process = MongoDbProcess(db_dir=db_dir, logging_dir=config_options.data_dir) mongo_db_process.start() return mongo_db_process From 3b958f5a6169555fc2d43c75e0b384c25a21e8c3 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 15:18:07 -0400 Subject: [PATCH 14/31] island: Inline method _start_mongodb_process() in MongoDbProcess --- .../cc/setup/mongo/mongo_db_process.py | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py index 96341b7d8..3ada3bdb1 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py @@ -23,7 +23,17 @@ class MongoDbProcess: self._process = None def start(self): - self._start_mongodb_process() + logger.info("Starting MongoDb process.") + + mongo_run_cmd = MongoDbProcess._build_mongo_launch_cmd(MONGO_EXECUTABLE_PATH, self._db_dir) + logger.info(f"Mongodb will be launched with command: {' '.join(mongo_run_cmd)}.") + + mongo_log_path = os.path.join(self._logging_dir, MONGO_LOG_FILENAME) + logger.info(f"Mongodb log will be available at {mongo_log_path}.") + + with open(mongo_log_path, "w") as log: + self._process = subprocess.Popen(mongo_run_cmd, stderr=subprocess.STDOUT, stdout=log) + logger.info("MongoDb launched successfully!") def stop(self): if self._process: @@ -38,19 +48,6 @@ class MongoDbProcess: ) self._process.kill() - def _start_mongodb_process(self): - logger.info("Starting MongoDb process.") - - mongo_run_cmd = MongoDbProcess._build_mongo_launch_cmd(MONGO_EXECUTABLE_PATH, self._db_dir) - logger.info(f"Mongodb will be launched with command: {' '.join(mongo_run_cmd)}.") - - mongo_log_path = os.path.join(self._logging_dir, MONGO_LOG_FILENAME) - logger.info(f"Mongodb log will be available at {mongo_log_path}.") - - with open(mongo_log_path, "w") as log: - self._process = subprocess.Popen(mongo_run_cmd, stderr=subprocess.STDOUT, stdout=log) - logger.info("MongoDb launched successfully!") - @staticmethod def _build_mongo_launch_cmd(exec_path: str, db_dir: str) -> List[str]: return [exec_path, DB_DIR_PARAM, db_dir] From 12b11ce260543711671ae0d2bc47da9c55f0c3a3 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 15:19:42 -0400 Subject: [PATCH 15/31] island: Rename _build_mongo_launch_cmd -> _build_mongo_run_cmd --- monkey/monkey_island/cc/setup/mongo/mongo_db_process.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py index 3ada3bdb1..29951b1d6 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py @@ -25,7 +25,7 @@ class MongoDbProcess: def start(self): logger.info("Starting MongoDb process.") - mongo_run_cmd = MongoDbProcess._build_mongo_launch_cmd(MONGO_EXECUTABLE_PATH, self._db_dir) + mongo_run_cmd = MongoDbProcess._build_mongo_run_cmd(MONGO_EXECUTABLE_PATH, self._db_dir) logger.info(f"Mongodb will be launched with command: {' '.join(mongo_run_cmd)}.") mongo_log_path = os.path.join(self._logging_dir, MONGO_LOG_FILENAME) @@ -49,5 +49,5 @@ class MongoDbProcess: self._process.kill() @staticmethod - def _build_mongo_launch_cmd(exec_path: str, db_dir: str) -> List[str]: + def _build_mongo_run_cmd(exec_path: str, db_dir: str) -> List[str]: return [exec_path, DB_DIR_PARAM, db_dir] From e80ac4c9439034e8f546508b1b08cb1038e274f6 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 15:32:43 -0400 Subject: [PATCH 16/31] island; Build log file path in MongoDbProcess constructor --- .../cc/setup/mongo/mongo_db_process.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py index 29951b1d6..925405ba5 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py @@ -19,21 +19,21 @@ class MongoDbProcess: @param logging_dir: Path to a folder where mongodb logs will be created """ self._db_dir = db_dir - self._logging_dir = logging_dir + self._log_file = os.path.join(logging_dir, MONGO_LOG_FILENAME) self._process = None def start(self): - logger.info("Starting MongoDb process.") + logger.info("Starting MongoDB process.") mongo_run_cmd = MongoDbProcess._build_mongo_run_cmd(MONGO_EXECUTABLE_PATH, self._db_dir) - logger.info(f"Mongodb will be launched with command: {' '.join(mongo_run_cmd)}.") - mongo_log_path = os.path.join(self._logging_dir, MONGO_LOG_FILENAME) - logger.info(f"Mongodb log will be available at {mongo_log_path}.") + logger.info(f"MongoDB will be launched with command: {' '.join(mongo_run_cmd)}.") + logger.info(f"MongoDB log will be available at {self._log_file}.") - with open(mongo_log_path, "w") as log: + with open(self._log_file, "w") as log: self._process = subprocess.Popen(mongo_run_cmd, stderr=subprocess.STDOUT, stdout=log) - logger.info("MongoDb launched successfully!") + + logger.info("MongoDB launched successfully!") def stop(self): if self._process: From cc1865dc5b470753b2e7d2e730fedc4bed63e6ee Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 15:35:46 -0400 Subject: [PATCH 17/31] island: Log a warning if MongoDbProcess.stop() is erroniously called --- .../cc/setup/mongo/mongo_db_process.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py index 925405ba5..0d517bf6d 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py @@ -36,17 +36,21 @@ class MongoDbProcess: logger.info("MongoDB launched successfully!") def stop(self): - if self._process: - logger.info("Terminating MongoDB process") - self._process.terminate() - try: - self._process.wait(timeout=TERMINATE_TIMEOUT) - logger.info("MongoDB process terminated successfully") - except subprocess.TimeoutExpired as te: - logger.warning( - f"MongoDB did not terminate gracefully and will be forcefully killed: {te}" - ) - self._process.kill() + if not self._process: + logger.warning("Failed to stop MongoDB process: No process found") + return + + logger.info("Terminating MongoDB process") + self._process.terminate() + + try: + self._process.wait(timeout=TERMINATE_TIMEOUT) + logger.info("MongoDB process terminated successfully") + except subprocess.TimeoutExpired as te: + logger.warning( + f"MongoDB did not terminate gracefully and will be forcefully killed: {te}" + ) + self._process.kill() @staticmethod def _build_mongo_run_cmd(exec_path: str, db_dir: str) -> List[str]: From 19e8042ee49bd7c067e3bf944b7949955919c52e Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 15:39:49 -0400 Subject: [PATCH 18/31] island: Construct mongo run command in MongoDbProcess constructor --- .../cc/setup/mongo/mongo_db_process.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py index 0d517bf6d..61fc6943a 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py @@ -1,7 +1,6 @@ import logging import os import subprocess -from typing import List from monkey_island.cc.server_utils.consts import MONGO_EXECUTABLE_PATH @@ -18,20 +17,19 @@ class MongoDbProcess: @param db_dir: Path where a folder for database contents will be created @param logging_dir: Path to a folder where mongodb logs will be created """ - self._db_dir = db_dir + self._mongo_run_cmd = [MONGO_EXECUTABLE_PATH, DB_DIR_PARAM, db_dir] self._log_file = os.path.join(logging_dir, MONGO_LOG_FILENAME) self._process = None def start(self): logger.info("Starting MongoDB process.") - - mongo_run_cmd = MongoDbProcess._build_mongo_run_cmd(MONGO_EXECUTABLE_PATH, self._db_dir) - - logger.info(f"MongoDB will be launched with command: {' '.join(mongo_run_cmd)}.") + logger.debug(f"MongoDB will be launched with command: {' '.join(self._mongo_run_cmd)}.") logger.info(f"MongoDB log will be available at {self._log_file}.") with open(self._log_file, "w") as log: - self._process = subprocess.Popen(mongo_run_cmd, stderr=subprocess.STDOUT, stdout=log) + self._process = subprocess.Popen( + self._mongo_run_cmd, stderr=subprocess.STDOUT, stdout=log + ) logger.info("MongoDB launched successfully!") @@ -51,7 +49,3 @@ class MongoDbProcess: f"MongoDB did not terminate gracefully and will be forcefully killed: {te}" ) self._process.kill() - - @staticmethod - def _build_mongo_run_cmd(exec_path: str, db_dir: str) -> List[str]: - return [exec_path, DB_DIR_PARAM, db_dir] From 1c73a154bcc7f7575e0dec6b6f379223ac88ee77 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 1 Jun 2021 19:43:15 -0400 Subject: [PATCH 19/31] appimage: Remove `run_appimage.sh` as it is no longer needed The functionality of setting up the data directory and starting mongodb is now handled in python code. --- appimage/AppRun | 2 +- appimage/build_appimage.sh | 1 - appimage/run_appimage.sh | 14 -------------- 3 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 appimage/run_appimage.sh diff --git a/appimage/AppRun b/appimage/AppRun index 1a39dddb6..47f17a778 100755 --- a/appimage/AppRun +++ b/appimage/AppRun @@ -25,5 +25,5 @@ do fi done -(PYTHONHOME="${APPDIR}/opt/python3.7" exec "/bin/bash" "${APPDIR}/usr/src/monkey_island/linux/run_appimage.sh") +(PYTHONHOME="${APPDIR}/opt/python3.7" exec "${APPDIR}/opt/python3.7/bin/python3.7" "${APPDIR}/usr/src/monkey_island.py" $@) exit "$?" diff --git a/appimage/build_appimage.sh b/appimage/build_appimage.sh index bce51bc89..cbc1eec93 100755 --- a/appimage/build_appimage.sh +++ b/appimage/build_appimage.sh @@ -165,7 +165,6 @@ copy_monkey_island_to_appdir() { cp "$1"/monkey_island.py "$INSTALL_DIR" cp -r "$1"/common "$INSTALL_DIR/" cp -r "$1"/monkey_island "$INSTALL_DIR/" - cp ./run_appimage.sh "$INSTALL_DIR"/monkey_island/linux/ cp ./server_config.json.standard "$INSTALL_DIR"/monkey_island/cc/ # TODO: This is a workaround that may be able to be removed after PR #848 is diff --git a/appimage/run_appimage.sh b/appimage/run_appimage.sh deleted file mode 100644 index d31b41843..000000000 --- a/appimage/run_appimage.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -PYTHON_CMD="$APPDIR"/opt/python3.7/bin/python3.7 -DOT_MONKEY="$HOME"/.monkey_island/ - -# shellcheck disable=SC2174 -mkdir --mode=0700 --parents "$DOT_MONKEY" - -DB_DIR="$DOT_MONKEY"/db -mkdir --parents "$DB_DIR" - -cd "$APPDIR"/usr/src || exit 1 -./monkey_island/bin/mongodb/bin/mongod --dbpath "$DB_DIR" & -${PYTHON_CMD} ./monkey_island.py --server-config "$DOT_MONKEY"/server_config.json From 4a1653ed5f7f510ee9dfdfe4b0b589185d823c9a Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 19:22:26 -0400 Subject: [PATCH 20/31] island: Do not start bootloader server if --setup-only is passed --- monkey/monkey_island/cc/server_setup.py | 26 +++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index ba0ed1411..692a945ff 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -50,9 +50,7 @@ def run_monkey_island(): connect_to_mongodb() - bootloader_server_thread = _start_bootloader_server(MONGO_URL) _start_island_server(island_args.setup_only, config_options) - bootloader_server_thread.join() def _configure_logging(config_options): @@ -67,16 +65,6 @@ def _initialize_global_resources(config_options: IslandConfigOptions, server_con initialize_services(config_options.data_dir) -def _start_bootloader_server(mongo_url) -> Thread: - bootloader_server_thread = Thread( - target=BootloaderHttpServer(mongo_url).serve_forever, daemon=True - ) - - bootloader_server_thread.start() - - return bootloader_server_thread - - def _start_island_server(should_setup_only, config_options: IslandConfigOptions): populate_exporter_list() app = init_app(MONGO_URL) @@ -90,6 +78,8 @@ def _start_island_server(should_setup_only, config_options: IslandConfigOptions) logger.warning("Setup only flag passed. Exiting.") return + bootloader_server_thread = _start_bootloader_server(MONGO_URL) + if env_singleton.env.is_debug(): app.run(host="0.0.0.0", debug=True, ssl_context=(crt_path, key_path)) else: @@ -102,6 +92,18 @@ def _start_island_server(should_setup_only, config_options: IslandConfigOptions) _log_init_info() http_server.serve_forever() + bootloader_server_thread.join() + + +def _start_bootloader_server(mongo_url) -> Thread: + bootloader_server_thread = Thread( + target=BootloaderHttpServer(mongo_url).serve_forever, daemon=True + ) + + bootloader_server_thread.start() + + return bootloader_server_thread + def _log_init_info(): logger.info("Monkey Island Server is running!") From edbb2c39f3f4088ab4baa95bfce9561aab1bdbff Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 19:44:31 -0400 Subject: [PATCH 21/31] island: Trap OSError and JSONDecodeError when reading config --- monkey/monkey_island/cc/server_setup.py | 18 ++++++++++++++++-- monkey/monkey_island/setup/config_setup.py | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index 692a945ff..1b5803364 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -1,8 +1,10 @@ +import json import logging import os import sys from pathlib import Path from threading import Thread +from typing import Tuple from gevent.pywsgi import WSGIServer @@ -13,8 +15,10 @@ if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path: sys.path.insert(0, MONKEY_ISLAND_DIR_BASE_PATH) import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 +import monkey_island.setup.config_setup as config_setup # noqa: E402 from common.version import get_version # noqa: E402 from monkey_island.cc.app import init_app # noqa: E402 +from monkey_island.cc.arg_parser import IslandCmdArgs # noqa: E402 from monkey_island.cc.arg_parser import parse_cli_args # noqa: E402 from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 @@ -31,7 +35,6 @@ from monkey_island.cc.setup.mongo.mongo_setup import ( # noqa: E402 register_mongo_shutdown_callback, start_mongodb, ) -from monkey_island.setup.config_setup import setup_data_dir # noqa: E402 from monkey_island.setup.island_config_options import IslandConfigOptions # noqa: E402 logger = logging.getLogger(__name__) @@ -39,7 +42,7 @@ logger = logging.getLogger(__name__) def run_monkey_island(): island_args = parse_cli_args() - config_options, server_config_path = setup_data_dir(island_args) + config_options, server_config_path = _setup_data_dir(island_args) _configure_logging(config_options) _initialize_global_resources(config_options, server_config_path) @@ -53,6 +56,17 @@ def run_monkey_island(): _start_island_server(island_args.setup_only, config_options) +def _setup_data_dir(island_args: IslandCmdArgs) -> Tuple[IslandConfigOptions, str]: + try: + return config_setup.setup_data_dir(island_args) + except OSError as ex: + print(f"Error opening server config file: {ex}") + exit(1) + except json.JSONDecodeError as ex: + print(f"Error loading server config: {ex}") + exit(1) + + def _configure_logging(config_options): reset_logger() setup_logging(config_options.data_dir, config_options.log_level) diff --git a/monkey/monkey_island/setup/config_setup.py b/monkey/monkey_island/setup/config_setup.py index 95ebbc194..f33d2ff24 100644 --- a/monkey/monkey_island/setup/config_setup.py +++ b/monkey/monkey_island/setup/config_setup.py @@ -8,7 +8,7 @@ from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVE from monkey_island.setup.island_config_options import IslandConfigOptions -def setup_data_dir(island_args: IslandCmdArgs): +def setup_data_dir(island_args: IslandCmdArgs) -> Tuple[IslandConfigOptions, str]: if island_args.server_config_path: return _setup_config_by_cmd_arg(island_args.server_config_path) From 5f25debfd2e03199ac667e7e90da5a02606692bc Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 20:12:03 -0400 Subject: [PATCH 22/31] island: Decouple mongo_setup from IslandConfigOptions --- monkey/monkey_island/cc/server_setup.py | 2 +- monkey/monkey_island/cc/setup/mongo/mongo_setup.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index 1b5803364..0ee516db3 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -48,7 +48,7 @@ def run_monkey_island(): _initialize_global_resources(config_options, server_config_path) if config_options.start_mongodb: - mongo_db_process = start_mongodb(config_options) + mongo_db_process = start_mongodb(config_options.data_dir) register_mongo_shutdown_callback(mongo_db_process) connect_to_mongodb() diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py index 2920fc8bb..44252c202 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py @@ -9,7 +9,6 @@ from monkey_island.cc.environment.utils import create_secure_directory from monkey_island.cc.setup.mongo import mongo_connector from monkey_island.cc.setup.mongo.mongo_connector import MONGO_DB_HOST, MONGO_DB_NAME, MONGO_DB_PORT from monkey_island.cc.setup.mongo.mongo_db_process import MongoDbProcess -from monkey_island.setup.island_config_options import IslandConfigOptions DB_DIR_NAME = "db" MONGO_URL = os.environ.get( @@ -21,10 +20,10 @@ MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" logger = logging.getLogger(__name__) -def start_mongodb(config_options: IslandConfigOptions) -> MongoDbProcess: - db_dir = _create_db_dir(config_options.data_dir) +def start_mongodb(data_dir: str) -> MongoDbProcess: + db_dir = _create_db_dir(data_dir) - mongo_db_process = MongoDbProcess(db_dir=db_dir, logging_dir=config_options.data_dir) + mongo_db_process = MongoDbProcess(db_dir=db_dir, logging_dir=data_dir) mongo_db_process.start() return mongo_db_process From 44a3456c626d85228608c877c5130ec22bc36e0f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 2 Jun 2021 20:15:16 -0400 Subject: [PATCH 23/31] island: Remove the responsibility of choosing a log file from MongoDbProcess --- monkey/monkey_island/cc/setup/mongo/mongo_db_process.py | 8 +++----- monkey/monkey_island/cc/setup/mongo/mongo_setup.py | 4 +++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py index 61fc6943a..0a3b55d19 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_db_process.py @@ -1,5 +1,4 @@ import logging -import os import subprocess from monkey_island.cc.server_utils.consts import MONGO_EXECUTABLE_PATH @@ -7,18 +6,17 @@ from monkey_island.cc.server_utils.consts import MONGO_EXECUTABLE_PATH logger = logging.getLogger(__name__) DB_DIR_PARAM = "--dbpath" -MONGO_LOG_FILENAME = "mongodb.log" TERMINATE_TIMEOUT = 10 class MongoDbProcess: - def __init__(self, db_dir: str, logging_dir: str): + def __init__(self, db_dir: str, log_file: str): """ @param db_dir: Path where a folder for database contents will be created - @param logging_dir: Path to a folder where mongodb logs will be created + @param log_file: Path to the file that will contain mongodb logs """ self._mongo_run_cmd = [MONGO_EXECUTABLE_PATH, DB_DIR_PARAM, db_dir] - self._log_file = os.path.join(logging_dir, MONGO_LOG_FILENAME) + self._log_file = log_file self._process = None def start(self): diff --git a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py index 44252c202..e62bbcdb7 100644 --- a/monkey/monkey_island/cc/setup/mongo/mongo_setup.py +++ b/monkey/monkey_island/cc/setup/mongo/mongo_setup.py @@ -11,6 +11,7 @@ from monkey_island.cc.setup.mongo.mongo_connector import MONGO_DB_HOST, MONGO_DB from monkey_island.cc.setup.mongo.mongo_db_process import MongoDbProcess DB_DIR_NAME = "db" +MONGO_LOG_FILENAME = "mongodb.log" MONGO_URL = os.environ.get( "MONKEY_MONGO_URL", "mongodb://{0}:{1}/{2}".format(MONGO_DB_HOST, MONGO_DB_PORT, MONGO_DB_NAME), @@ -22,8 +23,9 @@ logger = logging.getLogger(__name__) def start_mongodb(data_dir: str) -> MongoDbProcess: db_dir = _create_db_dir(data_dir) + log_file = os.path.join(data_dir, MONGO_LOG_FILENAME) - mongo_db_process = MongoDbProcess(db_dir=db_dir, logging_dir=data_dir) + mongo_db_process = MongoDbProcess(db_dir=db_dir, log_file=log_file) mongo_db_process.start() return mongo_db_process From 3917663c5caaae466ff70bd8841cd9aeca526f8d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 3 Jun 2021 11:00:12 -0400 Subject: [PATCH 24/31] island: Rename startup.py -> main.py --- monkey/monkey_island/{startup.py => main.py} | 0 monkey/monkey_island/monkey_island.spec | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename monkey/monkey_island/{startup.py => main.py} (100%) diff --git a/monkey/monkey_island/startup.py b/monkey/monkey_island/main.py similarity index 100% rename from monkey/monkey_island/startup.py rename to monkey/monkey_island/main.py diff --git a/monkey/monkey_island/monkey_island.spec b/monkey/monkey_island/monkey_island.spec index 9994fbffd..7122227c2 100644 --- a/monkey/monkey_island/monkey_island.spec +++ b/monkey/monkey_island/monkey_island.spec @@ -16,7 +16,7 @@ def main(): ("../monkey_island/cc/services/attack/attack_data", "/monkey_island/cc/services/attack/attack_data") ] - a = Analysis(['startup.py'], + a = Analysis(['main.py'], pathex=['..'], hiddenimports=get_hidden_imports(), hookspath=[os.path.join(".", "pyinstaller_hooks")], From c40af5073b1fc61cd17aed0ef4a653952ec93013 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 3 Jun 2021 11:01:03 -0400 Subject: [PATCH 25/31] island: Rename start_island() -> main() --- monkey/monkey_island.py | 4 ++-- monkey/monkey_island/main.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index b51ffcb94..d1510c46f 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -1,4 +1,4 @@ -from monkey_island.startup import start_island +from monkey_island.main import main if "__main__" == __name__: - start_island() + main() diff --git a/monkey/monkey_island/main.py b/monkey/monkey_island/main.py index 90dcd577e..19cf07d9f 100644 --- a/monkey/monkey_island/main.py +++ b/monkey/monkey_island/main.py @@ -4,7 +4,7 @@ import monkey_island.setup.gevent_setup # noqa: F401 isort:skip from monkey_island.cc.server_utils.island_logger import setup_default_failsafe_logging -def start_island(): +def main(): # This is here in order to catch EVERYTHING, some functions are being called on # imports, so the log init needs to be first. try: @@ -19,4 +19,4 @@ def start_island(): if "__main__" == __name__: - start_island() + main() From 92293e601c307e335911cb6d536b68f7ab7e5771 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 3 Jun 2021 11:04:16 -0400 Subject: [PATCH 26/31] island: Add unit test for MONKEY_ISLAND_ABS_PATH If the consts.py is moved, MONKEY_ISLAND_ABS_PATH may be incorrect. This unit test guards against the introduction of a bug if consts.py is moved. --- .../unit_tests/monkey_island/cc/server_utils/test_consts.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 monkey/tests/unit_tests/monkey_island/cc/server_utils/test_consts.py diff --git a/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_consts.py b/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_consts.py new file mode 100644 index 000000000..eeb600b9b --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_consts.py @@ -0,0 +1,5 @@ +from monkey_island.cc.server_utils import consts + + +def test_monkey_island_abs_path(): + assert consts.MONKEY_ISLAND_ABS_PATH.endswith("monkey_island") From 4f97be59c33ef5d11cb020ff63bac3e2c10a2d07 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 3 Jun 2021 11:06:04 -0400 Subject: [PATCH 27/31] island: Rename _initialize_global_resources -> _initialize_globals --- monkey/monkey_island/cc/server_setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index 0ee516db3..2e1cb2ebe 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -45,7 +45,7 @@ def run_monkey_island(): config_options, server_config_path = _setup_data_dir(island_args) _configure_logging(config_options) - _initialize_global_resources(config_options, server_config_path) + _initialize_globals(config_options, server_config_path) if config_options.start_mongodb: mongo_db_process = start_mongodb(config_options.data_dir) @@ -72,7 +72,7 @@ def _configure_logging(config_options): setup_logging(config_options.data_dir, config_options.log_level) -def _initialize_global_resources(config_options: IslandConfigOptions, server_config_path: str): +def _initialize_globals(config_options: IslandConfigOptions, server_config_path: str): env_singleton.initialize_from_file(server_config_path) initialize_encryptor(config_options.data_dir) From ac407bf48a557292a54aff0595e7a16dcfaf9b00 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 3 Jun 2021 11:09:07 -0400 Subject: [PATCH 28/31] island: Remove unused mongo_url from BootloaderHttpServer constructor --- monkey/monkey_island/cc/server_setup.py | 8 +++----- monkey/monkey_island/cc/server_utils/bootloader_server.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index 2e1cb2ebe..c9b5df436 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -92,7 +92,7 @@ def _start_island_server(should_setup_only, config_options: IslandConfigOptions) logger.warning("Setup only flag passed. Exiting.") return - bootloader_server_thread = _start_bootloader_server(MONGO_URL) + bootloader_server_thread = _start_bootloader_server() if env_singleton.env.is_debug(): app.run(host="0.0.0.0", debug=True, ssl_context=(crt_path, key_path)) @@ -109,10 +109,8 @@ def _start_island_server(should_setup_only, config_options: IslandConfigOptions) bootloader_server_thread.join() -def _start_bootloader_server(mongo_url) -> Thread: - bootloader_server_thread = Thread( - target=BootloaderHttpServer(mongo_url).serve_forever, daemon=True - ) +def _start_bootloader_server() -> Thread: + bootloader_server_thread = Thread(target=BootloaderHttpServer().serve_forever, daemon=True) bootloader_server_thread.start() diff --git a/monkey/monkey_island/cc/server_utils/bootloader_server.py b/monkey/monkey_island/cc/server_utils/bootloader_server.py index d43d9318b..bfdd42cf2 100644 --- a/monkey/monkey_island/cc/server_utils/bootloader_server.py +++ b/monkey/monkey_island/cc/server_utils/bootloader_server.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) class BootloaderHttpServer(ThreadingMixIn, HTTPServer): - def __init__(self, mongo_url): + def __init__(self): server_address = ("", 5001) super().__init__(server_address, BootloaderHTTPRequestHandler) From 623d551c6c666623777ef12c47f600515568ed6e Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 3 Jun 2021 11:12:39 -0400 Subject: [PATCH 29/31] island: Move setup/config_setup.py -> cc/setup/config_setup.py --- monkey/monkey_island/cc/server_setup.py | 2 +- monkey/monkey_island/{ => cc}/setup/config_setup.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename monkey/monkey_island/{ => cc}/setup/config_setup.py (100%) diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index c9b5df436..095d7197f 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -15,7 +15,7 @@ if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path: sys.path.insert(0, MONKEY_ISLAND_DIR_BASE_PATH) import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 -import monkey_island.setup.config_setup as config_setup # noqa: E402 +import monkey_island.cc.setup.config_setup as config_setup # noqa: E402 from common.version import get_version # noqa: E402 from monkey_island.cc.app import init_app # noqa: E402 from monkey_island.cc.arg_parser import IslandCmdArgs # noqa: E402 diff --git a/monkey/monkey_island/setup/config_setup.py b/monkey/monkey_island/cc/setup/config_setup.py similarity index 100% rename from monkey/monkey_island/setup/config_setup.py rename to monkey/monkey_island/cc/setup/config_setup.py From 03b543f7f6a7f8fab21c530318607071a5d54815 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 3 Jun 2021 11:16:02 -0400 Subject: [PATCH 30/31] island: Move setup/island_config_options.py -> cc/setup/ --- monkey/monkey_island/cc/environment/server_config_handler.py | 2 +- monkey/monkey_island/cc/server_setup.py | 2 +- monkey/monkey_island/cc/setup/config_setup.py | 2 +- monkey/monkey_island/{ => cc}/setup/island_config_options.py | 0 .../monkey_island/{ => cc}/setup/test_island_config_options.py | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename monkey/monkey_island/{ => cc}/setup/island_config_options.py (100%) rename monkey/tests/unit_tests/monkey_island/{ => cc}/setup/test_island_config_options.py (96%) diff --git a/monkey/monkey_island/cc/environment/server_config_handler.py b/monkey/monkey_island/cc/environment/server_config_handler.py index fc5938694..f1e2358c4 100644 --- a/monkey/monkey_island/cc/environment/server_config_handler.py +++ b/monkey/monkey_island/cc/environment/server_config_handler.py @@ -6,7 +6,7 @@ from monkey_island.cc.server_utils.consts import ( DEFAULT_DEVELOP_SERVER_CONFIG_PATH, DEFAULT_SERVER_CONFIG_PATH, ) -from monkey_island.setup.island_config_options import IslandConfigOptions +from monkey_island.cc.setup.island_config_options import IslandConfigOptions def create_default_server_config_file() -> None: diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index 095d7197f..4eaa13131 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -28,6 +28,7 @@ from monkey_island.cc.server_utils.island_logger import reset_logger, setup_logg from monkey_island.cc.services.initialize import initialize_services # noqa: E402 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 +from monkey_island.cc.setup.island_config_options import IslandConfigOptions # noqa: E402 from monkey_island.cc.setup.mongo.database_initializer import init_collections # noqa: E402 from monkey_island.cc.setup.mongo.mongo_setup import ( # noqa: E402 MONGO_URL, @@ -35,7 +36,6 @@ from monkey_island.cc.setup.mongo.mongo_setup import ( # noqa: E402 register_mongo_shutdown_callback, start_mongodb, ) -from monkey_island.setup.island_config_options import IslandConfigOptions # noqa: E402 logger = logging.getLogger(__name__) diff --git a/monkey/monkey_island/cc/setup/config_setup.py b/monkey/monkey_island/cc/setup/config_setup.py index f33d2ff24..6cc036760 100644 --- a/monkey/monkey_island/cc/setup/config_setup.py +++ b/monkey/monkey_island/cc/setup/config_setup.py @@ -5,7 +5,7 @@ from monkey_island.cc.arg_parser import IslandCmdArgs from monkey_island.cc.environment import server_config_handler from monkey_island.cc.environment.utils import create_secure_directory from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH -from monkey_island.setup.island_config_options import IslandConfigOptions +from monkey_island.cc.setup.island_config_options import IslandConfigOptions def setup_data_dir(island_args: IslandCmdArgs) -> Tuple[IslandConfigOptions, str]: diff --git a/monkey/monkey_island/setup/island_config_options.py b/monkey/monkey_island/cc/setup/island_config_options.py similarity index 100% rename from monkey/monkey_island/setup/island_config_options.py rename to monkey/monkey_island/cc/setup/island_config_options.py diff --git a/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py b/monkey/tests/unit_tests/monkey_island/cc/setup/test_island_config_options.py similarity index 96% rename from monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py rename to monkey/tests/unit_tests/monkey_island/cc/setup/test_island_config_options.py index 35126a278..ae26d5145 100644 --- a/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py +++ b/monkey/tests/unit_tests/monkey_island/cc/setup/test_island_config_options.py @@ -5,7 +5,7 @@ from monkey_island.cc.server_utils.consts import ( DEFAULT_LOG_LEVEL, DEFAULT_START_MONGO_DB, ) -from monkey_island.setup.island_config_options import IslandConfigOptions +from monkey_island.cc.setup.island_config_options import IslandConfigOptions TEST_CONFIG_FILE_CONTENTS_SPECIFIED = { "data_dir": "/tmp", From 19e47583e9a5db88f208bdd8775f1cfce27f1b82 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 3 Jun 2021 15:51:36 -0400 Subject: [PATCH 31/31] island: Use `os.getcwd()` for MONKEY_ISLAND_ABS_PATH on Windows See issue #1207 for more details. --- monkey/monkey_island/cc/server_utils/consts.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 6f4382b87..a14c69d0b 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -13,9 +13,17 @@ def get_default_data_dir() -> str: return r"$HOME/.monkey_island" +# TODO: Figure out why windows requires the use of `os.getcwd()`. See issue #1207. +def _get_monkey_island_abs_path() -> str: + if is_windows_os(): + return os.path.join(os.getcwd(), "monkey_island") + else: + return str(Path(__file__).resolve().parent.parent.parent) + + SERVER_CONFIG_FILENAME = "server_config.json" -MONKEY_ISLAND_ABS_PATH = str(Path(__file__).resolve().parent.parent.parent) +MONKEY_ISLAND_ABS_PATH = _get_monkey_island_abs_path() DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir())