From 8bb6e2f21f1a4b6384e9f17a920fc1ab1db263c3 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 18 May 2021 15:53:21 +0300 Subject: [PATCH 01/19] Added mongodb parameters to appimage server config and develop server config --- appimage/server_config.json.standard | 3 +++ monkey/monkey_island/cc/server_config.json.develop | 3 +++ 2 files changed, 6 insertions(+) diff --git a/appimage/server_config.json.standard b/appimage/server_config.json.standard index 8c894b849..af975a9e0 100644 --- a/appimage/server_config.json.standard +++ b/appimage/server_config.json.standard @@ -4,5 +4,8 @@ "environment": { "server_config": "password", "deployment": "standard" + }, + "mongodb": { + "start_mongodb": true } } diff --git a/monkey/monkey_island/cc/server_config.json.develop b/monkey/monkey_island/cc/server_config.json.develop index fe9e2687f..5d8dc85aa 100644 --- a/monkey/monkey_island/cc/server_config.json.develop +++ b/monkey/monkey_island/cc/server_config.json.develop @@ -3,5 +3,8 @@ "environment": { "server_config": "password", "deployment": "develop" + }, + "mongodb": { + "start_mongodb": true } } From d768d60f9f8c2d325f8c7d0097f1cfb7d1ac0642 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 19 May 2021 14:14:01 +0300 Subject: [PATCH 02/19] Extracted island setup parameter extraction into a separate workflow/DTO --- monkey/monkey_island.py | 16 +++---- monkey/monkey_island/cc/arg_parser.py | 9 +--- .../monkey_island/cc/server_utils/consts.py | 4 ++ monkey/monkey_island/config_file_parser.py | 8 ++++ monkey/monkey_island/config_loader.py | 25 ----------- monkey/setup_param_factory.py | 43 +++++++++++++++++++ monkey/setup_params.py | 18 ++++++++ 7 files changed, 82 insertions(+), 41 deletions(-) create mode 100644 monkey/monkey_island/config_file_parser.py delete mode 100644 monkey/monkey_island/config_loader.py create mode 100644 monkey/setup_param_factory.py create mode 100644 monkey/setup_params.py diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 650cfe95d..0ac7ebca3 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -1,26 +1,24 @@ from gevent import monkey as gevent_monkey +from setup_param_factory import SetupParamFactory from monkey_island.cc.arg_parser import parse_cli_args gevent_monkey.patch_all() import json # noqa: E402 -import os # noqa: E402 -from monkey_island import config_loader # noqa: E402 from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E402 if "__main__" == __name__: island_args = parse_cli_args() - # This is here in order to catch EVERYTHING, some functions are being called on - # imports, so the log init needs to be first. + setup_params = SetupParamFactory.build(island_args) + try: - server_config_path = os.path.expanduser(island_args.server_config) - config = config_loader.load_server_config_from_file(server_config_path) - - setup_logging(config["data_dir"], config["log_level"]) + # This is here in order to catch EVERYTHING, some functions are being called on + # imports, so the log init needs to be first. + setup_logging(setup_params.data_dir, setup_params.log_level) except OSError as ex: print(f"Error opening server config file: {ex}") @@ -32,4 +30,4 @@ if "__main__" == __name__: from monkey_island.cc.main import main # noqa: E402 - main(config["data_dir"], island_args.setup_only, island_args.server_config) + main(setup_params.data_dir, setup_params.setup_only, setup_params.server_config_path) diff --git a/monkey/monkey_island/cc/arg_parser.py b/monkey/monkey_island/cc/arg_parser.py index 6e12ef38f..338db153d 100644 --- a/monkey/monkey_island/cc/arg_parser.py +++ b/monkey/monkey_island/cc/arg_parser.py @@ -1,12 +1,10 @@ from dataclasses import dataclass -from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH - @dataclass class IslandArgs: setup_only: bool - server_config: str + server_config_path: str def parse_cli_args() -> IslandArgs: @@ -25,10 +23,7 @@ def parse_cli_args() -> IslandArgs: "compiling/packaging Islands.", ) parser.add_argument( - "--server-config", - action="store", - help="The path to the server configuration file.", - default=DEFAULT_SERVER_CONFIG_PATH, + "--server-config", action="store", help="The path to the server configuration file." ) args = parser.parse_args() diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index f0dba26dc..c25822c6f 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -5,6 +5,7 @@ __author__ = "itay.mizeretz" MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island") DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5 +# TODO move setup consts DEFAULT_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json") DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( @@ -12,3 +13,6 @@ DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( ) DEFAULT_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc") +DEFAULT_LOG_LEVEL = "INFO" +DEFAULT_START_MONGO_DB = True +DEFAULT_SHOULD_SETUP_ONLY = False diff --git a/monkey/monkey_island/config_file_parser.py b/monkey/monkey_island/config_file_parser.py new file mode 100644 index 000000000..6c9c5c707 --- /dev/null +++ b/monkey/monkey_island/config_file_parser.py @@ -0,0 +1,8 @@ +import json + + +def load_server_config_from_file(server_config_path): + with open(server_config_path, "r") as f: + config_content = f.read() + config = json.loads(config_content) + return config diff --git a/monkey/monkey_island/config_loader.py b/monkey/monkey_island/config_loader.py deleted file mode 100644 index aaa9185d7..000000000 --- a/monkey/monkey_island/config_loader.py +++ /dev/null @@ -1,25 +0,0 @@ -import json -import os - -from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR - -DEFAULT_LOG_LEVEL = "INFO" - - -def load_server_config_from_file(server_config_path): - with open(server_config_path, "r") as f: - config_content = f.read() - config = json.loads(config_content) - add_default_values_to_config(config) - - return config - - -def add_default_values_to_config(config): - config["data_dir"] = os.path.abspath( - os.path.expanduser(os.path.expandvars(config.get("data_dir", DEFAULT_DATA_DIR))) - ) - - config.setdefault("log_level", DEFAULT_LOG_LEVEL) - - return config diff --git a/monkey/setup_param_factory.py b/monkey/setup_param_factory.py new file mode 100644 index 000000000..416a408a5 --- /dev/null +++ b/monkey/setup_param_factory.py @@ -0,0 +1,43 @@ +import os + +from setup_params import SetupParams + +from monkey_island import config_file_parser +from monkey_island.cc.arg_parser import IslandArgs + + +class SetupParamFactory: + @staticmethod + def build(cmd_args: IslandArgs) -> SetupParams: + + setup_params = SetupParams() + + setup_params = SetupParamFactory._update_by_cmd_args(setup_params, cmd_args) + setup_params = SetupParamFactory._update_by_config_file(setup_params) + + return setup_params + + @staticmethod + def _update_by_cmd_args(setup_params: SetupParams, cmd_args: IslandArgs) -> SetupParams: + if type(cmd_args.setup_only) == bool: + setup_params.setup_only = cmd_args.setup_only + + if cmd_args.server_config_path: + setup_params.server_config_path = os.path.expanduser(cmd_args.server_config_path) + + return setup_params + + @staticmethod + def _update_by_config_file(setup_params: SetupParams): + config = config_file_parser.load_server_config_from_file(setup_params.server_config_path) + + if "data_dir" in config: + setup_params.data_dir = config["data_dir"] + + if "log_level" in config: + setup_params.log_level = config["log_level"] + + if "mongodb" in config and "start_mongodb" in config["mongodb"]: + setup_params.start_mongodb = config["mongodb"]["start_mongodb"] + + return setup_params diff --git a/monkey/setup_params.py b/monkey/setup_params.py new file mode 100644 index 000000000..c302c7fc0 --- /dev/null +++ b/monkey/setup_params.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass + +from monkey_island.cc.server_utils.consts import ( + DEFAULT_DATA_DIR, + DEFAULT_LOG_LEVEL, + DEFAULT_SERVER_CONFIG_PATH, + DEFAULT_SHOULD_SETUP_ONLY, + DEFAULT_START_MONGO_DB, +) + + +@dataclass +class SetupParams: + server_config_path = DEFAULT_SERVER_CONFIG_PATH + log_level = DEFAULT_LOG_LEVEL + data_dir = DEFAULT_DATA_DIR + start_mongodb = DEFAULT_START_MONGO_DB + setup_only = DEFAULT_SHOULD_SETUP_ONLY From 9055381873f76feeda59064d0650912763c82a6c Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 19 May 2021 14:21:00 +0300 Subject: [PATCH 03/19] Moved island setup param related code into cc/setup folder --- monkey/monkey_island/setup/__init__.py | 0 monkey/{ => monkey_island/setup}/setup_param_factory.py | 3 +-- monkey/{ => monkey_island/setup}/setup_params.py | 0 3 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 monkey/monkey_island/setup/__init__.py rename monkey/{ => monkey_island/setup}/setup_param_factory.py (95%) rename monkey/{ => monkey_island/setup}/setup_params.py (100%) diff --git a/monkey/monkey_island/setup/__init__.py b/monkey/monkey_island/setup/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/setup_param_factory.py b/monkey/monkey_island/setup/setup_param_factory.py similarity index 95% rename from monkey/setup_param_factory.py rename to monkey/monkey_island/setup/setup_param_factory.py index 416a408a5..72b236623 100644 --- a/monkey/setup_param_factory.py +++ b/monkey/monkey_island/setup/setup_param_factory.py @@ -1,9 +1,8 @@ import os -from setup_params import SetupParams - from monkey_island import config_file_parser from monkey_island.cc.arg_parser import IslandArgs +from monkey_island.setup.setup_params import SetupParams class SetupParamFactory: diff --git a/monkey/setup_params.py b/monkey/monkey_island/setup/setup_params.py similarity index 100% rename from monkey/setup_params.py rename to monkey/monkey_island/setup/setup_params.py From 8c210fc21f45f612ddc57ccd6efb9c30c4ba4b60 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 20 May 2021 12:04:44 +0300 Subject: [PATCH 04/19] Improved setup_param_factory.py further, by extracting loading server config file out of the factory --- monkey/monkey_island.py | 6 ++- monkey/monkey_island/config_file_parser.py | 7 +++- .../setup/setup_param_factory.py | 39 ++++++++----------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 0ac7ebca3..9532e3654 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -1,7 +1,8 @@ from gevent import monkey as gevent_monkey -from setup_param_factory import SetupParamFactory from monkey_island.cc.arg_parser import parse_cli_args +from monkey_island.config_file_parser import load_server_config_from_file +from monkey_island.setup.setup_param_factory import SetupParamFactory gevent_monkey.patch_all() @@ -11,8 +12,9 @@ from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E if "__main__" == __name__: island_args = parse_cli_args() + config_contents = load_server_config_from_file(island_args.server_config_path) - setup_params = SetupParamFactory.build(island_args) + setup_params = SetupParamFactory().build(island_args, config_contents) try: diff --git a/monkey/monkey_island/config_file_parser.py b/monkey/monkey_island/config_file_parser.py index 6c9c5c707..78a621ab2 100644 --- a/monkey/monkey_island/config_file_parser.py +++ b/monkey/monkey_island/config_file_parser.py @@ -1,7 +1,12 @@ import json +from os.path import isfile + +from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH -def load_server_config_from_file(server_config_path): +def load_server_config_from_file(server_config_path: str) -> dict: + if not server_config_path or not isfile(server_config_path): + server_config_path = DEFAULT_SERVER_CONFIG_PATH with open(server_config_path, "r") as f: config_content = f.read() config = json.loads(config_content) diff --git a/monkey/monkey_island/setup/setup_param_factory.py b/monkey/monkey_island/setup/setup_param_factory.py index 72b236623..0c0fe8881 100644 --- a/monkey/monkey_island/setup/setup_param_factory.py +++ b/monkey/monkey_island/setup/setup_param_factory.py @@ -6,37 +6,30 @@ from monkey_island.setup.setup_params import SetupParams class SetupParamFactory: - @staticmethod - def build(cmd_args: IslandArgs) -> SetupParams: + def __init__(self): + self.setup_params = SetupParams() - setup_params = SetupParams() + def build(self, cmd_args: IslandArgs, config_contents: dict) -> SetupParams: - setup_params = SetupParamFactory._update_by_cmd_args(setup_params, cmd_args) - setup_params = SetupParamFactory._update_by_config_file(setup_params) + self._update_by_cmd_args(cmd_args) + self._update_by_config_file(config_contents) - return setup_params + return self.setup_params - @staticmethod - def _update_by_cmd_args(setup_params: SetupParams, cmd_args: IslandArgs) -> SetupParams: + def _update_by_cmd_args(self, cmd_args: IslandArgs): if type(cmd_args.setup_only) == bool: - setup_params.setup_only = cmd_args.setup_only + self.setup_params.setup_only = cmd_args.setup_only if cmd_args.server_config_path: - setup_params.server_config_path = os.path.expanduser(cmd_args.server_config_path) + self.setup_params.server_config_path = os.path.expanduser(cmd_args.server_config_path) - return setup_params + def _update_by_config_file(self, config_contents: dict): - @staticmethod - def _update_by_config_file(setup_params: SetupParams): - config = config_file_parser.load_server_config_from_file(setup_params.server_config_path) + if "data_dir" in config_contents: + self.setup_params.data_dir = config_contents["data_dir"] - if "data_dir" in config: - setup_params.data_dir = config["data_dir"] + if "log_level" in config_contents: + self.setup_params.log_level = config_contents["log_level"] - if "log_level" in config: - setup_params.log_level = config["log_level"] - - if "mongodb" in config and "start_mongodb" in config["mongodb"]: - setup_params.start_mongodb = config["mongodb"]["start_mongodb"] - - return setup_params + if "mongodb" in config_contents and "start_mongodb" in config_contents["mongodb"]: + self.setup_params.start_mongodb = config_contents["mongodb"]["start_mongodb"] From 9f469a8ec0133a74e9c0836a4cd7d813bbaf97fb Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 20 May 2021 13:35:00 +0300 Subject: [PATCH 05/19] Added tests for refactored config file parser --- .../monkey_island/test_config_file_parser.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 monkey/tests/unit_tests/monkey_island/test_config_file_parser.py diff --git a/monkey/tests/unit_tests/monkey_island/test_config_file_parser.py b/monkey/tests/unit_tests/monkey_island/test_config_file_parser.py new file mode 100644 index 000000000..8a0bb3fff --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/test_config_file_parser.py @@ -0,0 +1,16 @@ +from monkey_island import config_file_parser + + +def test_load_server_config_from_file(test_server_config): + config = config_file_parser.load_server_config_from_file(test_server_config) + + assert config["data_dir"] == "~/.monkey_island" + assert config["log_level"] == "NOTICE" + + +def test_load_server_config_from_file_default_path(monkeypatch, test_server_config): + monkeypatch.setattr(config_file_parser, "DEFAULT_SERVER_CONFIG_PATH", test_server_config) + config = config_file_parser.load_server_config_from_file("") + + assert config["data_dir"] == "~/.monkey_island" + assert config["log_level"] == "NOTICE" From df5cec63236db71dcf60dac40f9bc3c0cb43ad94 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 20 May 2021 13:51:52 +0300 Subject: [PATCH 06/19] Added a simple unit test for setup_param_factory.py --- .../setup/setup_param_factory.py | 1 - .../cc/setup/test_setup_param_factory.py | 16 +++++++++++ .../monkey_island/test_config_loader.py | 27 ------------------- 3 files changed, 16 insertions(+), 28 deletions(-) create mode 100644 monkey/tests/unit_tests/monkey_island/cc/setup/test_setup_param_factory.py delete mode 100644 monkey/tests/unit_tests/monkey_island/test_config_loader.py diff --git a/monkey/monkey_island/setup/setup_param_factory.py b/monkey/monkey_island/setup/setup_param_factory.py index 0c0fe8881..32b1b60db 100644 --- a/monkey/monkey_island/setup/setup_param_factory.py +++ b/monkey/monkey_island/setup/setup_param_factory.py @@ -1,6 +1,5 @@ import os -from monkey_island import config_file_parser from monkey_island.cc.arg_parser import IslandArgs from monkey_island.setup.setup_params import SetupParams diff --git a/monkey/tests/unit_tests/monkey_island/cc/setup/test_setup_param_factory.py b/monkey/tests/unit_tests/monkey_island/cc/setup/test_setup_param_factory.py new file mode 100644 index 000000000..d8ea9692e --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/cc/setup/test_setup_param_factory.py @@ -0,0 +1,16 @@ +from monkey_island import config_file_parser +from monkey_island.cc.arg_parser import IslandArgs +from monkey_island.setup.setup_param_factory import SetupParamFactory + +MOCK_ISLAND_CMD_ARGS = IslandArgs(setup_only=True, server_config_path="/temp/test_path") + + +def test_setup_param_factory_build(monkeypatch, test_server_config): + config_contents = config_file_parser.load_server_config_from_file(test_server_config) + + setup_params = SetupParamFactory().build(MOCK_ISLAND_CMD_ARGS, config_contents) + assert setup_params.setup_only + assert setup_params.server_config_path == MOCK_ISLAND_CMD_ARGS.server_config_path + assert setup_params.start_mongodb + assert setup_params.log_level == "NOTICE" + assert setup_params.data_dir == "~/.monkey_island" diff --git a/monkey/tests/unit_tests/monkey_island/test_config_loader.py b/monkey/tests/unit_tests/monkey_island/test_config_loader.py deleted file mode 100644 index 20c330f6a..000000000 --- a/monkey/tests/unit_tests/monkey_island/test_config_loader.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -from monkey_island import config_loader -from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR - - -def test_load_server_config_from_file(test_server_config, mock_home_env): - config = config_loader.load_server_config_from_file(test_server_config) - - assert config["data_dir"] == os.path.join(mock_home_env, ".monkey_island") - assert config["log_level"] == "NOTICE" - - -def test_default_log_level(): - test_config = {} - config = config_loader.add_default_values_to_config(test_config) - - assert "log_level" in config - assert config["log_level"] == "INFO" - - -def test_default_data_dir(mock_home_env): - test_config = {} - config = config_loader.add_default_values_to_config(test_config) - - assert "data_dir" in config - assert config["data_dir"] == DEFAULT_DATA_DIR From 411345d6030cdee69725127721ac371bea0a2202 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 20 May 2021 14:02:47 +0300 Subject: [PATCH 07/19] Refactored "main" method in island to use setup params --- monkey/monkey_island.py | 2 +- monkey/monkey_island/cc/main.py | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 9532e3654..7f21bb0b4 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -32,4 +32,4 @@ if "__main__" == __name__: from monkey_island.cc.main import main # noqa: E402 - main(setup_params.data_dir, setup_params.setup_only, setup_params.server_config_path) + main(setup_params) diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index cf56144ed..549504218 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -9,6 +9,8 @@ from threading import Thread # "monkey_island." work. from gevent.pywsgi import WSGIServer +from monkey_island.setup.setup_params import SetupParams + MONKEY_ISLAND_DIR_BASE_PATH = str(Path(__file__).parent.parent) if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path: sys.path.insert(0, MONKEY_ISLAND_DIR_BASE_PATH) @@ -24,7 +26,6 @@ from monkey_island.cc.database import get_db_version # noqa: E402 from monkey_island.cc.database import is_db_server_up # noqa: E402 from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 -from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH # noqa: E402 from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402 from monkey_island.cc.services.initialize import initialize_services # noqa: E402 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 @@ -34,16 +35,12 @@ from monkey_island.cc.setup import setup # noqa: E402 MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" -def main( - data_dir, - should_setup_only=False, - server_config_filename=DEFAULT_SERVER_CONFIG_PATH, -): +def main(setup_params: SetupParams): logger.info("Starting bootloader server") - env_singleton.initialize_from_file(server_config_filename) - initialize_encryptor(data_dir) - initialize_services(data_dir) + env_singleton.initialize_from_file(setup_params.server_config_path) + initialize_encryptor(setup_params.data_dir) + initialize_services(setup_params.data_dir) mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url()) bootloader_server_thread = Thread( @@ -51,7 +48,7 @@ def main( ) bootloader_server_thread.start() - start_island_server(should_setup_only) + start_island_server(setup_params.setup_only) bootloader_server_thread.join() From 488143b1d3a052289365099c21fe7aede31c7382 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 20 May 2021 08:42:51 +0300 Subject: [PATCH 08/19] BB tests: added the ability for BB tests to "register". If they need registration to run monkeys, BB tests selects passwordless option --- .../island_client/monkey_island_requests.py | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py index f7be1b3cf..8e8392b9e 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py @@ -12,6 +12,10 @@ NO_AUTH_CREDS = "1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()" LOGGER = logging.getLogger(__name__) +class AuthenticationFailedError(Exception): + pass + + # noinspection PyArgumentList class MonkeyIslandRequests(object): def __init__(self, server_address): @@ -43,6 +47,9 @@ class MonkeyIslandRequests(object): def try_get_jwt_from_server(self): try: return self.get_jwt_from_server() + except AuthenticationFailedError: + self.try_set_island_to_no_password() + return self.get_jwt_from_server() except requests.ConnectionError as err: LOGGER.error( "Unable to connect to island, aborting! Error information: {}. Server: {}".format( @@ -51,6 +58,21 @@ class MonkeyIslandRequests(object): ) assert False + def get_jwt_from_server(self): + resp = requests.post( # noqa: DUO123 + self.addr + "api/auth", + json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS}, + verify=False, + ) + if resp.status_code == 401: + raise AuthenticationFailedError + return resp.json()["access_token"] + + def try_set_island_to_no_password(self): + requests.patch( # noqa: DUO123 + self.addr + "api/environment", json={"server_config": "standard"}, verify=False + ) + class _Decorators: @classmethod def refresh_jwt_token(cls, request_function): @@ -62,14 +84,6 @@ class MonkeyIslandRequests(object): return request_function_wrapper - def get_jwt_from_server(self): - resp = requests.post( # noqa: DUO123 - self.addr + "api/auth", - json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS}, - verify=False, - ) - return resp.json()["access_token"] - @_Decorators.refresh_jwt_token def get(self, url, data=None): return requests.get( # noqa: DUO123 From af049b468b4c4156f69dc8d84952a47ba89ffce3 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 20 May 2021 08:44:12 +0300 Subject: [PATCH 09/19] BB tests: removed island connectivity test. Now the connection is tested in fixture and if anything goes wrong tests are not launched --- envs/monkey_zoo/blackbox/test_blackbox.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 20f495151..2fe39a399 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -98,7 +98,15 @@ def wait_machine_bootup(): @pytest.fixture(scope="class") def island_client(island, quick_performance_tests): - island_client_object = MonkeyIslandClient(island) + client_established = False + try: + island_client_object = MonkeyIslandClient(island) + client_established = island_client_object.get_api_status() + except Exception: + logging.exception("message") + finally: + if not client_established: + pytest.exit("BB tests couldn't establish communication to the island.") if not quick_performance_tests: island_client_object.reset_env() yield island_client_object @@ -158,10 +166,6 @@ class TestMonkeyBlackbox: def get_log_dir_path(): return os.path.abspath(LOG_DIR_PATH) - def test_server_online(self, island_client): - if not island_client.get_api_status(): - pytest.exit("BB tests couldn't reach the Island server, quiting.") - def test_ssh_exploiter(self, island_client): TestMonkeyBlackbox.run_exploitation_test(island_client, Ssh, "SSH_exploiter_and_keys") From cc365a74c54d484c5af450d3907d68653aac86b0 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 20 May 2021 16:45:26 +0300 Subject: [PATCH 10/19] Added a CHANGELOG.md entry about BB tests being able to self-register --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3021263d4..aa2a96cb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Authentication mechanism to use bcrypt on server side. #1139 - `server_config.json` puts environment config options in a separate section named "environment". #1161 +- BlackBox tests can now register if they are ran on a fresh installation. #1180 - Improved the structure of unit tests by scoping fixtures only to relevant modules instead of having a one huge fixture file, improved and renamed the directory structure of unit tests and unit test infrastructure. #1178 From 49e63fcf1b77f7efd56fce4c0710018569f71f97 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 21 May 2021 08:41:28 +0300 Subject: [PATCH 11/19] Improve exception message, thrown when trying to establish connection to island in BB tests --- envs/monkey_zoo/blackbox/test_blackbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 2fe39a399..5ee5f63c7 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -103,7 +103,7 @@ def island_client(island, quick_performance_tests): island_client_object = MonkeyIslandClient(island) client_established = island_client_object.get_api_status() except Exception: - logging.exception("message") + logging.exception("Got an exception while trying to establish connection to the Island.") finally: if not client_established: pytest.exit("BB tests couldn't establish communication to the island.") From a17c01c7ee6fca980aa240e5fe286ca22aa5f1c5 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 21 May 2021 11:20:54 +0300 Subject: [PATCH 12/19] Improved readability in arg_parser.py --- monkey/monkey_island/cc/arg_parser.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/monkey/monkey_island/cc/arg_parser.py b/monkey/monkey_island/cc/arg_parser.py index 338db153d..457ffbac2 100644 --- a/monkey/monkey_island/cc/arg_parser.py +++ b/monkey/monkey_island/cc/arg_parser.py @@ -1,13 +1,21 @@ -from dataclasses import dataclass +from monkey_island.cc.server_utils.consts import ( + DEFAULT_SERVER_CONFIG_PATH, + DEFAULT_SHOULD_SETUP_ONLY, +) -@dataclass -class IslandArgs: - setup_only: bool - server_config_path: str +class IslandCmdArgs: + setup_only: bool = DEFAULT_SHOULD_SETUP_ONLY + server_config_path: str = DEFAULT_SERVER_CONFIG_PATH + + def __init__(self, setup_only: None, server_config_path: None): + if setup_only: + self.setup_only = setup_only + if server_config_path: + self.server_config_path = server_config_path -def parse_cli_args() -> IslandArgs: +def parse_cli_args() -> IslandCmdArgs: import argparse parser = argparse.ArgumentParser( @@ -27,4 +35,4 @@ def parse_cli_args() -> IslandArgs: ) args = parser.parse_args() - return IslandArgs(args.setup_only, args.server_config) + return IslandCmdArgs(args.setup_only, args.server_config) From 9cffb9e9d1d977cd8b5c6b60004e45668ca5e826 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 21 May 2021 11:26:02 +0300 Subject: [PATCH 13/19] Refactored `SetupParams` into IslandConfigOptions and altered setup workflow to use it --- monkey/monkey_island.py | 14 ++++---- monkey/monkey_island/cc/main.py | 13 ++++--- monkey/monkey_island/config_file_parser.py | 8 ++++- .../setup/island_config_options.py | 30 ++++++++++++++++ .../setup/setup_param_factory.py | 34 ------------------- monkey/monkey_island/setup/setup_params.py | 18 ---------- whitelist.py | 2 +- 7 files changed, 50 insertions(+), 69 deletions(-) create mode 100644 monkey/monkey_island/setup/island_config_options.py delete mode 100644 monkey/monkey_island/setup/setup_param_factory.py delete mode 100644 monkey/monkey_island/setup/setup_params.py diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 7f21bb0b4..59ac8301a 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -1,8 +1,8 @@ from gevent import monkey as gevent_monkey from monkey_island.cc.arg_parser import parse_cli_args -from monkey_island.config_file_parser import load_server_config_from_file -from monkey_island.setup.setup_param_factory import SetupParamFactory +from monkey_island.config_file_parser import load_island_config_from_file +from monkey_island.setup.island_config_options import IslandConfigOptions gevent_monkey.patch_all() @@ -12,15 +12,13 @@ from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E if "__main__" == __name__: island_args = parse_cli_args() - config_contents = load_server_config_from_file(island_args.server_config_path) - - setup_params = SetupParamFactory().build(island_args, config_contents) + config_options = IslandConfigOptions() try: - # This is here in order to catch EVERYTHING, some functions are being called on # imports, so the log init needs to be first. - setup_logging(setup_params.data_dir, setup_params.log_level) + config_options = load_island_config_from_file(island_args.server_config_path) + setup_logging(config_options.data_dir, config_options.log_level) except OSError as ex: print(f"Error opening server config file: {ex}") @@ -32,4 +30,4 @@ if "__main__" == __name__: from monkey_island.cc.main import main # noqa: E402 - main(setup_params) + main(island_args.setup_only, island_args.server_config_path, config_options) diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index 549504218..b9dd9b197 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -9,7 +9,7 @@ from threading import Thread # "monkey_island." work. from gevent.pywsgi import WSGIServer -from monkey_island.setup.setup_params import SetupParams +from monkey_island.setup.island_config_options import IslandConfigOptions MONKEY_ISLAND_DIR_BASE_PATH = str(Path(__file__).parent.parent) if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path: @@ -35,12 +35,11 @@ from monkey_island.cc.setup import setup # noqa: E402 MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" -def main(setup_params: SetupParams): - logger.info("Starting bootloader server") +def main(setup_only: bool, server_config_path: str, config_options: IslandConfigOptions): - env_singleton.initialize_from_file(setup_params.server_config_path) - initialize_encryptor(setup_params.data_dir) - initialize_services(setup_params.data_dir) + env_singleton.initialize_from_file(server_config_path) + initialize_encryptor(config_options.data_dir) + initialize_services(config_options.data_dir) mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url()) bootloader_server_thread = Thread( @@ -48,7 +47,7 @@ def main(setup_params: SetupParams): ) bootloader_server_thread.start() - start_island_server(setup_params.setup_only) + start_island_server(setup_only) bootloader_server_thread.join() diff --git a/monkey/monkey_island/config_file_parser.py b/monkey/monkey_island/config_file_parser.py index 78a621ab2..c7aa4f146 100644 --- a/monkey/monkey_island/config_file_parser.py +++ b/monkey/monkey_island/config_file_parser.py @@ -2,9 +2,15 @@ import json from os.path import isfile from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH +from monkey_island.setup.island_config_options import IslandConfigOptions -def load_server_config_from_file(server_config_path: str) -> dict: +def load_island_config_from_file(server_config_path: str) -> IslandConfigOptions: + config_contents = read_config_file(server_config_path) + return IslandConfigOptions.build_from_config_file_contents(config_contents) + + +def read_config_file(server_config_path: str) -> dict: if not server_config_path or not isfile(server_config_path): server_config_path = DEFAULT_SERVER_CONFIG_PATH with open(server_config_path, "r") as f: diff --git a/monkey/monkey_island/setup/island_config_options.py b/monkey/monkey_island/setup/island_config_options.py new file mode 100644 index 000000000..ce4a96f7b --- /dev/null +++ b/monkey/monkey_island/setup/island_config_options.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from monkey_island.cc.server_utils.consts import ( + DEFAULT_DATA_DIR, + DEFAULT_LOG_LEVEL, + DEFAULT_START_MONGO_DB, +) + + +@dataclass +class IslandConfigOptions: + log_level = DEFAULT_LOG_LEVEL + data_dir = DEFAULT_DATA_DIR + start_mongodb = DEFAULT_START_MONGO_DB + + @staticmethod + def build_from_config_file_contents(config_contents: dict) -> IslandConfigOptions: + config = IslandConfigOptions() + if "data_dir" in config_contents: + config.data_dir = config_contents["data_dir"] + + if "log_level" in config_contents: + config.log_level = config_contents["log_level"] + + if "mongodb" in config_contents and "start_mongodb" in config_contents["mongodb"]: + config.start_mongodb = config_contents["mongodb"]["start_mongodb"] + + return config diff --git a/monkey/monkey_island/setup/setup_param_factory.py b/monkey/monkey_island/setup/setup_param_factory.py deleted file mode 100644 index 32b1b60db..000000000 --- a/monkey/monkey_island/setup/setup_param_factory.py +++ /dev/null @@ -1,34 +0,0 @@ -import os - -from monkey_island.cc.arg_parser import IslandArgs -from monkey_island.setup.setup_params import SetupParams - - -class SetupParamFactory: - def __init__(self): - self.setup_params = SetupParams() - - def build(self, cmd_args: IslandArgs, config_contents: dict) -> SetupParams: - - self._update_by_cmd_args(cmd_args) - self._update_by_config_file(config_contents) - - return self.setup_params - - def _update_by_cmd_args(self, cmd_args: IslandArgs): - if type(cmd_args.setup_only) == bool: - self.setup_params.setup_only = cmd_args.setup_only - - if cmd_args.server_config_path: - self.setup_params.server_config_path = os.path.expanduser(cmd_args.server_config_path) - - def _update_by_config_file(self, config_contents: dict): - - if "data_dir" in config_contents: - self.setup_params.data_dir = config_contents["data_dir"] - - if "log_level" in config_contents: - self.setup_params.log_level = config_contents["log_level"] - - if "mongodb" in config_contents and "start_mongodb" in config_contents["mongodb"]: - self.setup_params.start_mongodb = config_contents["mongodb"]["start_mongodb"] diff --git a/monkey/monkey_island/setup/setup_params.py b/monkey/monkey_island/setup/setup_params.py deleted file mode 100644 index c302c7fc0..000000000 --- a/monkey/monkey_island/setup/setup_params.py +++ /dev/null @@ -1,18 +0,0 @@ -from dataclasses import dataclass - -from monkey_island.cc.server_utils.consts import ( - DEFAULT_DATA_DIR, - DEFAULT_LOG_LEVEL, - DEFAULT_SERVER_CONFIG_PATH, - DEFAULT_SHOULD_SETUP_ONLY, - DEFAULT_START_MONGO_DB, -) - - -@dataclass -class SetupParams: - server_config_path = DEFAULT_SERVER_CONFIG_PATH - log_level = DEFAULT_LOG_LEVEL - data_dir = DEFAULT_DATA_DIR - start_mongodb = DEFAULT_START_MONGO_DB - setup_only = DEFAULT_SHOULD_SETUP_ONLY diff --git a/whitelist.py b/whitelist.py index bd147220a..51d4c22b8 100644 --- a/whitelist.py +++ b/whitelist.py @@ -165,7 +165,7 @@ ALIBABA # unused variable (monkey/common/cloud/environment_names.py:10) IBM # unused variable (monkey/common/cloud/environment_names.py:11) DigitalOcean # unused variable (monkey/common/cloud/environment_names.py:12) _.aws_info # unused attribute (monkey/monkey_island/cc/environment/aws.py:13) - +build_from_config_file_contents # unused method 'build_from_config_file_contents' (\monkey_island\setup\island_config_options.py:18) # these are not needed for it to work, but may be useful extra information to understand what's going on WINDOWS_PBA_TYPE # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23) From 2621458b37e7543af7fa81a1c9b084f869c0cfb7 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 21 May 2021 11:49:33 +0300 Subject: [PATCH 14/19] Refactored tests to test config file parsing and IslandConfig param extraction --- .../server_configs/server_config_empty.json | 2 ++ ...onfig.json => server_config_init_only.json} | 0 .../cc/setup/test_setup_param_factory.py | 16 ---------------- .../tests/unit_tests/monkey_island/conftest.py | 9 +++++++-- .../monkey_island/test_config_file_parser.py | 18 +++++++++--------- 5 files changed, 18 insertions(+), 27 deletions(-) create mode 100644 monkey/tests/data_for_tests/server_configs/server_config_empty.json rename monkey/tests/data_for_tests/server_configs/{test_server_config.json => server_config_init_only.json} (100%) delete mode 100644 monkey/tests/unit_tests/monkey_island/cc/setup/test_setup_param_factory.py diff --git a/monkey/tests/data_for_tests/server_configs/server_config_empty.json b/monkey/tests/data_for_tests/server_configs/server_config_empty.json new file mode 100644 index 000000000..2c63c0851 --- /dev/null +++ b/monkey/tests/data_for_tests/server_configs/server_config_empty.json @@ -0,0 +1,2 @@ +{ +} diff --git a/monkey/tests/data_for_tests/server_configs/test_server_config.json b/monkey/tests/data_for_tests/server_configs/server_config_init_only.json similarity index 100% rename from monkey/tests/data_for_tests/server_configs/test_server_config.json rename to monkey/tests/data_for_tests/server_configs/server_config_init_only.json diff --git a/monkey/tests/unit_tests/monkey_island/cc/setup/test_setup_param_factory.py b/monkey/tests/unit_tests/monkey_island/cc/setup/test_setup_param_factory.py deleted file mode 100644 index d8ea9692e..000000000 --- a/monkey/tests/unit_tests/monkey_island/cc/setup/test_setup_param_factory.py +++ /dev/null @@ -1,16 +0,0 @@ -from monkey_island import config_file_parser -from monkey_island.cc.arg_parser import IslandArgs -from monkey_island.setup.setup_param_factory import SetupParamFactory - -MOCK_ISLAND_CMD_ARGS = IslandArgs(setup_only=True, server_config_path="/temp/test_path") - - -def test_setup_param_factory_build(monkeypatch, test_server_config): - config_contents = config_file_parser.load_server_config_from_file(test_server_config) - - setup_params = SetupParamFactory().build(MOCK_ISLAND_CMD_ARGS, config_contents) - assert setup_params.setup_only - assert setup_params.server_config_path == MOCK_ISLAND_CMD_ARGS.server_config_path - assert setup_params.start_mongodb - assert setup_params.log_level == "NOTICE" - assert setup_params.data_dir == "~/.monkey_island" diff --git a/monkey/tests/unit_tests/monkey_island/conftest.py b/monkey/tests/unit_tests/monkey_island/conftest.py index 4b7149595..4d0d48e1f 100644 --- a/monkey/tests/unit_tests/monkey_island/conftest.py +++ b/monkey/tests/unit_tests/monkey_island/conftest.py @@ -9,5 +9,10 @@ def server_configs_dir(data_for_tests_dir): @pytest.fixture(scope="module") -def test_server_config(server_configs_dir): - return os.path.join(server_configs_dir, "test_server_config.json") +def server_config_init_only(server_configs_dir): + return os.path.join(server_configs_dir, "server_config_init_only.json") + + +@pytest.fixture(scope="module") +def server_config_empty(server_configs_dir): + return os.path.join(server_configs_dir, "server_config_empty.json") diff --git a/monkey/tests/unit_tests/monkey_island/test_config_file_parser.py b/monkey/tests/unit_tests/monkey_island/test_config_file_parser.py index 8a0bb3fff..db377e6e7 100644 --- a/monkey/tests/unit_tests/monkey_island/test_config_file_parser.py +++ b/monkey/tests/unit_tests/monkey_island/test_config_file_parser.py @@ -1,16 +1,16 @@ from monkey_island import config_file_parser +from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_LOG_LEVEL -def test_load_server_config_from_file(test_server_config): - config = config_file_parser.load_server_config_from_file(test_server_config) +def test_load_server_config_from_file(server_config_init_only): + config = config_file_parser.load_island_config_from_file(server_config_init_only) - assert config["data_dir"] == "~/.monkey_island" - assert config["log_level"] == "NOTICE" + assert config.data_dir == "~/.monkey_island" + assert config.log_level == "NOTICE" -def test_load_server_config_from_file_default_path(monkeypatch, test_server_config): - monkeypatch.setattr(config_file_parser, "DEFAULT_SERVER_CONFIG_PATH", test_server_config) - config = config_file_parser.load_server_config_from_file("") +def test_load_server_config_from_file_empty_file(monkeypatch, server_config_empty): + config = config_file_parser.load_island_config_from_file(server_config_empty) - assert config["data_dir"] == "~/.monkey_island" - assert config["log_level"] == "NOTICE" + assert config.data_dir == DEFAULT_DATA_DIR + assert config.log_level == DEFAULT_LOG_LEVEL From 66b3fb1d4749646ca4673194ff44c60e2381cf7a Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 21 May 2021 12:10:24 +0300 Subject: [PATCH 15/19] Added boilerplate methods for mongodb launch. --- monkey/monkey_island/cc/main.py | 10 ++++++---- monkey/monkey_island/cc/{setup.py => mongo_setup.py} | 7 ++++++- 2 files changed, 12 insertions(+), 5 deletions(-) rename monkey/monkey_island/cc/{setup.py => mongo_setup.py} (94%) diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index b9dd9b197..21b118447 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -24,13 +24,13 @@ from common.version import get_version # noqa: E402 from monkey_island.cc.app import init_app # noqa: E402 from monkey_island.cc.database import get_db_version # noqa: E402 from monkey_island.cc.database import is_db_server_up # noqa: E402 +from monkey_island.cc.mongo_setup import init_collections, launch_mongodb # noqa: E402 from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402 from monkey_island.cc.services.initialize import initialize_services # noqa: E402 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 -from monkey_island.cc.setup import setup # noqa: E402 MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" @@ -47,11 +47,13 @@ def main(setup_only: bool, server_config_path: str, config_options: IslandConfig ) bootloader_server_thread.start() - start_island_server(setup_only) + start_island_server(setup_only, config_options) bootloader_server_thread.join() -def start_island_server(should_setup_only): +def start_island_server(should_setup_only, config_options: IslandConfigOptions): + if config_options.start_mongodb: + launch_mongodb() mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url()) wait_for_mongo_db_server(mongo_url) assert_mongo_db_version(mongo_url) @@ -62,7 +64,7 @@ def start_island_server(should_setup_only): crt_path = str(Path(MONKEY_ISLAND_ABS_PATH, "cc", "server.crt")) key_path = str(Path(MONKEY_ISLAND_ABS_PATH, "cc", "server.key")) - setup() + init_collections() if should_setup_only: logger.warning("Setup only flag passed. Exiting.") diff --git a/monkey/monkey_island/cc/setup.py b/monkey/monkey_island/cc/mongo_setup.py similarity index 94% rename from monkey/monkey_island/cc/setup.py rename to monkey/monkey_island/cc/mongo_setup.py index a03c554be..74cb29fc2 100644 --- a/monkey/monkey_island/cc/setup.py +++ b/monkey/monkey_island/cc/mongo_setup.py @@ -9,7 +9,12 @@ from monkey_island.cc.services.attack.mitre_api_interface import MitreApiInterfa logger = logging.getLogger(__name__) -def setup(): +def launch_mongodb(): + # TODO: Implement the launch of mongodb process + pass + + +def init_collections(): logger.info("Setting up the Monkey Island, this might take a while...") try_store_mitigations_on_mongo() From 0f495791485c4b16c2a6d109de33b8b1d1f3d1d5 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 11:34:55 +0300 Subject: [PATCH 16/19] Added typehints to IslandConfigOptions --- monkey/monkey_island/setup/island_config_options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/setup/island_config_options.py b/monkey/monkey_island/setup/island_config_options.py index ce4a96f7b..134630178 100644 --- a/monkey/monkey_island/setup/island_config_options.py +++ b/monkey/monkey_island/setup/island_config_options.py @@ -11,9 +11,9 @@ from monkey_island.cc.server_utils.consts import ( @dataclass class IslandConfigOptions: - log_level = DEFAULT_LOG_LEVEL - data_dir = DEFAULT_DATA_DIR - start_mongodb = DEFAULT_START_MONGO_DB + log_level: str = DEFAULT_LOG_LEVEL + data_dir: str = DEFAULT_DATA_DIR + start_mongodb: bool = DEFAULT_START_MONGO_DB @staticmethod def build_from_config_file_contents(config_contents: dict) -> IslandConfigOptions: From 2afdfa297bff967784e57416bcbde82a74c9d4d2 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 13:37:02 +0300 Subject: [PATCH 17/19] Improved IslandConfigOptions readability by removing defaults and added a unit test for constructor --- monkey/monkey_island.py | 2 -- monkey/monkey_island/config_file_parser.py | 2 +- .../setup/island_config_options.py | 24 +++++-------------- .../setup/test_island_config_options.py | 20 ++++++++++++++++ 4 files changed, 27 insertions(+), 21 deletions(-) create mode 100644 monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 59ac8301a..d723a6923 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -2,7 +2,6 @@ from gevent import monkey as gevent_monkey from monkey_island.cc.arg_parser import parse_cli_args from monkey_island.config_file_parser import load_island_config_from_file -from monkey_island.setup.island_config_options import IslandConfigOptions gevent_monkey.patch_all() @@ -12,7 +11,6 @@ from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E if "__main__" == __name__: island_args = parse_cli_args() - config_options = IslandConfigOptions() try: # This is here in order to catch EVERYTHING, some functions are being called on diff --git a/monkey/monkey_island/config_file_parser.py b/monkey/monkey_island/config_file_parser.py index c7aa4f146..830ae6720 100644 --- a/monkey/monkey_island/config_file_parser.py +++ b/monkey/monkey_island/config_file_parser.py @@ -7,7 +7,7 @@ from monkey_island.setup.island_config_options import IslandConfigOptions def load_island_config_from_file(server_config_path: str) -> IslandConfigOptions: config_contents = read_config_file(server_config_path) - return IslandConfigOptions.build_from_config_file_contents(config_contents) + return IslandConfigOptions(config_contents) def read_config_file(server_config_path: str) -> dict: diff --git a/monkey/monkey_island/setup/island_config_options.py b/monkey/monkey_island/setup/island_config_options.py index 134630178..bf1c06e1b 100644 --- a/monkey/monkey_island/setup/island_config_options.py +++ b/monkey/monkey_island/setup/island_config_options.py @@ -1,7 +1,5 @@ from __future__ import annotations -from dataclasses import dataclass - from monkey_island.cc.server_utils.consts import ( DEFAULT_DATA_DIR, DEFAULT_LOG_LEVEL, @@ -9,22 +7,12 @@ from monkey_island.cc.server_utils.consts import ( ) -@dataclass class IslandConfigOptions: - log_level: str = DEFAULT_LOG_LEVEL - data_dir: str = DEFAULT_DATA_DIR - start_mongodb: bool = DEFAULT_START_MONGO_DB + def __init__(self, config_contents: dict): + self.data_dir = config_contents.get("data_dir", DEFAULT_DATA_DIR) - @staticmethod - def build_from_config_file_contents(config_contents: dict) -> IslandConfigOptions: - config = IslandConfigOptions() - if "data_dir" in config_contents: - config.data_dir = config_contents["data_dir"] + self.log_level = config_contents.get("log_level", DEFAULT_LOG_LEVEL) - if "log_level" in config_contents: - config.log_level = config_contents["log_level"] - - if "mongodb" in config_contents and "start_mongodb" in config_contents["mongodb"]: - config.start_mongodb = config_contents["mongodb"]["start_mongodb"] - - return config + self.start_mongodb = config_contents.get( + "mongodb", {"start_mongodb": DEFAULT_START_MONGO_DB} + ).get("start_mongodb", DEFAULT_START_MONGO_DB) diff --git a/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py b/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py new file mode 100644 index 000000000..564df30a0 --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py @@ -0,0 +1,20 @@ +from monkey_island.cc.server_utils.consts import DEFAULT_LOG_LEVEL, DEFAULT_START_MONGO_DB +from monkey_island.setup.island_config_options import IslandConfigOptions + +MOCKED_CONFIG_FILE_CONTENTS_STANDARD = {"data_dir": "/tmp", "mongodb": {"start_mongodb": False}} + +MOCKED_CONFIG_FILE_CONTENTS_NO_STARTMONGO = {"data_dir": "/tmp", "mongodb": {}} + + +def test_island_config_options__standard(): + options = IslandConfigOptions(MOCKED_CONFIG_FILE_CONTENTS_STANDARD) + assert not options.start_mongodb + assert options.data_dir == "/tmp" + assert options.log_level == DEFAULT_LOG_LEVEL + + +def test_island_config_options__no_starmongo(): + options = IslandConfigOptions(MOCKED_CONFIG_FILE_CONTENTS_NO_STARTMONGO) + assert options.start_mongodb == DEFAULT_START_MONGO_DB + assert options.data_dir == "/tmp" + assert options.log_level == DEFAULT_LOG_LEVEL From d52e80a3f3c92e96c86e529526aa7c72a70f34d7 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 14:03:49 +0300 Subject: [PATCH 18/19] Improved the readability IslandConfigOptions test --- .../monkey_island/setup/test_island_config_options.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py b/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py index 564df30a0..0e79e0324 100644 --- a/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py +++ b/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py @@ -1,20 +1,18 @@ from monkey_island.cc.server_utils.consts import DEFAULT_LOG_LEVEL, DEFAULT_START_MONGO_DB from monkey_island.setup.island_config_options import IslandConfigOptions -MOCKED_CONFIG_FILE_CONTENTS_STANDARD = {"data_dir": "/tmp", "mongodb": {"start_mongodb": False}} +TEST_CONFIG_FILE_CONTENTS_STANDARD = {"data_dir": "/tmp", "mongodb": {"start_mongodb": False}} -MOCKED_CONFIG_FILE_CONTENTS_NO_STARTMONGO = {"data_dir": "/tmp", "mongodb": {}} +TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO = {"data_dir": "/tmp", "mongodb": {}} def test_island_config_options__standard(): - options = IslandConfigOptions(MOCKED_CONFIG_FILE_CONTENTS_STANDARD) + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_STANDARD) assert not options.start_mongodb assert options.data_dir == "/tmp" assert options.log_level == DEFAULT_LOG_LEVEL def test_island_config_options__no_starmongo(): - options = IslandConfigOptions(MOCKED_CONFIG_FILE_CONTENTS_NO_STARTMONGO) + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO) assert options.start_mongodb == DEFAULT_START_MONGO_DB - assert options.data_dir == "/tmp" - assert options.log_level == DEFAULT_LOG_LEVEL From 0248a6e281532089f55687dd92ffb12d9ac0f102 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 14:25:22 +0300 Subject: [PATCH 19/19] Refactored IslandConfigOptions tests to be test per option, rather than based on code workflow branches --- .../setup/test_island_config_options.py | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py b/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py index 0e79e0324..9175bc274 100644 --- a/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py +++ b/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py @@ -1,18 +1,39 @@ -from monkey_island.cc.server_utils.consts import DEFAULT_LOG_LEVEL, DEFAULT_START_MONGO_DB +from monkey_island.cc.server_utils.consts import ( + DEFAULT_DATA_DIR, + DEFAULT_LOG_LEVEL, + DEFAULT_START_MONGO_DB, +) from monkey_island.setup.island_config_options import IslandConfigOptions -TEST_CONFIG_FILE_CONTENTS_STANDARD = {"data_dir": "/tmp", "mongodb": {"start_mongodb": False}} +TEST_CONFIG_FILE_CONTENTS_SPECIFIED = { + "data_dir": "/tmp", + "log_level": "test", + "mongodb": {"start_mongodb": False}, +} -TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO = {"data_dir": "/tmp", "mongodb": {}} +TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED = {} + +TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO = {"mongodb": {}} -def test_island_config_options__standard(): - options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_STANDARD) - assert not options.start_mongodb +def test_island_config_options__data_dir(): + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED) assert options.data_dir == "/tmp" + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED) + assert options.data_dir == DEFAULT_DATA_DIR + + +def test_island_config_options__log_level(): + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED) + assert options.log_level == "test" + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED) assert options.log_level == DEFAULT_LOG_LEVEL -def test_island_config_options__no_starmongo(): +def test_island_config_options__mongodb(): + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED) + assert not options.start_mongodb + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED) + assert options.start_mongodb == DEFAULT_START_MONGO_DB options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO) assert options.start_mongodb == DEFAULT_START_MONGO_DB