From 7e21635733ddbca5dd91b082692c15a537ed7720 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 25 Nov 2021 12:11:09 +0200 Subject: [PATCH 01/25] Island: separate the workflow of config extraction from data directory setup --- .../cc/environment/server_config_handler.py | 16 -------- monkey/monkey_island/cc/server_setup.py | 25 +++++++----- monkey/monkey_island/cc/setup/config_setup.py | 38 +++---------------- monkey/monkey_island/cc/setup/data_dir.py | 4 +- 4 files changed, 22 insertions(+), 61 deletions(-) diff --git a/monkey/monkey_island/cc/environment/server_config_handler.py b/monkey/monkey_island/cc/environment/server_config_handler.py index 363b7c2e6..be70a8bec 100644 --- a/monkey/monkey_island/cc/environment/server_config_handler.py +++ b/monkey/monkey_island/cc/environment/server_config_handler.py @@ -1,24 +1,8 @@ import json -import os -from pathlib import Path -from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH, SERVER_CONFIG_FILENAME from monkey_island.cc.setup.island_config_options import IslandConfigOptions -def create_default_server_config_file(data_dir: str) -> str: - config_file_path = os.path.join(data_dir, SERVER_CONFIG_FILENAME) - if not os.path.isfile(config_file_path): - write_default_server_config_to_file(config_file_path) - - return config_file_path - - -def write_default_server_config_to_file(path: str) -> None: - default_config = Path(DEFAULT_SERVER_CONFIG_PATH).read_text() - Path(path).write_text(default_config) - - def load_server_config_from_file(server_config_path) -> IslandConfigOptions: with open(server_config_path, "r") as f: config_content = f.read() diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index 65617774d..3de58ef8e 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -5,12 +5,12 @@ import sys from pathlib import Path from sys import exit from threading import Thread -from typing import Tuple import gevent.hub from gevent.pywsgi import WSGIServer from monkey_island.cc.server_utils.consts import ISLAND_PORT +from monkey_island.cc.setup.config_setup import extract_server_config # Add the monkey_island directory to the path, to make sure imports that don't start with # "monkey_island." work. @@ -18,7 +18,6 @@ MONKEY_ISLAND_DIR_BASE_PATH = str(Path(__file__).parent.parent) if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path: sys.path.insert(0, MONKEY_ISLAND_DIR_BASE_PATH) -import monkey_island.cc.setup.config_setup as config_setup # noqa: E402 from common.version import get_version # noqa: E402 from monkey_island.cc.app import init_app # noqa: E402 from monkey_island.cc.arg_parser import IslandCmdArgs # noqa: E402 @@ -34,7 +33,7 @@ from monkey_island.cc.services.initialize import initialize_services # noqa: E4 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 from monkey_island.cc.setup import island_config_options_validator # noqa: E402 -from monkey_island.cc.setup.data_dir import IncompatibleDataDirectory # noqa: E402 +from monkey_island.cc.setup.data_dir import IncompatibleDataDirectory, setup_data_dir # noqa: E402 from monkey_island.cc.setup.gevent_hub_error_handler import GeventHubErrorHandler # noqa: E402 from monkey_island.cc.setup.island_config_options import IslandConfigOptions # noqa: E402 from monkey_island.cc.setup.mongo import mongo_setup # noqa: E402 @@ -45,12 +44,13 @@ logger = logging.getLogger(__name__) def run_monkey_island(): island_args = parse_cli_args() - config_options, server_config_path = _setup_data_dir(island_args) + config_options = _extract_config(island_args) + _setup_data_dir(config_options.data_dir) _exit_on_invalid_config_options(config_options) _configure_logging(config_options) - _initialize_globals(config_options, server_config_path) + _initialize_globals(config_options.data_dir) mongo_db_process = None if config_options.start_mongodb: @@ -58,19 +58,24 @@ def run_monkey_island(): _connect_to_mongodb(mongo_db_process) - _configure_gevent_exception_handling(Path(config_options.data_dir)) + _configure_gevent_exception_handling(config_options.data_dir) _start_island_server(island_args.setup_only, config_options) -def _setup_data_dir(island_args: IslandCmdArgs) -> Tuple[IslandConfigOptions, str]: +def _extract_config(island_args: IslandCmdArgs) -> IslandConfigOptions: try: - return config_setup.setup_server_config(island_args) + return extract_server_config(island_args) except OSError as ex: print(f"Error opening server config file: {ex}") exit(1) except json.JSONDecodeError as ex: print(f"Error loading server config: {ex}") exit(1) + + +def _setup_data_dir(data_dir_path: Path): + try: + setup_data_dir(data_dir_path) except IncompatibleDataDirectory as ex: print(f"Incompatible data directory: {ex}") exit(1) @@ -89,8 +94,8 @@ def _configure_logging(config_options): setup_logging(config_options.data_dir, config_options.log_level) -def _initialize_globals(config_options: IslandConfigOptions, server_config_path: str): - initialize_services(config_options.data_dir) +def _initialize_globals(data_dir: Path): + initialize_services(data_dir) def _start_mongodb(data_dir: Path) -> MongoDbProcess: diff --git a/monkey/monkey_island/cc/setup/config_setup.py b/monkey/monkey_island/cc/setup/config_setup.py index b1e76b51d..501df5b73 100644 --- a/monkey/monkey_island/cc/setup/config_setup.py +++ b/monkey/monkey_island/cc/setup/config_setup.py @@ -1,42 +1,14 @@ -from typing import Tuple - from common.utils.file_utils import expand_path from monkey_island.cc.arg_parser import IslandCmdArgs from monkey_island.cc.environment import server_config_handler from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH -from monkey_island.cc.setup.data_dir import setup_data_dir from monkey_island.cc.setup.island_config_options import IslandConfigOptions -def setup_server_config(island_args: IslandCmdArgs) -> Tuple[IslandConfigOptions, str]: +def extract_server_config(island_args: IslandCmdArgs) -> IslandConfigOptions: if island_args.server_config_path: - return _setup_config_by_cmd_arg(island_args.server_config_path) + path_to_config = expand_path(island_args.server_config_path) + else: + path_to_config = DEFAULT_SERVER_CONFIG_PATH - return _setup_default_config() - - -def _setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, str]: - server_config_path = expand_path(server_config_path) - config = server_config_handler.load_server_config_from_file(server_config_path) - - # TODO refactor like in https://github.com/guardicore/monkey/pull/1528 because - # there's absolutely no reason to be exposed to IslandConfigOptions extraction logic - # if you want to modify data directory related code. - setup_data_dir(config.data_dir) - - return config, server_config_path - - -def _setup_default_config() -> Tuple[IslandConfigOptions, str]: - default_config = server_config_handler.load_server_config_from_file(DEFAULT_SERVER_CONFIG_PATH) - default_data_dir = default_config.data_dir - - # TODO refactor like in https://github.com/guardicore/monkey/pull/1528 because - # there's absolutely no reason to be exposed to IslandConfigOptions extraction logic - # if you want to modify data directory related code. - setup_data_dir(default_data_dir) - - server_config_path = server_config_handler.create_default_server_config_file(default_data_dir) - config = server_config_handler.load_server_config_from_file(server_config_path) - - return config, server_config_path + return server_config_handler.load_server_config_from_file(path_to_config) diff --git a/monkey/monkey_island/cc/setup/data_dir.py b/monkey/monkey_island/cc/setup/data_dir.py index af01da050..124b2a6f7 100644 --- a/monkey/monkey_island/cc/setup/data_dir.py +++ b/monkey/monkey_island/cc/setup/data_dir.py @@ -15,14 +15,14 @@ class IncompatibleDataDirectory(Exception): pass -def setup_data_dir(data_dir_path: Path) -> None: +def setup_data_dir(data_dir_path: Path): logger.info(f"Setting up data directory at {data_dir_path}.") if _is_data_dir_old(data_dir_path): logger.info("Version in data directory does not match the Island's version.") _handle_old_data_directory(data_dir_path) create_secure_directory(str(data_dir_path)) write_version(data_dir_path) - logger.info("Data directory set up.") + logger.info(f"Data directory set up in {data_dir_path}.") def _is_data_dir_old(data_dir_path: Path) -> bool: From 00819ccf40a683b52d3eb82d24a66ccf2fac4bdb Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 25 Nov 2021 13:15:01 +0200 Subject: [PATCH 02/25] Build: remove unused server config options from docker Options removed match the defaults so there's no reason to keep them --- build_scripts/docker/server_config.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build_scripts/docker/server_config.json b/build_scripts/docker/server_config.json index ea464f181..7f95b67ee 100644 --- a/build_scripts/docker/server_config.json +++ b/build_scripts/docker/server_config.json @@ -1,9 +1,5 @@ { "data_dir": "/monkey_island_data", - "log_level": "DEBUG", - "environment": { - "server_config": "password" - }, "mongodb": { "start_mongodb": false } From 00665cbae0301c37bb8f30c947cd91ee0fb114b1 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 25 Nov 2021 18:18:12 +0200 Subject: [PATCH 03/25] Island: implements the logic of server config extraction and adds unit tests for it Since server_config.json no longer needs to be writable, we can load defaults, then override package specific options and lastly override user specified options to form the final config for island --- .../monkey_island/cc/environment/__init__.py | 0 .../cc/environment/server_config_handler.py | 11 --- monkey/monkey_island/cc/server_setup.py | 7 +- monkey/monkey_island/cc/setup/config_setup.py | 33 +++++++-- .../cc/setup/island_config_options.py | 3 + .../monkey_island/cc/environment/__init__.py | 0 .../cc/setup/test_server_setup.py | 69 +++++++++++++++++++ 7 files changed, 102 insertions(+), 21 deletions(-) delete mode 100644 monkey/monkey_island/cc/environment/__init__.py delete mode 100644 monkey/monkey_island/cc/environment/server_config_handler.py delete mode 100644 monkey/tests/unit_tests/monkey_island/cc/environment/__init__.py create mode 100644 monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py diff --git a/monkey/monkey_island/cc/environment/__init__.py b/monkey/monkey_island/cc/environment/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/monkey/monkey_island/cc/environment/server_config_handler.py b/monkey/monkey_island/cc/environment/server_config_handler.py deleted file mode 100644 index be70a8bec..000000000 --- a/monkey/monkey_island/cc/environment/server_config_handler.py +++ /dev/null @@ -1,11 +0,0 @@ -import json - -from monkey_island.cc.setup.island_config_options import IslandConfigOptions - - -def load_server_config_from_file(server_config_path) -> IslandConfigOptions: - with open(server_config_path, "r") as f: - config_content = f.read() - config = json.loads(config_content) - - return IslandConfigOptions(config) diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index 3de58ef8e..61d13a96a 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -10,7 +10,7 @@ import gevent.hub from gevent.pywsgi import WSGIServer from monkey_island.cc.server_utils.consts import ISLAND_PORT -from monkey_island.cc.setup.config_setup import extract_server_config +from monkey_island.cc.setup.config_setup import get_server_config # Add the monkey_island directory to the path, to make sure imports that don't start with # "monkey_island." work. @@ -64,10 +64,7 @@ def run_monkey_island(): def _extract_config(island_args: IslandCmdArgs) -> IslandConfigOptions: try: - return extract_server_config(island_args) - except OSError as ex: - print(f"Error opening server config file: {ex}") - exit(1) + return get_server_config(island_args) except json.JSONDecodeError as ex: print(f"Error loading server config: {ex}") exit(1) diff --git a/monkey/monkey_island/cc/setup/config_setup.py b/monkey/monkey_island/cc/setup/config_setup.py index 501df5b73..b997e35d4 100644 --- a/monkey/monkey_island/cc/setup/config_setup.py +++ b/monkey/monkey_island/cc/setup/config_setup.py @@ -1,14 +1,37 @@ +import json +from logging import getLogger +from pathlib import Path + from common.utils.file_utils import expand_path from monkey_island.cc.arg_parser import IslandCmdArgs -from monkey_island.cc.environment import server_config_handler from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH from monkey_island.cc.setup.island_config_options import IslandConfigOptions +logger = getLogger(__name__) + + +def get_server_config(island_args: IslandCmdArgs) -> IslandConfigOptions: + config = IslandConfigOptions({}) + + update_config_from_file(config, DEFAULT_SERVER_CONFIG_PATH) -def extract_server_config(island_args: IslandCmdArgs) -> IslandConfigOptions: if island_args.server_config_path: path_to_config = expand_path(island_args.server_config_path) - else: - path_to_config = DEFAULT_SERVER_CONFIG_PATH + update_config_from_file(config, path_to_config) - return server_config_handler.load_server_config_from_file(path_to_config) + return config + + +def update_config_from_file(config: IslandConfigOptions, config_path: Path): + try: + config_from_file = load_server_config_from_file(config_path) + config.update(config_from_file) + except OSError: + logger.info(f"Server config not found in path {config_path}") + + +def load_server_config_from_file(server_config_path) -> IslandConfigOptions: + with open(server_config_path, "r") as f: + config_content = f.read() + config = json.loads(config_content) + return IslandConfigOptions(config) diff --git a/monkey/monkey_island/cc/setup/island_config_options.py b/monkey/monkey_island/cc/setup/island_config_options.py index 66a49306a..12b141923 100644 --- a/monkey/monkey_island/cc/setup/island_config_options.py +++ b/monkey/monkey_island/cc/setup/island_config_options.py @@ -31,3 +31,6 @@ class IslandConfigOptions: "ssl_certificate_key_file", DEFAULT_KEY_PATH ) ) + + def update(self, target: IslandConfigOptions): + self.__dict__.update(target.__dict__) diff --git a/monkey/tests/unit_tests/monkey_island/cc/environment/__init__.py b/monkey/tests/unit_tests/monkey_island/cc/environment/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py new file mode 100644 index 000000000..cb3511b10 --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py @@ -0,0 +1,69 @@ +from json import dumps +from pathlib import Path + +import pytest + +import monkey_island.cc.setup.config_setup # noqa: F401 +from monkey_island.cc.arg_parser import IslandCmdArgs +from monkey_island.cc.server_setup import _extract_config +from monkey_island.cc.setup.island_config_options import IslandConfigOptions + +BAD_JSON = '{"data_dir": "C:\\test\\test"' + + +@pytest.fixture +def user_server_config_path(tmpdir) -> Path: + return tmpdir / "fake_server_config.json" + + +@pytest.fixture +def deployment_server_config_path(tmpdir) -> Path: + return tmpdir / "fake_server_config2.json" + + +def create_server_config(config_contents: str, server_config_path: Path): + with open(server_config_path, "w") as file: + file.write(config_contents) + + +@pytest.fixture(autouse=True) +def mock_deployment_config_path(monkeypatch, deployment_server_config_path): + monkeypatch.setattr( + "monkey_island.cc.setup.config_setup.DEFAULT_SERVER_CONFIG_PATH", + deployment_server_config_path, + ) + + +def test_extract_config_defaults(): + expected = IslandConfigOptions({}) + assert ( + expected.__dict__ + == _extract_config(IslandCmdArgs(setup_only=False, server_config_path=None)).__dict__ + ) + + +def test_package_config_overrides_defaults(deployment_server_config_path): + expected = IslandConfigOptions({"key_path": "/key_path_2"}) + create_server_config(dumps({"key_path": "/key_path_2"}), deployment_server_config_path) + assert ( + expected.__dict__ + == _extract_config(IslandCmdArgs(setup_only=False, server_config_path=None)).__dict__ + ) + + +def test_user_config_overrides_package_config( + deployment_server_config_path, user_server_config_path +): + expected = IslandConfigOptions({"key_path": "/key_path_3"}) + create_server_config(dumps({"key_path": "/key_path_2"}), deployment_server_config_path) + create_server_config(dumps({"key_path": "/key_path_3"}), user_server_config_path) + extracted_config = _extract_config( + IslandCmdArgs(setup_only=False, server_config_path=user_server_config_path) + ) + assert expected.__dict__ == extracted_config.__dict__ + + +def test_malformed_json(user_server_config_path): + create_server_config(BAD_JSON, user_server_config_path) + with pytest.raises(SystemExit): + _extract_config(IslandCmdArgs(setup_only=False, server_config_path=user_server_config_path)) From dcc71faaa9903e205497eb83aa83203bf66f3df1 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 26 Nov 2021 11:54:18 +0200 Subject: [PATCH 04/25] Island: try reading server_config.json from install directory On windows it's not easy to pass server_config as a commandline parameter. It's easier to just create a file in install directory. --- .../monkey_island/cc/server_utils/consts.py | 3 +- monkey/monkey_island/cc/setup/config_setup.py | 7 ++- .../cc/server_utils/test_consts.py | 2 +- .../cc/setup/test_server_setup.py | 49 +++++++++++++++---- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index ab9261140..f5cff557c 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -36,7 +36,8 @@ MONGO_EXECUTABLE_PATH = ( ) MONGO_CONNECTION_TIMEOUT = 15 -DEFAULT_SERVER_CONFIG_PATH = str(Path(MONKEY_ISLAND_ABS_PATH, "cc", SERVER_CONFIG_FILENAME)) +PACKAGE_CONFIG_PATH = Path(MONKEY_ISLAND_ABS_PATH, "cc", SERVER_CONFIG_FILENAME) +USER_CONFIG_PATH = Path(os.getcwd(), SERVER_CONFIG_FILENAME) DEFAULT_LOG_LEVEL = "INFO" diff --git a/monkey/monkey_island/cc/setup/config_setup.py b/monkey/monkey_island/cc/setup/config_setup.py index b997e35d4..77652ddba 100644 --- a/monkey/monkey_island/cc/setup/config_setup.py +++ b/monkey/monkey_island/cc/setup/config_setup.py @@ -4,7 +4,7 @@ from pathlib import Path from common.utils.file_utils import expand_path from monkey_island.cc.arg_parser import IslandCmdArgs -from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH +from monkey_island.cc.server_utils.consts import PACKAGE_CONFIG_PATH, USER_CONFIG_PATH from monkey_island.cc.setup.island_config_options import IslandConfigOptions logger = getLogger(__name__) @@ -13,7 +13,9 @@ logger = getLogger(__name__) def get_server_config(island_args: IslandCmdArgs) -> IslandConfigOptions: config = IslandConfigOptions({}) - update_config_from_file(config, DEFAULT_SERVER_CONFIG_PATH) + update_config_from_file(config, PACKAGE_CONFIG_PATH) + + update_config_from_file(config, USER_CONFIG_PATH) if island_args.server_config_path: path_to_config = expand_path(island_args.server_config_path) @@ -26,6 +28,7 @@ def update_config_from_file(config: IslandConfigOptions, config_path: Path): try: config_from_file = load_server_config_from_file(config_path) config.update(config_from_file) + logger.info(f"Server config updated from {config_path}") except OSError: logger.info(f"Server config not found in path {config_path}") diff --git a/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_consts.py b/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_consts.py index 26d7bc583..996226e0d 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_consts.py +++ b/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_consts.py @@ -15,4 +15,4 @@ def test_default_server_config_file_path(): else: server_file_path = f"{consts.MONKEY_ISLAND_ABS_PATH}/cc/{consts.SERVER_CONFIG_FILENAME}" - assert consts.DEFAULT_SERVER_CONFIG_PATH == server_file_path + assert consts.PACKAGE_CONFIG_PATH == server_file_path diff --git a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py index cb3511b10..c3a6ee636 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py +++ b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py @@ -12,15 +12,23 @@ BAD_JSON = '{"data_dir": "C:\\test\\test"' @pytest.fixture -def user_server_config_path(tmpdir) -> Path: +def cmd_server_config_path(tmpdir) -> Path: + # Represents the config that user can provide via cmd arguments return tmpdir / "fake_server_config.json" @pytest.fixture -def deployment_server_config_path(tmpdir) -> Path: +def user_default_server_config_path(tmpdir) -> Path: + # Represents the config that can be put into the install dir return tmpdir / "fake_server_config2.json" +@pytest.fixture +def deployment_server_config_path(tmpdir) -> Path: + # Represents the config that is built in, deployment specific + return tmpdir / "fake_server_config3.json" + + def create_server_config(config_contents: str, server_config_path: Path): with open(server_config_path, "w") as file: file.write(config_contents) @@ -34,6 +42,14 @@ def mock_deployment_config_path(monkeypatch, deployment_server_config_path): ) +@pytest.fixture(autouse=True) +def mock_user_config_path(monkeypatch, deployment_server_config_path): + monkeypatch.setattr( + "monkey_island.cc.setup.config_setup.USER_CONFIG_PATH", + deployment_server_config_path, + ) + + def test_extract_config_defaults(): expected = IslandConfigOptions({}) assert ( @@ -42,7 +58,7 @@ def test_extract_config_defaults(): ) -def test_package_config_overrides_defaults(deployment_server_config_path): +def test_deployment_config_overrides_defaults(deployment_server_config_path): expected = IslandConfigOptions({"key_path": "/key_path_2"}) create_server_config(dumps({"key_path": "/key_path_2"}), deployment_server_config_path) assert ( @@ -51,19 +67,32 @@ def test_package_config_overrides_defaults(deployment_server_config_path): ) -def test_user_config_overrides_package_config( - deployment_server_config_path, user_server_config_path +def test_user_config_overrides_deployment( + deployment_server_config_path, cmd_server_config_path, user_default_server_config_path ): expected = IslandConfigOptions({"key_path": "/key_path_3"}) create_server_config(dumps({"key_path": "/key_path_2"}), deployment_server_config_path) - create_server_config(dumps({"key_path": "/key_path_3"}), user_server_config_path) + create_server_config(dumps({"key_path": "/key_path_3"}), user_default_server_config_path) extracted_config = _extract_config( - IslandCmdArgs(setup_only=False, server_config_path=user_server_config_path) + IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path) ) assert expected.__dict__ == extracted_config.__dict__ -def test_malformed_json(user_server_config_path): - create_server_config(BAD_JSON, user_server_config_path) +def test_cmd_config_overrides_everything( + deployment_server_config_path, cmd_server_config_path, user_default_server_config_path +): + expected = IslandConfigOptions({"key_path": "/key_path_4"}) + create_server_config(dumps({"key_path": "/key_path_2"}), deployment_server_config_path) + create_server_config(dumps({"key_path": "/key_path_3"}), user_default_server_config_path) + create_server_config(dumps({"key_path": "/key_path_4"}), cmd_server_config_path) + extracted_config = _extract_config( + IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path) + ) + assert expected.__dict__ == extracted_config.__dict__ + + +def test_malformed_json(cmd_server_config_path): + create_server_config(BAD_JSON, cmd_server_config_path) with pytest.raises(SystemExit): - _extract_config(IslandCmdArgs(setup_only=False, server_config_path=user_server_config_path)) + _extract_config(IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path)) From 3e32dbbc52eaa189b9c07442f4b9d796deecd044 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 26 Nov 2021 16:55:43 +0200 Subject: [PATCH 05/25] Docs: extract the server configuration workflow to a separate server_configuration.md page, which explains how to setup and use server_configuration.json This change extracts server_config.json usage into a single page, which can then be referred to from any page that requires island configuration --- docs/content/FAQ/_index.md | 4 + docs/content/reference/data_directory.md | 2 +- .../content/reference/server_configuration.md | 100 ++++++++++++++++++ docs/content/setup/docker.md | 32 +----- docs/content/setup/linux.md | 23 +--- docs/content/setup/windows.md | 20 +--- 6 files changed, 112 insertions(+), 69 deletions(-) create mode 100644 docs/content/reference/server_configuration.md diff --git a/docs/content/FAQ/_index.md b/docs/content/FAQ/_index.md index 49cc4e0b8..24f26b0be 100644 --- a/docs/content/FAQ/_index.md +++ b/docs/content/FAQ/_index.md @@ -174,6 +174,10 @@ The log enables you to see which requests were requested from the server and ext 2019-07-23 10:52:24,027 - report.py:580 - get_domain_issues() - INFO - Domain issues generated for reporting ``` +It's also possible to change the default log level by editing `log_level` value in a [server configuration file](../../reference/server_configuration). +You can use any of the default Python log levels. + + ### Infection Monkey agent logs The Infection Monkey agent log file can be found in the following paths on machines where it was executed: diff --git a/docs/content/reference/data_directory.md b/docs/content/reference/data_directory.md index 2ab7ca78e..2fc6605cd 100644 --- a/docs/content/reference/data_directory.md +++ b/docs/content/reference/data_directory.md @@ -22,7 +22,7 @@ On Windows, the default path is `%AppData%\monkey_island`. The location of the data directory is set in the `data_dir` field in the `server_config.json` file. -1. Create a custom `server_config.json` file and set the `data_dir` field. Its +1. [Create a custom server_config.json file](../server_configuration) and set the `data_dir` field. Its contents will look like: ```json diff --git a/docs/content/reference/server_configuration.md b/docs/content/reference/server_configuration.md new file mode 100644 index 000000000..60f0dd12c --- /dev/null +++ b/docs/content/reference/server_configuration.md @@ -0,0 +1,100 @@ +--- +title: "Server configuration" +date: 2021-11-26T12:00:19+02:00 +draft: true +pre: ' ' +weight: 1 +--- + +## Configuring the Island + +The Island Server(C&C) is configured by creating a `server_config.json` file. + +### Creating a configuration file + +Here's an example `server_config.json` with all options specified: +```json +{ + "log_level": "DEBUG", + "ssl_certificate": { + "ssl_certificate_file": "", + "ssl_certificate_key_file": "" + }, + "mongodb": { + "start_mongodb": true + }, + "data_dir": "/monkey_island_data" +} +``` + +Only relevant options can be specified, for example: +```json +{ + "ssl_certificate": { + "ssl_certificate_file": "", + "ssl_certificate_key_file": "" + } +} +``` + +### Applying configuration to the island + +#### AppImage (Linux) + +Specify the path to the `server_config.json` through a command line argument. + +Example: `./InfectionMonkey-v1.12.0.AppImage --server-config="/tmp/server_config.json"` + +#### Windows + +Move the created `server_config.json` to the install directory, monkey island directory. +If you haven't changed the default install directory, the path should look like: + +`C:\Program Files\Guardicore\Monkey Island\monkey\monkey_island\server_config.json` + +#### Docker + +Best way to configure the docker is to is to map server's [data directory](../data_directory) to a volume: + +1. Create a directory for server configuration and other files, e.g. `monkey_island_data`. If you already have it, + **make sure it's empty**. + + ```bash + mkdir ./monkey_island_data + chmod 700 ./monkey_island_data + ``` +1. Establish and populate the created directory with server files (modify the `VERSION` to the one you downloaded): +```bash +sudo docker run \ + --rm \ + --name monkey-island \ + --network=host \ + --user "$(id -u ${USER}):$(id -g ${USER})" \ + --volume "$(realpath ./monkey_island_data)":/monkey_island_data \ + guardicore/monkey-island:VERSION --setup-only +``` + +Once the volume is mapped, we can put `server_config.json` there. +`server_config.json` for docker **must** contain a valid data directory field and `start_mongodb` set to false. + +So, at minimum your `server_config.json` should look like this: + +```json +{ + "data_dir": "/monkey_island_data", + "mongodb": { + "start_mongodb": false + } +} +``` + +Then, the container can be launched by providing `server_config.json` path in the arguments: +```bash +sudo docker run \ + --rm \ + --name monkey-island \ + --network=host \ + --user "$(id -u ${USER}):$(id -g ${USER})" \ + --volume "$(realpath ./monkey_island_data)":/monkey_island_data \ + guardicore/monkey-island:VERSION --server-config="/monkey_island_data/server_config.json" +``` diff --git a/docs/content/setup/docker.md b/docs/content/setup/docker.md index db5979fc6..1feeec07d 100644 --- a/docs/content/setup/docker.md +++ b/docs/content/setup/docker.md @@ -73,28 +73,9 @@ If you are upgrading the Infection Monkey to a new version, be sure to remove any volumes associated with the previous version. {{% /notice %}} -1. Create a directory named `monkey_island_data`. If you already have it, - **make sure it's empty**. This will serve as the location where Infection - Monkey stores its configuration and runtime artifacts. +1. [Setup a volume with configuration file](../../reference/server_configuration/#docker). - ```bash - mkdir ./monkey_island_data - chmod 700 ./monkey_island_data - ``` - -1. Run Monkey Island with the `--setup-only` flag to populate the `./monkey_island_data` directory with a default `server_config.json` file. - - ```bash - sudo docker run \ - --rm \ - --name monkey-island \ - --network=host \ - --user "$(id -u ${USER}):$(id -g ${USER})" \ - --volume "$(realpath ./monkey_island_data)":/monkey_island_data \ - guardicore/monkey-island:VERSION --setup-only - ``` - -1. Move your `.crt` and `.key` files to `./monkey_island_data`. +1. Move your `.crt` and `.key` files to the volume created in the previous step (`./monkey_island_data`). 1. Make sure that your `.crt` and `.key` files are readable and writeable only by you. @@ -109,11 +90,6 @@ any volumes associated with the previous version. ```json {linenos=inline,hl_lines=["11-14"]} { "data_dir": "/monkey_island_data", - "log_level": "DEBUG", - "environment": { - "server_config": "password", - "deployment": "docker" - }, "mongodb": { "start_mongodb": false }, @@ -124,7 +100,7 @@ any volumes associated with the previous version. } ``` -1. Start the Monkey Island server: +1. Start/restart the Monkey Island server: ```bash sudo docker run \ @@ -134,7 +110,7 @@ any volumes associated with the previous version. --network=host \ --user "$(id -u ${USER}):$(id -g ${USER})" \ --volume "$(realpath ./monkey_island_data)":/monkey_island_data \ - guardicore/monkey-island:VERSION + guardicore/monkey-island:VERSION --server-config="/monkey_island_data/server_config.json" ``` ### 4. Accessing Monkey Island diff --git a/docs/content/setup/linux.md b/docs/content/setup/linux.md index 275330c2c..b1791c617 100644 --- a/docs/content/setup/linux.md +++ b/docs/content/setup/linux.md @@ -72,30 +72,11 @@ private certificate authority. chmod 600 ``` -1. Edit `$HOME/.monkey_island/server_config.json` to configure Monkey Island - to use your certificate. Your config should look something like this: - - ```json {linenos=inline,hl_lines=["11-14"]} - { - "data_dir": "~/.monkey_island", - "log_level": "DEBUG", - "environment": { - "server_config": "password", - "deployment": "linux" - }, - "mongodb": { - "start_mongodb": true - }, - "ssl_certificate": { - "ssl_certificate_file": "", - "ssl_certificate_key_file": "" - } - } - ``` +1. Create a [server configuration file and provide the path to the certificate](../../reference/server_configuration). 1. Start Monkey Island by running the Infection Monkey AppImage package: ```bash - ./InfectionMonkey-v1.12.0.AppImage + ./InfectionMonkey-v1.12.0.AppImage --server-config="/path/to/server_config.json" ``` 1. Access the Monkey Island web UI by pointing your browser at diff --git a/docs/content/setup/windows.md b/docs/content/setup/windows.md index f9fd5acaf..f76e80de1 100644 --- a/docs/content/setup/windows.md +++ b/docs/content/setup/windows.md @@ -39,25 +39,7 @@ private certificate authority. `%AppData%\monkey_island`. 1. Stop the Monkey Island process. 1. (Optional but recommended) Move your `.crt` and `.key` files to `%AppData%\monkey_island`. -1. Edit `%AppData%\monkey_island\server_config.json` to configure Monkey Island - to use your certificate. Your config should look something like this: - - ```json {linenos=inline,hl_lines=["11-14"]} - { - "log_level": "DEBUG", - "environment": { - "server_config": "password", - "deployment": "windows" - }, - "mongodb": { - "start_mongodb": true - }, - "ssl_certificate": { - "ssl_certificate_file": "", - "ssl_certificate_key_file": "" - } - } - ``` +1. Create a [server configuration file and provide the path to the certificate](../../reference/server_configuration). 1. Run the Monkey Island by clicking on the desktop shortcut. 1. Access the Monkey Island web UI by pointing your browser at From 811983a8b5e6a87d510dbc50a5aead2dd981427d Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 29 Nov 2021 11:35:22 +0200 Subject: [PATCH 06/25] Island: refactor/change exit() to the syntax of sys.exit --- monkey/monkey_island/cc/server_setup.py | 7 +++---- monkey/monkey_island/main.py | 6 ++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index 61d13a96a..a3c0cf750 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -3,7 +3,6 @@ import json import logging import sys from pathlib import Path -from sys import exit from threading import Thread import gevent.hub @@ -67,7 +66,7 @@ def _extract_config(island_args: IslandCmdArgs) -> IslandConfigOptions: return get_server_config(island_args) except json.JSONDecodeError as ex: print(f"Error loading server config: {ex}") - exit(1) + sys.exit(1) def _setup_data_dir(data_dir_path: Path): @@ -75,7 +74,7 @@ def _setup_data_dir(data_dir_path: Path): setup_data_dir(data_dir_path) except IncompatibleDataDirectory as ex: print(f"Incompatible data directory: {ex}") - exit(1) + sys.exit(1) def _exit_on_invalid_config_options(config_options: IslandConfigOptions): @@ -83,7 +82,7 @@ def _exit_on_invalid_config_options(config_options: IslandConfigOptions): island_config_options_validator.raise_on_invalid_options(config_options) except Exception as ex: print(f"Configuration error: {ex}") - exit(1) + sys.exit(1) def _configure_logging(config_options): diff --git a/monkey/monkey_island/main.py b/monkey/monkey_island/main.py index 19cf07d9f..ca91c054b 100644 --- a/monkey/monkey_island/main.py +++ b/monkey/monkey_island/main.py @@ -1,8 +1,10 @@ # This import patches other imports and needs to be first -import monkey_island.setup.gevent_setup # noqa: F401 isort:skip +import sys from monkey_island.cc.server_utils.island_logger import setup_default_failsafe_logging +import monkey_island.setup.gevent_setup # noqa: F401 isort:skip + def main(): # This is here in order to catch EVERYTHING, some functions are being called on @@ -11,7 +13,7 @@ def main(): setup_default_failsafe_logging() except Exception as ex: print(f"Error configuring logging: {ex}") - exit(1) + sys.exit(1) from monkey_island.cc.server_setup import run_monkey_island # noqa: E402 From 7e479ec3dfb855005a66e41e5c5a85cf09a9a5c7 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 29 Nov 2021 14:47:37 +0200 Subject: [PATCH 07/25] Island: improve readability in config_setup.py by renaming methods and adding default parameter to IslandConfigOptions --- monkey/monkey_island/cc/setup/config_setup.py | 14 +++++++------- .../cc/setup/island_config_options.py | 4 +++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/monkey/monkey_island/cc/setup/config_setup.py b/monkey/monkey_island/cc/setup/config_setup.py index 77652ddba..dee694b68 100644 --- a/monkey/monkey_island/cc/setup/config_setup.py +++ b/monkey/monkey_island/cc/setup/config_setup.py @@ -11,29 +11,29 @@ logger = getLogger(__name__) def get_server_config(island_args: IslandCmdArgs) -> IslandConfigOptions: - config = IslandConfigOptions({}) + config = IslandConfigOptions() - update_config_from_file(config, PACKAGE_CONFIG_PATH) + _update_config_from_file(config, PACKAGE_CONFIG_PATH) - update_config_from_file(config, USER_CONFIG_PATH) + _update_config_from_file(config, USER_CONFIG_PATH) if island_args.server_config_path: path_to_config = expand_path(island_args.server_config_path) - update_config_from_file(config, path_to_config) + _update_config_from_file(config, path_to_config) return config -def update_config_from_file(config: IslandConfigOptions, config_path: Path): +def _update_config_from_file(config: IslandConfigOptions, config_path: Path): try: - config_from_file = load_server_config_from_file(config_path) + config_from_file = _load_server_config_from_file(config_path) config.update(config_from_file) logger.info(f"Server config updated from {config_path}") except OSError: logger.info(f"Server config not found in path {config_path}") -def load_server_config_from_file(server_config_path) -> IslandConfigOptions: +def _load_server_config_from_file(server_config_path) -> IslandConfigOptions: with open(server_config_path, "r") as f: config_content = f.read() config = json.loads(config_content) diff --git a/monkey/monkey_island/cc/setup/island_config_options.py b/monkey/monkey_island/cc/setup/island_config_options.py index 12b141923..a9cf66785 100644 --- a/monkey/monkey_island/cc/setup/island_config_options.py +++ b/monkey/monkey_island/cc/setup/island_config_options.py @@ -12,7 +12,9 @@ from monkey_island.cc.server_utils.consts import ( class IslandConfigOptions: - def __init__(self, config_contents: dict): + def __init__(self, config_contents: dict = None): + if not config_contents: + config_contents = {} self.data_dir = expand_path(config_contents.get("data_dir", DEFAULT_DATA_DIR)) self.log_level = config_contents.get("log_level", DEFAULT_LOG_LEVEL) From 04feb1b31d921e1d59dc40d27d63b6e07e7554f8 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 29 Nov 2021 15:09:20 +0200 Subject: [PATCH 08/25] Island,UT: move PACKAGE_CONFIG_PATH and USER_CONFIG_PATH consts to config_setup.py, where they are used --- monkey/monkey_island/cc/server_utils/consts.py | 3 --- monkey/monkey_island/cc/setup/config_setup.py | 6 +++++- .../monkey_island/cc/server_utils/test_consts.py | 10 ---------- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index f5cff557c..29fe78933 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -36,9 +36,6 @@ MONGO_EXECUTABLE_PATH = ( ) MONGO_CONNECTION_TIMEOUT = 15 -PACKAGE_CONFIG_PATH = Path(MONKEY_ISLAND_ABS_PATH, "cc", SERVER_CONFIG_FILENAME) -USER_CONFIG_PATH = Path(os.getcwd(), SERVER_CONFIG_FILENAME) - DEFAULT_LOG_LEVEL = "INFO" DEFAULT_START_MONGO_DB = True diff --git a/monkey/monkey_island/cc/setup/config_setup.py b/monkey/monkey_island/cc/setup/config_setup.py index dee694b68..3dca8e3dc 100644 --- a/monkey/monkey_island/cc/setup/config_setup.py +++ b/monkey/monkey_island/cc/setup/config_setup.py @@ -1,14 +1,18 @@ import json +import os from logging import getLogger from pathlib import Path from common.utils.file_utils import expand_path from monkey_island.cc.arg_parser import IslandCmdArgs -from monkey_island.cc.server_utils.consts import PACKAGE_CONFIG_PATH, USER_CONFIG_PATH +from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH, SERVER_CONFIG_FILENAME from monkey_island.cc.setup.island_config_options import IslandConfigOptions logger = getLogger(__name__) +PACKAGE_CONFIG_PATH = Path(MONKEY_ISLAND_ABS_PATH, "cc", SERVER_CONFIG_FILENAME) +USER_CONFIG_PATH = Path(os.getcwd(), SERVER_CONFIG_FILENAME) + def get_server_config(island_args: IslandCmdArgs) -> IslandConfigOptions: config = IslandConfigOptions() diff --git a/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_consts.py b/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_consts.py index 996226e0d..8d0b0b1b4 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_consts.py +++ b/monkey/tests/unit_tests/monkey_island/cc/server_utils/test_consts.py @@ -1,5 +1,4 @@ import os -import platform from monkey_island.cc.server_utils import consts @@ -7,12 +6,3 @@ from monkey_island.cc.server_utils import consts def test_monkey_island_abs_path(): assert consts.MONKEY_ISLAND_ABS_PATH.endswith("monkey_island") assert os.path.isdir(consts.MONKEY_ISLAND_ABS_PATH) - - -def test_default_server_config_file_path(): - if platform.system() == "Windows": - server_file_path = f"{consts.MONKEY_ISLAND_ABS_PATH}\\cc\\{consts.SERVER_CONFIG_FILENAME}" - else: - server_file_path = f"{consts.MONKEY_ISLAND_ABS_PATH}/cc/{consts.SERVER_CONFIG_FILENAME}" - - assert consts.PACKAGE_CONFIG_PATH == server_file_path From ffe9a6503743660221b892cef8203c5a791d8974 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 29 Nov 2021 17:21:46 +0200 Subject: [PATCH 09/25] Docs: improve the documentation by specifying that user can use log levels `info` and `debug`, instead of saying "default Python log levels" --- docs/content/FAQ/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/FAQ/_index.md b/docs/content/FAQ/_index.md index 24f26b0be..922a5e803 100644 --- a/docs/content/FAQ/_index.md +++ b/docs/content/FAQ/_index.md @@ -175,7 +175,7 @@ The log enables you to see which requests were requested from the server and ext ``` It's also possible to change the default log level by editing `log_level` value in a [server configuration file](../../reference/server_configuration). -You can use any of the default Python log levels. +`log_level` can be set to `info`(default, less verbose) or `debug`(more verbose). ### Infection Monkey agent logs From 68ea983458a13624b413cb76242a91bce1244180 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 29 Nov 2021 17:27:26 +0200 Subject: [PATCH 10/25] UT: fixed the path to consts in test_server_setup.py --- .../unit_tests/monkey_island/cc/setup/test_server_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py index c3a6ee636..e564c7c81 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py +++ b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py @@ -37,7 +37,7 @@ def create_server_config(config_contents: str, server_config_path: Path): @pytest.fixture(autouse=True) def mock_deployment_config_path(monkeypatch, deployment_server_config_path): monkeypatch.setattr( - "monkey_island.cc.setup.config_setup.DEFAULT_SERVER_CONFIG_PATH", + "monkey_island.cc.setup.config_setup.PACKAGE_CONFIG_PATH", deployment_server_config_path, ) From e95df875beba116d8082b5498fc124d2d576d87b Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 30 Nov 2021 10:27:09 +0200 Subject: [PATCH 11/25] Island: fix a bug in server's config options extraction that caused unspecified properties to get overridden by defaults --- monkey/monkey_island/cc/setup/config_setup.py | 5 ++- .../cc/setup/island_config_options.py | 4 +-- .../cc/setup/test_server_setup.py | 35 +++++++++++++------ 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/monkey/monkey_island/cc/setup/config_setup.py b/monkey/monkey_island/cc/setup/config_setup.py index 3dca8e3dc..c38ee791b 100644 --- a/monkey/monkey_island/cc/setup/config_setup.py +++ b/monkey/monkey_island/cc/setup/config_setup.py @@ -37,8 +37,7 @@ def _update_config_from_file(config: IslandConfigOptions, config_path: Path): logger.info(f"Server config not found in path {config_path}") -def _load_server_config_from_file(server_config_path) -> IslandConfigOptions: +def _load_server_config_from_file(server_config_path) -> dict: with open(server_config_path, "r") as f: config_content = f.read() - config = json.loads(config_content) - return IslandConfigOptions(config) + return json.loads(config_content) diff --git a/monkey/monkey_island/cc/setup/island_config_options.py b/monkey/monkey_island/cc/setup/island_config_options.py index a9cf66785..71e9ba27f 100644 --- a/monkey/monkey_island/cc/setup/island_config_options.py +++ b/monkey/monkey_island/cc/setup/island_config_options.py @@ -34,5 +34,5 @@ class IslandConfigOptions: ) ) - def update(self, target: IslandConfigOptions): - self.__dict__.update(target.__dict__) + def update(self, target: dict): + self.__dict__.update(target) diff --git a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py index e564c7c81..7517aa665 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py +++ b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py @@ -43,10 +43,10 @@ def mock_deployment_config_path(monkeypatch, deployment_server_config_path): @pytest.fixture(autouse=True) -def mock_user_config_path(monkeypatch, deployment_server_config_path): +def mock_user_config_path(monkeypatch, user_default_server_config_path): monkeypatch.setattr( "monkey_island.cc.setup.config_setup.USER_CONFIG_PATH", - deployment_server_config_path, + user_default_server_config_path, ) @@ -59,8 +59,8 @@ def test_extract_config_defaults(): def test_deployment_config_overrides_defaults(deployment_server_config_path): - expected = IslandConfigOptions({"key_path": "/key_path_2"}) - create_server_config(dumps({"key_path": "/key_path_2"}), deployment_server_config_path) + expected = IslandConfigOptions({"log_level": "/log_level_2"}) + create_server_config(dumps({"log_level": "/log_level_2"}), deployment_server_config_path) assert ( expected.__dict__ == _extract_config(IslandCmdArgs(setup_only=False, server_config_path=None)).__dict__ @@ -70,9 +70,9 @@ def test_deployment_config_overrides_defaults(deployment_server_config_path): def test_user_config_overrides_deployment( deployment_server_config_path, cmd_server_config_path, user_default_server_config_path ): - expected = IslandConfigOptions({"key_path": "/key_path_3"}) - create_server_config(dumps({"key_path": "/key_path_2"}), deployment_server_config_path) - create_server_config(dumps({"key_path": "/key_path_3"}), user_default_server_config_path) + expected = IslandConfigOptions({"log_level": "/log_level_3"}) + create_server_config(dumps({"log_level": "/log_level_2"}), deployment_server_config_path) + create_server_config(dumps({"log_level": "/log_level_3"}), user_default_server_config_path) extracted_config = _extract_config( IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path) ) @@ -82,10 +82,23 @@ def test_user_config_overrides_deployment( def test_cmd_config_overrides_everything( deployment_server_config_path, cmd_server_config_path, user_default_server_config_path ): - expected = IslandConfigOptions({"key_path": "/key_path_4"}) - create_server_config(dumps({"key_path": "/key_path_2"}), deployment_server_config_path) - create_server_config(dumps({"key_path": "/key_path_3"}), user_default_server_config_path) - create_server_config(dumps({"key_path": "/key_path_4"}), cmd_server_config_path) + expected = IslandConfigOptions({"log_level": "/log_level_4"}) + create_server_config(dumps({"log_level": "/log_level_2"}), deployment_server_config_path) + create_server_config(dumps({"log_level": "/log_level_3"}), user_default_server_config_path) + create_server_config(dumps({"log_level": "/log_level_4"}), cmd_server_config_path) + extracted_config = _extract_config( + IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path) + ) + assert expected.__dict__ == extracted_config.__dict__ + + +def test_not_overriding_unspecified_values( + deployment_server_config_path, cmd_server_config_path, user_default_server_config_path +): + expected = IslandConfigOptions({"log_level": "/log_level_4", "data_dir": "/data_dir1"}) + create_server_config(dumps({"data_dir": "/data_dir1"}), deployment_server_config_path) + create_server_config(dumps({"log_level": "/log_level_3"}), user_default_server_config_path) + create_server_config(dumps({"log_level": "/log_level_4"}), cmd_server_config_path) extracted_config = _extract_config( IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path) ) From 03566d2966b9d76b45d650f9607f1a043993d135 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 30 Nov 2021 10:52:28 +0200 Subject: [PATCH 12/25] Island: remove the server config extraction from server_config.json in island's cwd All deployments can be configured via command line OR by modifying the server_config.json that comes with the deployment --- monkey/monkey_island/cc/setup/config_setup.py | 4 --- .../cc/setup/test_server_setup.py | 26 ------------------- 2 files changed, 30 deletions(-) diff --git a/monkey/monkey_island/cc/setup/config_setup.py b/monkey/monkey_island/cc/setup/config_setup.py index c38ee791b..c362feb82 100644 --- a/monkey/monkey_island/cc/setup/config_setup.py +++ b/monkey/monkey_island/cc/setup/config_setup.py @@ -1,5 +1,4 @@ import json -import os from logging import getLogger from pathlib import Path @@ -11,7 +10,6 @@ from monkey_island.cc.setup.island_config_options import IslandConfigOptions logger = getLogger(__name__) PACKAGE_CONFIG_PATH = Path(MONKEY_ISLAND_ABS_PATH, "cc", SERVER_CONFIG_FILENAME) -USER_CONFIG_PATH = Path(os.getcwd(), SERVER_CONFIG_FILENAME) def get_server_config(island_args: IslandCmdArgs) -> IslandConfigOptions: @@ -19,8 +17,6 @@ def get_server_config(island_args: IslandCmdArgs) -> IslandConfigOptions: _update_config_from_file(config, PACKAGE_CONFIG_PATH) - _update_config_from_file(config, USER_CONFIG_PATH) - if island_args.server_config_path: path_to_config = expand_path(island_args.server_config_path) _update_config_from_file(config, path_to_config) diff --git a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py index 7517aa665..a6f4c59ca 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py +++ b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py @@ -17,12 +17,6 @@ def cmd_server_config_path(tmpdir) -> Path: return tmpdir / "fake_server_config.json" -@pytest.fixture -def user_default_server_config_path(tmpdir) -> Path: - # Represents the config that can be put into the install dir - return tmpdir / "fake_server_config2.json" - - @pytest.fixture def deployment_server_config_path(tmpdir) -> Path: # Represents the config that is built in, deployment specific @@ -42,14 +36,6 @@ def mock_deployment_config_path(monkeypatch, deployment_server_config_path): ) -@pytest.fixture(autouse=True) -def mock_user_config_path(monkeypatch, user_default_server_config_path): - monkeypatch.setattr( - "monkey_island.cc.setup.config_setup.USER_CONFIG_PATH", - user_default_server_config_path, - ) - - def test_extract_config_defaults(): expected = IslandConfigOptions({}) assert ( @@ -67,18 +53,6 @@ def test_deployment_config_overrides_defaults(deployment_server_config_path): ) -def test_user_config_overrides_deployment( - deployment_server_config_path, cmd_server_config_path, user_default_server_config_path -): - expected = IslandConfigOptions({"log_level": "/log_level_3"}) - create_server_config(dumps({"log_level": "/log_level_2"}), deployment_server_config_path) - create_server_config(dumps({"log_level": "/log_level_3"}), user_default_server_config_path) - extracted_config = _extract_config( - IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path) - ) - assert expected.__dict__ == extracted_config.__dict__ - - def test_cmd_config_overrides_everything( deployment_server_config_path, cmd_server_config_path, user_default_server_config_path ): From 06f31791fc4dc9741df72120ff9cede310031bbf Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 30 Nov 2021 12:12:29 +0200 Subject: [PATCH 13/25] Island, UT: fix island config option extraction to also expand paths and add a UT for that --- .../cc/setup/island_config_options.py | 40 +++++++++++++++---- .../cc/setup/test_server_setup.py | 29 ++++++++------ 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/monkey/monkey_island/cc/setup/island_config_options.py b/monkey/monkey_island/cc/setup/island_config_options.py index 71e9ba27f..384ed3079 100644 --- a/monkey/monkey_island/cc/setup/island_config_options.py +++ b/monkey/monkey_island/cc/setup/island_config_options.py @@ -1,5 +1,7 @@ from __future__ import annotations +from dpath import util + from common.utils.file_utils import expand_path from monkey_island.cc.server_utils.consts import ( DEFAULT_CERTIFICATE_PATHS, @@ -10,29 +12,51 @@ from monkey_island.cc.server_utils.consts import ( DEFAULT_START_MONGO_DB, ) +_DATA_DIR = "data_dir" +_SSL_CERT = "ssl_certificate" +_SSL_CERT_FILE = "ssl_certificate_file" +_SSL_CERT_KEY = "ssl_certificate_key_file" +_MONGODB = "mongodb" +_START_MONGODB = "start_mongodb" +_LOG_LEVEL = "log_level" + class IslandConfigOptions: def __init__(self, config_contents: dict = None): if not config_contents: config_contents = {} - self.data_dir = expand_path(config_contents.get("data_dir", DEFAULT_DATA_DIR)) + self.data_dir = expand_path(config_contents.get(_DATA_DIR, DEFAULT_DATA_DIR)) - self.log_level = config_contents.get("log_level", DEFAULT_LOG_LEVEL) + self.log_level = config_contents.get(_LOG_LEVEL, DEFAULT_LOG_LEVEL) self.start_mongodb = config_contents.get( - "mongodb", {"start_mongodb": DEFAULT_START_MONGO_DB} - ).get("start_mongodb", DEFAULT_START_MONGO_DB) + _MONGODB, {_START_MONGODB: DEFAULT_START_MONGO_DB} + ).get(_START_MONGODB, DEFAULT_START_MONGO_DB) self.crt_path = expand_path( - config_contents.get("ssl_certificate", DEFAULT_CERTIFICATE_PATHS).get( - "ssl_certificate_file", DEFAULT_CRT_PATH + config_contents.get(_SSL_CERT, DEFAULT_CERTIFICATE_PATHS).get( + _SSL_CERT_FILE, DEFAULT_CRT_PATH ) ) self.key_path = expand_path( - config_contents.get("ssl_certificate", DEFAULT_CERTIFICATE_PATHS).get( - "ssl_certificate_key_file", DEFAULT_KEY_PATH + config_contents.get(_SSL_CERT, DEFAULT_CERTIFICATE_PATHS).get( + _SSL_CERT_KEY, DEFAULT_KEY_PATH ) ) def update(self, target: dict): + target = self._expand_config_paths(target) self.__dict__.update(target) + + @staticmethod + def _expand_config_paths(config: dict) -> dict: + config_paths = [_DATA_DIR, f"{_SSL_CERT}.{_SSL_CERT_FILE}", f"{_SSL_CERT}.{_SSL_CERT_KEY}"] + + for config_path in config_paths: + try: + expanded_val = expand_path(util.get(config, config_path, ".")) + util.set(config, config_path, expanded_val, ".") + except KeyError: + pass + + return config diff --git a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py index a6f4c59ca..226842607 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py +++ b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py @@ -6,6 +6,7 @@ import pytest import monkey_island.cc.setup.config_setup # noqa: F401 from monkey_island.cc.arg_parser import IslandCmdArgs from monkey_island.cc.server_setup import _extract_config +from monkey_island.cc.server_utils.file_utils import is_windows_os from monkey_island.cc.setup.island_config_options import IslandConfigOptions BAD_JSON = '{"data_dir": "C:\\test\\test"' @@ -53,32 +54,36 @@ def test_deployment_config_overrides_defaults(deployment_server_config_path): ) -def test_cmd_config_overrides_everything( - deployment_server_config_path, cmd_server_config_path, user_default_server_config_path -): - expected = IslandConfigOptions({"log_level": "/log_level_4"}) +def test_cmd_config_overrides_everything(deployment_server_config_path, cmd_server_config_path): + expected = IslandConfigOptions({"log_level": "/log_level_3"}) create_server_config(dumps({"log_level": "/log_level_2"}), deployment_server_config_path) - create_server_config(dumps({"log_level": "/log_level_3"}), user_default_server_config_path) - create_server_config(dumps({"log_level": "/log_level_4"}), cmd_server_config_path) + create_server_config(dumps({"log_level": "/log_level_3"}), cmd_server_config_path) extracted_config = _extract_config( IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path) ) assert expected.__dict__ == extracted_config.__dict__ -def test_not_overriding_unspecified_values( - deployment_server_config_path, cmd_server_config_path, user_default_server_config_path -): - expected = IslandConfigOptions({"log_level": "/log_level_4", "data_dir": "/data_dir1"}) +def test_not_overriding_unspecified_values(deployment_server_config_path, cmd_server_config_path): + expected = IslandConfigOptions({"log_level": "/log_level_2", "data_dir": "/data_dir1"}) create_server_config(dumps({"data_dir": "/data_dir1"}), deployment_server_config_path) - create_server_config(dumps({"log_level": "/log_level_3"}), user_default_server_config_path) - create_server_config(dumps({"log_level": "/log_level_4"}), cmd_server_config_path) + create_server_config(dumps({"log_level": "/log_level_2"}), cmd_server_config_path) extracted_config = _extract_config( IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path) ) assert expected.__dict__ == extracted_config.__dict__ +def test_paths_get_expanded(deployment_server_config_path): + if is_windows_os(): + path = "%temp%/path" + else: + path = "$HOME/path" + create_server_config(dumps({"data_dir": path}), deployment_server_config_path) + extracted_config = _extract_config(IslandCmdArgs(setup_only=False, server_config_path=None)) + assert not extracted_config.data_dir == path + + def test_malformed_json(cmd_server_config_path): create_server_config(BAD_JSON, cmd_server_config_path) with pytest.raises(SystemExit): From 0a32ac888e8f1b3a002a104e7cdd476a23d473bf Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 30 Nov 2021 15:44:46 +0200 Subject: [PATCH 14/25] Docs: moved server configuration and common configuration operation docs to corresponding deployment option setup pages --- docs/content/FAQ/_index.md | 20 +--- .../content/reference/server_configuration.md | 65 ++----------- docs/content/setup/docker.md | 97 +++++++++++++------ docs/content/setup/linux.md | 47 +++++++-- docs/content/setup/windows.md | 34 ++++++- 5 files changed, 143 insertions(+), 120 deletions(-) diff --git a/docs/content/FAQ/_index.md b/docs/content/FAQ/_index.md index 922a5e803..0ccbe2872 100644 --- a/docs/content/FAQ/_index.md +++ b/docs/content/FAQ/_index.md @@ -201,25 +201,7 @@ The logs contain information about the internals of the Infection Monkey agent's 2019-07-22 19:16:45,013 [77598:140654230214464:DEBUG] connectionpool._make_request.396: https://updates.infectionmonkey.com:443 "GET / HTTP/1.1" 200 61 ``` -### How do I change the log level of the Monkey Island logger? - -The log level of the Monkey Island logger is set in the `log_level` field -in the `server_config.json` file (located in the [data directory]({{< ref "/reference/data_directory" >}})). -Make sure to leave everything else in `server_config.json` unchanged: - -```json -{ - ... - "log_level": "DEBUG", - ... -} -``` - -Logging levels correspond to [the logging level constants in python](https://docs.python.org/3.7/library/logging.html#logging-levels). - -To apply the changes, reset the Monkey Island process. -On Linux, use `sudo systemctl restart monkey-island.service`. -On Windows, restart the program. +Logging level/verbosity can also be changed, see [setup page](../../setup) for you operating system. ## Running the Infection Monkey in a production environment diff --git a/docs/content/reference/server_configuration.md b/docs/content/reference/server_configuration.md index 60f0dd12c..9e470a19b 100644 --- a/docs/content/reference/server_configuration.md +++ b/docs/content/reference/server_configuration.md @@ -37,64 +37,11 @@ Only relevant options can be specified, for example: } ``` -### Applying configuration to the island +### Configuration options -#### AppImage (Linux) +See setup instructions for your operating system to understand how to apply these. -Specify the path to the `server_config.json` through a command line argument. - -Example: `./InfectionMonkey-v1.12.0.AppImage --server-config="/tmp/server_config.json"` - -#### Windows - -Move the created `server_config.json` to the install directory, monkey island directory. -If you haven't changed the default install directory, the path should look like: - -`C:\Program Files\Guardicore\Monkey Island\monkey\monkey_island\server_config.json` - -#### Docker - -Best way to configure the docker is to is to map server's [data directory](../data_directory) to a volume: - -1. Create a directory for server configuration and other files, e.g. `monkey_island_data`. If you already have it, - **make sure it's empty**. - - ```bash - mkdir ./monkey_island_data - chmod 700 ./monkey_island_data - ``` -1. Establish and populate the created directory with server files (modify the `VERSION` to the one you downloaded): -```bash -sudo docker run \ - --rm \ - --name monkey-island \ - --network=host \ - --user "$(id -u ${USER}):$(id -g ${USER})" \ - --volume "$(realpath ./monkey_island_data)":/monkey_island_data \ - guardicore/monkey-island:VERSION --setup-only -``` - -Once the volume is mapped, we can put `server_config.json` there. -`server_config.json` for docker **must** contain a valid data directory field and `start_mongodb` set to false. - -So, at minimum your `server_config.json` should look like this: - -```json -{ - "data_dir": "/monkey_island_data", - "mongodb": { - "start_mongodb": false - } -} -``` - -Then, the container can be launched by providing `server_config.json` path in the arguments: -```bash -sudo docker run \ - --rm \ - --name monkey-island \ - --network=host \ - --user "$(id -u ${USER}):$(id -g ${USER})" \ - --volume "$(realpath ./monkey_island_data)":/monkey_island_data \ - guardicore/monkey-island:VERSION --server-config="/monkey_island_data/server_config.json" -``` + - `log_level` - can be set to `"DEBUG"`(verbose), `"INFO"`(less verbose) or `"ERROR"`(silent, except errors). + - `ssl_certificate` - contains paths for files, required to run the Island server with custom certificate. + - `data_dir` - path to a writeable directory where the Island will store the database and other files. + - `mongodb` - options for MongoDB. Should not be changed unless you want to run your own instance of MongoDB. diff --git a/docs/content/setup/docker.md b/docs/content/setup/docker.md index 1feeec07d..7c5645bb9 100644 --- a/docs/content/setup/docker.md +++ b/docs/content/setup/docker.md @@ -49,12 +49,12 @@ any MongoDB containers or volumes associated with the previous version. mongo:4.2 ``` -### 3a. Start Monkey Island with default certificate +### 3. Start Monkey Island with default certificate By default, Infection Monkey comes with a [self-signed SSL certificate](https://aboutssl.org/what-is-self-sign-certificate/). In enterprise or other security-sensitive environments, it is recommended that the user [provide Infection Monkey with a -certificate](#3b-start-monkey-island-with-user-provided-certificate) that has +certificate](#start-monkey-island-with-user-provided-certificate) that has been signed by a private certificate authority. 1. Run the Monkey Island server @@ -67,55 +67,90 @@ been signed by a private certificate authority. guardicore/monkey-island:VERSION ``` -### 3b. Start Monkey Island with user-provided certificate -{{% notice info %}} -If you are upgrading the Infection Monkey to a new version, be sure to remove -any volumes associated with the previous version. -{{% /notice %}} +### 4. Accessing Monkey Island -1. [Setup a volume with configuration file](../../reference/server_configuration/#docker). +After the Monkey Island docker container starts, you can access Monkey Island by pointing your browser at `https://localhost:5000`. -1. Move your `.crt` and `.key` files to the volume created in the previous step (`./monkey_island_data`). +## Configuring the server -1. Make sure that your `.crt` and `.key` files are readable and writeable only by you. +You can configure the server by mounting a volume and specifying a + [server configuration file](../../reference/server_configuration): +1. Create a directory for server configuration file, e.g. `monkey_island_data`: ```bash - chmod 600 ./monkey_island_data/ - chmod 600 ./monkey_island_data/ + mkdir ./monkey_island_data + chmod 700 ./monkey_island_data ``` +1. Move your `server_config.json` file to `./monkey_island_data` directory. +1. Run the container with a mounted volume, specify the path to the `server_config.json`: +```bash +sudo docker run \ + --rm \ + --name monkey-island \ + --network=host \ + --user "$(id -u ${USER}):$(id -g ${USER})" \ + --volume "$(realpath ./monkey_island_data)":/monkey_island_data \ + guardicore/monkey-island:VERSION --setup-only --server-config="/monkey_island_data/server_config.json" +``` -1. Edit `./monkey_island_data/server_config.json` to configure Monkey Island - to use your certificate. Your config should look something like this: +### Start Monkey Island with user-provided certificate - ```json {linenos=inline,hl_lines=["11-14"]} +By default, Infection Monkey comes with a [self-signed SSL +certificate](https://aboutssl.org/what-is-self-sign-certificate/). In +enterprise or other security-sensitive environments, it is recommended that the +user provide Infection Monkey with a certificate that has been signed by a +private certificate authority. + +1. Terminate the docker container it's already running. +1. Move your `.crt` and `.key` files to `./monkey_island_data` (directory created for the volume). +1. Make sure that your `.crt` and `.key` files are readable only by you. + ```bash + chmod 600 + chmod 600 + ``` +1. Modify the [server configuration file](../../reference/server_configuration) and add the following lines: + ```json { - "data_dir": "/monkey_island_data", - "mongodb": { - "start_mongodb": false - }, - "ssl_certificate": { - "ssl_certificate_file": "/monkey_island_data/", - "ssl_certificate_key_file": "/monkey_island_data/" - } + "ssl_certificate": { + "ssl_certificate_file": "/monkey_island_data/my_cert.crt", + "ssl_certificate_key_file": "/monkey_island_data/my_key.key" + } } ``` - -1. Start/restart the Monkey Island server: - +1. Run the container with a mounted volume, specify the path to the `server_config.json`: ```bash sudo docker run \ - --tty \ - --interactive \ + --rm \ --name monkey-island \ --network=host \ --user "$(id -u ${USER}):$(id -g ${USER})" \ --volume "$(realpath ./monkey_island_data)":/monkey_island_data \ - guardicore/monkey-island:VERSION --server-config="/monkey_island_data/server_config.json" + guardicore/monkey-island:VERSION --setup-only --server-config="/monkey_island_data/server_config.json" ``` +1. Access the Monkey Island web UI by pointing your browser at + `https://localhost:5000`. -### 4. Accessing Monkey Island +### Change logging level -After the Monkey Island docker container starts, you can access Monkey Island by pointing your browser at `https://localhost:5000`. +1. Stop the docker container it's already running. +1. Modify the [server configuration file](../../reference/server_configuration) by adding the following lines: + ```json + { + "log_level": "INFO" + } + ``` +1. Run the container with a mounted volume, specify the path to the `server_config.json`: + ```bash + sudo docker run \ + --rm \ + --name monkey-island \ + --network=host \ + --user "$(id -u ${USER}):$(id -g ${USER})" \ + --volume "$(realpath ./monkey_island_data)":/monkey_island_data \ + guardicore/monkey-island:VERSION --setup-only --server-config="/monkey_island_data/server_config.json" + ``` +1. Access the Monkey Island web UI by pointing your browser at + `https://localhost:5000`. ## Upgrading diff --git a/docs/content/setup/linux.md b/docs/content/setup/linux.md index b1791c617..2bf318a6b 100644 --- a/docs/content/setup/linux.md +++ b/docs/content/setup/linux.md @@ -46,6 +46,14 @@ do, see the [FAQ]({{< ref >}}) for more information. {{% /notice %}} +## Configuring the server + +You can configure the server by creating +a [server configuration file](../../reference/server_configuration) and +providing a path to it via command line parameters: + +`./InfectionMonkey-v1.12.0.AppImage --server-config="/path/to/server_config.json"` + ### Start Monkey Island with user-provided certificate By default, Infection Monkey comes with a [self-signed SSL @@ -54,13 +62,7 @@ enterprise or other security-sensitive environments, it is recommended that the user provide Infection Monkey with a certificate that has been signed by a private certificate authority. -1. Run the Infection Monkey AppImage package with the `--setup-only` flag to - populate the `$HOME/.monkey_island` directory with a default - `server_config.json` file. - - ```bash - ./InfectionMonkey-v1.12.0.AppImage --setup-only - ``` +1. Terminate the Island process if it's already running. 1. (Optional but recommended) Move your `.crt` and `.key` files to `$HOME/.monkey_island`. @@ -73,6 +75,37 @@ private certificate authority. ``` 1. Create a [server configuration file and provide the path to the certificate](../../reference/server_configuration). +Server configuration file should look something like: + +```json +{ + "ssl_certificate": { + "ssl_certificate_file": "$HOME/.monkey_island/my_cert.crt", + "ssl_certificate_key_file": "$HOME/.monkey_island/my_key.key" + } +} +``` + +1. Start Monkey Island by running the Infection Monkey AppImage package: + ```bash + ./InfectionMonkey-v1.12.0.AppImage --server-config="/path/to/server_config.json" + ``` + +1. Access the Monkey Island web UI by pointing your browser at + `https://localhost:5000`. + +### Change logging level + +1. Terminate the Island process if it's already running. + +1. Create a [server configuration file](../../reference/server_configuration). +Server configuration file should look something like: + +```json +{ + "log_level": "INFO" +} +``` 1. Start Monkey Island by running the Infection Monkey AppImage package: ```bash diff --git a/docs/content/setup/windows.md b/docs/content/setup/windows.md index f76e80de1..cf9dd7e2b 100644 --- a/docs/content/setup/windows.md +++ b/docs/content/setup/windows.md @@ -26,6 +26,11 @@ do, see the [FAQ]({{< ref "/faq/#i-updated-to-a-new-version-of-the-infection-monkey-and-im-being-asked-to-delete-my-existing-data-directory-why" >}}) for more information. {{% /notice %}} +> +## Configuring the server + +You can configure the server by editing [the configuration file](../../reference/server_configuration) located +in installation directory. By default, the path should be `C:\Program Files\Guardicore\Monkey Island\monkey\monkey_island\cc\server_config.json`. ### Start Monkey Island with user-provided certificate @@ -34,14 +39,35 @@ enterprise or other security-sensitive environments, it is recommended that the user provide Infection Monkey with a certificate that has been signed by a private certificate authority. -1. If you haven't already, run the Monkey Island by clicking on the desktop - shortcut. This will populate MongoDB, as well as create and populate - `%AppData%\monkey_island`. 1. Stop the Monkey Island process. 1. (Optional but recommended) Move your `.crt` and `.key` files to `%AppData%\monkey_island`. -1. Create a [server configuration file and provide the path to the certificate](../../reference/server_configuration). +1. Modify the `server_config.json` (by default located in `C:\Program Files\Guardicore\Monkey Island\monkey\monkey_island\cc\server_config.json`) by adding the following lines: + ```json + { + ... + "ssl_certificate": { + "ssl_certificate_file": "%AppData%\\monkey_island\\my_cert.crt", + "ssl_certificate_key_file": "%AppData%\\monkey_island\\my_key.key" + }, + ... + } + ``` 1. Run the Monkey Island by clicking on the desktop shortcut. +1. Access the Monkey Island web UI by pointing your browser at + `https://localhost:5000`. +### Change logging level + +1. Stop the Island server. +1. Modify the `server_config.json` (by default located in `C:\Program Files\Guardicore\Monkey Island\monkey\monkey_island\cc\server_config.json`) by adding the following lines: + ```json + { + ... + "log_level": "INFO", + ... + } + ``` +1. Run the Monkey Island by clicking on the desktop shortcut. 1. Access the Monkey Island web UI by pointing your browser at `https://localhost:5000`. From 0b7da7ed679f0aa5d52da7257ae41e6950cc2efb Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 30 Nov 2021 12:48:03 -0500 Subject: [PATCH 15/25] Docs: Fix "logging" links in FAQ --- docs/content/FAQ/_index.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/content/FAQ/_index.md b/docs/content/FAQ/_index.md index 0ccbe2872..bba8d6cda 100644 --- a/docs/content/FAQ/_index.md +++ b/docs/content/FAQ/_index.md @@ -16,9 +16,8 @@ Below are some of the most common questions we receive about the Infection Monke - [Should I run the Infection Monkey continuously?](#should-i-run-the-infection-monkey-continuously) - [Which queries does the Infection Monkey perform to the internet exactly?](#which-queries-does-the-infection-monkey-perform-to-the-internet-exactly) - [Logging and how to find logs](#logging-and-how-to-find-logs) - - [Monkey Island server](#monkey-island-server) - - [Infection Monkey agent](#infection-monkey-agent) - - [How do I change the log level of the Monkey Island logger?](#how-do-i-change-the-log-level-of-the-monkey-island-logger) + - [Monkey Island server logs](#monkey-island-server-logs) + - [Infection Monkey agent logs](#infection-monkey-agent-logs) - [Running the Infection Monkey in a production environment](#running-the-infection-monkey-in-a-production-environment) - [How much of a footprint does the Infection Monkey leave?](#how-much-of-a-footprint-does-the-infection-monkey-leave) - [What's the Infection Monkey Agent's impact on system resources usage?](#whats-the-infection-monkeys-impact-on-system-resources-usage) From 7570064ae7ff8ff9c233221c7e7970d16f534190 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 30 Nov 2021 12:49:19 -0500 Subject: [PATCH 16/25] Docs: Remove erroneous comment about log level for agents Agents' log level is not configurable at this time. --- docs/content/FAQ/_index.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/content/FAQ/_index.md b/docs/content/FAQ/_index.md index bba8d6cda..76fedf3a4 100644 --- a/docs/content/FAQ/_index.md +++ b/docs/content/FAQ/_index.md @@ -200,8 +200,6 @@ The logs contain information about the internals of the Infection Monkey agent's 2019-07-22 19:16:45,013 [77598:140654230214464:DEBUG] connectionpool._make_request.396: https://updates.infectionmonkey.com:443 "GET / HTTP/1.1" 200 61 ``` -Logging level/verbosity can also be changed, see [setup page](../../setup) for you operating system. - ## Running the Infection Monkey in a production environment ### How much of a footprint does the Infection Monkey leave? From 9e036c8853a937cb4011fe249c11e4bb5c99654e Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 30 Nov 2021 12:52:40 -0500 Subject: [PATCH 17/25] Doc: Make minor edits to windows setup documentation --- docs/content/setup/windows.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/content/setup/windows.md b/docs/content/setup/windows.md index cf9dd7e2b..b78712e71 100644 --- a/docs/content/setup/windows.md +++ b/docs/content/setup/windows.md @@ -29,8 +29,10 @@ do, see the [FAQ]({{< ref > ## Configuring the server -You can configure the server by editing [the configuration file](../../reference/server_configuration) located -in installation directory. By default, the path should be `C:\Program Files\Guardicore\Monkey Island\monkey\monkey_island\cc\server_config.json`. +You can configure the server by editing [the configuration +file](../../reference/server_configuration) located in installation directory. +The default path is +`C:\Program Files\Guardicore\Monkey Island\monkey\monkey_island\cc\server_config.json`. ### Start Monkey Island with user-provided certificate From 3e8f7382d0af2c958434647ce4187487ba29baad Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 30 Nov 2021 12:53:55 -0500 Subject: [PATCH 18/25] Docs: Minor edits to docker setup documentation --- docs/content/setup/docker.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/setup/docker.md b/docs/content/setup/docker.md index 7c5645bb9..5de3c9fb5 100644 --- a/docs/content/setup/docker.md +++ b/docs/content/setup/docker.md @@ -101,7 +101,7 @@ enterprise or other security-sensitive environments, it is recommended that the user provide Infection Monkey with a certificate that has been signed by a private certificate authority. -1. Terminate the docker container it's already running. +1. Terminate the docker container if it's already running. 1. Move your `.crt` and `.key` files to `./monkey_island_data` (directory created for the volume). 1. Make sure that your `.crt` and `.key` files are readable only by you. ```bash @@ -132,7 +132,7 @@ private certificate authority. ### Change logging level -1. Stop the docker container it's already running. +1. Stop the docker container if it's already running. 1. Modify the [server configuration file](../../reference/server_configuration) by adding the following lines: ```json { From 9ed689946c3e551d3d440dfc70ea13e156399d5a Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 30 Nov 2021 12:56:52 -0500 Subject: [PATCH 19/25] Docs: Minor edits to Linux setup documentation --- docs/content/setup/linux.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/content/setup/linux.md b/docs/content/setup/linux.md index 2bf318a6b..c39ec75bc 100644 --- a/docs/content/setup/linux.md +++ b/docs/content/setup/linux.md @@ -75,16 +75,16 @@ private certificate authority. ``` 1. Create a [server configuration file and provide the path to the certificate](../../reference/server_configuration). -Server configuration file should look something like: +The server configuration file should look something like: -```json -{ - "ssl_certificate": { - "ssl_certificate_file": "$HOME/.monkey_island/my_cert.crt", - "ssl_certificate_key_file": "$HOME/.monkey_island/my_key.key" + ```json + { + "ssl_certificate": { + "ssl_certificate_file": "$HOME/.monkey_island/my_cert.crt", + "ssl_certificate_key_file": "$HOME/.monkey_island/my_key.key" + } } -} -``` + ``` 1. Start Monkey Island by running the Infection Monkey AppImage package: ```bash @@ -99,13 +99,13 @@ Server configuration file should look something like: 1. Terminate the Island process if it's already running. 1. Create a [server configuration file](../../reference/server_configuration). -Server configuration file should look something like: +The server configuration file should look something like: -```json -{ - "log_level": "INFO" -} -``` + ```json + { + "log_level": "INFO" + } + ``` 1. Start Monkey Island by running the Infection Monkey AppImage package: ```bash From 707fbf41aa4faa6b9e856d8965efccb3b6682f66 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 30 Nov 2021 12:59:20 -0500 Subject: [PATCH 20/25] Island: Remove defunct "environment" from default server_config.json --- monkey/monkey_island/cc/server_config.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/monkey/monkey_island/cc/server_config.json b/monkey/monkey_island/cc/server_config.json index 7ae515179..ff6eb9a7f 100644 --- a/monkey/monkey_island/cc/server_config.json +++ b/monkey/monkey_island/cc/server_config.json @@ -1,8 +1,5 @@ { "log_level": "DEBUG", - "environment": { - "server_config": "password" - }, "mongodb": { "start_mongodb": true } From 854ca7fa8965d02c1b007d49acde20923455fffe Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 30 Nov 2021 13:02:06 -0500 Subject: [PATCH 21/25] Island: Change log level to warn if server config not found --- monkey/monkey_island/cc/setup/config_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/setup/config_setup.py b/monkey/monkey_island/cc/setup/config_setup.py index c362feb82..6835dfc61 100644 --- a/monkey/monkey_island/cc/setup/config_setup.py +++ b/monkey/monkey_island/cc/setup/config_setup.py @@ -30,7 +30,7 @@ def _update_config_from_file(config: IslandConfigOptions, config_path: Path): config.update(config_from_file) logger.info(f"Server config updated from {config_path}") except OSError: - logger.info(f"Server config not found in path {config_path}") + logger.warn(f"Server config not found in path {config_path}") def _load_server_config_from_file(server_config_path) -> dict: From 8304a4ea19fd3e9b33bb57f257ad1662f7238538 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 1 Dec 2021 10:54:05 +0200 Subject: [PATCH 22/25] UT: improve unit tests in test_server_setup.py --- .../cc/setup/test_server_setup.py | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py index 226842607..a43233026 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py +++ b/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py @@ -5,23 +5,21 @@ import pytest import monkey_island.cc.setup.config_setup # noqa: F401 from monkey_island.cc.arg_parser import IslandCmdArgs -from monkey_island.cc.server_setup import _extract_config +from monkey_island.cc.server_setup import _extract_config, get_server_config from monkey_island.cc.server_utils.file_utils import is_windows_os from monkey_island.cc.setup.island_config_options import IslandConfigOptions -BAD_JSON = '{"data_dir": "C:\\test\\test"' - @pytest.fixture -def cmd_server_config_path(tmpdir) -> Path: +def cmd_server_config_path(tmp_path) -> Path: # Represents the config that user can provide via cmd arguments - return tmpdir / "fake_server_config.json" + return tmp_path / "fake_server_config.json" @pytest.fixture -def deployment_server_config_path(tmpdir) -> Path: +def deployment_server_config_path(tmp_path) -> Path: # Represents the config that is built in, deployment specific - return tmpdir / "fake_server_config3.json" + return tmp_path / "fake_server_config3.json" def create_server_config(config_contents: str, server_config_path: Path): @@ -37,54 +35,61 @@ def mock_deployment_config_path(monkeypatch, deployment_server_config_path): ) -def test_extract_config_defaults(): +@pytest.fixture +def empty_cmd_args(): + return IslandCmdArgs(setup_only=False, server_config_path=None) + + +@pytest.fixture +def cmd_args_with_server_config(cmd_server_config_path): + return IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path) + + +def test_extract_config_defaults(empty_cmd_args): expected = IslandConfigOptions({}) - assert ( - expected.__dict__ - == _extract_config(IslandCmdArgs(setup_only=False, server_config_path=None)).__dict__ - ) + assert expected.__dict__ == get_server_config(empty_cmd_args).__dict__ -def test_deployment_config_overrides_defaults(deployment_server_config_path): +def test_deployment_config_overrides_defaults(deployment_server_config_path, empty_cmd_args): expected = IslandConfigOptions({"log_level": "/log_level_2"}) create_server_config(dumps({"log_level": "/log_level_2"}), deployment_server_config_path) - assert ( - expected.__dict__ - == _extract_config(IslandCmdArgs(setup_only=False, server_config_path=None)).__dict__ - ) + assert expected.__dict__ == get_server_config(empty_cmd_args).__dict__ -def test_cmd_config_overrides_everything(deployment_server_config_path, cmd_server_config_path): +def test_cmd_config_overrides_everything( + deployment_server_config_path, cmd_server_config_path, cmd_args_with_server_config +): expected = IslandConfigOptions({"log_level": "/log_level_3"}) create_server_config(dumps({"log_level": "/log_level_2"}), deployment_server_config_path) create_server_config(dumps({"log_level": "/log_level_3"}), cmd_server_config_path) - extracted_config = _extract_config( - IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path) - ) + extracted_config = get_server_config(cmd_args_with_server_config) assert expected.__dict__ == extracted_config.__dict__ -def test_not_overriding_unspecified_values(deployment_server_config_path, cmd_server_config_path): +def test_not_overriding_unspecified_values( + deployment_server_config_path, cmd_server_config_path, cmd_args_with_server_config +): expected = IslandConfigOptions({"log_level": "/log_level_2", "data_dir": "/data_dir1"}) create_server_config(dumps({"data_dir": "/data_dir1"}), deployment_server_config_path) create_server_config(dumps({"log_level": "/log_level_2"}), cmd_server_config_path) - extracted_config = _extract_config( - IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path) - ) + extracted_config = get_server_config(cmd_args_with_server_config) assert expected.__dict__ == extracted_config.__dict__ -def test_paths_get_expanded(deployment_server_config_path): +def test_paths_get_expanded(deployment_server_config_path, empty_cmd_args): if is_windows_os(): path = "%temp%/path" else: path = "$HOME/path" create_server_config(dumps({"data_dir": path}), deployment_server_config_path) - extracted_config = _extract_config(IslandCmdArgs(setup_only=False, server_config_path=None)) + extracted_config = get_server_config(empty_cmd_args) assert not extracted_config.data_dir == path -def test_malformed_json(cmd_server_config_path): +BAD_JSON = '{"data_dir": "C:\\test\\test"' + + +def test_malformed_json(cmd_server_config_path, cmd_args_with_server_config): create_server_config(BAD_JSON, cmd_server_config_path) with pytest.raises(SystemExit): - _extract_config(IslandCmdArgs(setup_only=False, server_config_path=cmd_server_config_path)) + _extract_config(cmd_args_with_server_config) From 6e7ddbc6c7909023bcc17044ffb5d94002f1fa1a Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 1 Dec 2021 14:35:37 +0200 Subject: [PATCH 23/25] Agent: improve the readability of island_config_options.py --- .../cc/setup/island_config_options.py | 38 +++++++------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/monkey/monkey_island/cc/setup/island_config_options.py b/monkey/monkey_island/cc/setup/island_config_options.py index 384ed3079..27df897e8 100644 --- a/monkey/monkey_island/cc/setup/island_config_options.py +++ b/monkey/monkey_island/cc/setup/island_config_options.py @@ -1,7 +1,5 @@ from __future__ import annotations -from dpath import util - from common.utils.file_utils import expand_path from monkey_island.cc.server_utils.consts import ( DEFAULT_CERTIFICATE_PATHS, @@ -25,7 +23,7 @@ class IslandConfigOptions: def __init__(self, config_contents: dict = None): if not config_contents: config_contents = {} - self.data_dir = expand_path(config_contents.get(_DATA_DIR, DEFAULT_DATA_DIR)) + self.data_dir = config_contents.get(_DATA_DIR, DEFAULT_DATA_DIR) self.log_level = config_contents.get(_LOG_LEVEL, DEFAULT_LOG_LEVEL) @@ -33,30 +31,20 @@ class IslandConfigOptions: _MONGODB, {_START_MONGODB: DEFAULT_START_MONGO_DB} ).get(_START_MONGODB, DEFAULT_START_MONGO_DB) - self.crt_path = expand_path( - config_contents.get(_SSL_CERT, DEFAULT_CERTIFICATE_PATHS).get( - _SSL_CERT_FILE, DEFAULT_CRT_PATH - ) + self.crt_path = config_contents.get(_SSL_CERT, DEFAULT_CERTIFICATE_PATHS).get( + _SSL_CERT_FILE, DEFAULT_CRT_PATH ) - self.key_path = expand_path( - config_contents.get(_SSL_CERT, DEFAULT_CERTIFICATE_PATHS).get( - _SSL_CERT_KEY, DEFAULT_KEY_PATH - ) + self.key_path = config_contents.get(_SSL_CERT, DEFAULT_CERTIFICATE_PATHS).get( + _SSL_CERT_KEY, DEFAULT_KEY_PATH ) + self._expand_paths() + + def _expand_paths(self): + self.data_dir = expand_path(str(self.data_dir)) + self.crt_path = expand_path(str(self.crt_path)) + self.key_path = expand_path(str(self.key_path)) + def update(self, target: dict): - target = self._expand_config_paths(target) self.__dict__.update(target) - - @staticmethod - def _expand_config_paths(config: dict) -> dict: - config_paths = [_DATA_DIR, f"{_SSL_CERT}.{_SSL_CERT_FILE}", f"{_SSL_CERT}.{_SSL_CERT_KEY}"] - - for config_path in config_paths: - try: - expanded_val = expand_path(util.get(config, config_path, ".")) - util.set(config, config_path, expanded_val, ".") - except KeyError: - pass - - return config + self._expand_paths() From e61bac0895b50c0e1b131d2c211becb1bfc03327 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 1 Dec 2021 07:38:37 -0500 Subject: [PATCH 24/25] Tests: Test get_server_config() instead of internal _extract_config() --- .../setup/{test_server_setup.py => test_config_setup.py} | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) rename monkey/tests/unit_tests/monkey_island/cc/setup/{test_server_setup.py => test_config_setup.py} (95%) diff --git a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py b/monkey/tests/unit_tests/monkey_island/cc/setup/test_config_setup.py similarity index 95% rename from monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py rename to monkey/tests/unit_tests/monkey_island/cc/setup/test_config_setup.py index a43233026..88e1850a7 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/setup/test_server_setup.py +++ b/monkey/tests/unit_tests/monkey_island/cc/setup/test_config_setup.py @@ -1,3 +1,4 @@ +import json from json import dumps from pathlib import Path @@ -5,8 +6,8 @@ import pytest import monkey_island.cc.setup.config_setup # noqa: F401 from monkey_island.cc.arg_parser import IslandCmdArgs -from monkey_island.cc.server_setup import _extract_config, get_server_config from monkey_island.cc.server_utils.file_utils import is_windows_os +from monkey_island.cc.setup.config_setup import get_server_config from monkey_island.cc.setup.island_config_options import IslandConfigOptions @@ -91,5 +92,5 @@ BAD_JSON = '{"data_dir": "C:\\test\\test"' def test_malformed_json(cmd_server_config_path, cmd_args_with_server_config): create_server_config(BAD_JSON, cmd_server_config_path) - with pytest.raises(SystemExit): - _extract_config(cmd_args_with_server_config) + with pytest.raises(json.JSONDecodeError): + get_server_config(cmd_args_with_server_config) From 1d7c80bfec8b36a07c9f428d76ccb1cb93c33ecc Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 1 Dec 2021 07:40:30 -0500 Subject: [PATCH 25/25] Changelog: Add changelog entry for #1576 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cfb18eae..61c2a177e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - "Communicate as Backdoor User" PBA's HTTP requests to request headers only and include a timeout. #1577 +- The setup procedure for custom server_config.json files to be simpler. #1576 ### Removed - The VSFTPD exploiter. #1533