Merge pull request #1181 from guardicore/setup_param_refactoring

Setup param refactoring

appimage/server_config.json.standard

@@ -4,5 +4,8 @@
   "environment": {
     "server_config": "password",
     "deployment": "standard"
+  },
+  "mongodb": {
+        "start_mongodb": true
   }
 }

monkey/monkey_island.py

@@ -1,26 +1,22 @@
 from gevent import monkey as gevent_monkey
 
 from monkey_island.cc.arg_parser import parse_cli_args
+from monkey_island.config_file_parser import load_island_config_from_file
 
 gevent_monkey.patch_all()
 
 import json  # noqa: E402
-import os  # noqa: E402
 
-from monkey_island import config_loader  # noqa: E402
 from monkey_island.cc.server_utils.island_logger import setup_logging  # noqa: E402
 
 if "__main__" == __name__:
     island_args = parse_cli_args()
 
-    # This is here in order to catch EVERYTHING, some functions are being called on
-    # imports, so the log init needs to be first.
     try:
-        server_config_path = os.path.expanduser(island_args.server_config)
-
-        config = config_loader.load_server_config_from_file(server_config_path)
-
-        setup_logging(config["data_dir"], config["log_level"])
+        # This is here in order to catch EVERYTHING, some functions are being called on
+        # imports, so the log init needs to be first.
+        config_options = load_island_config_from_file(island_args.server_config_path)
+        setup_logging(config_options.data_dir, config_options.log_level)
 
     except OSError as ex:
         print(f"Error opening server config file: {ex}")
@@ -32,4 +28,4 @@ if "__main__" == __name__:
 
     from monkey_island.cc.main import main  # noqa: E402
 
-    main(config["data_dir"], island_args.setup_only, island_args.server_config)
+    main(island_args.setup_only, island_args.server_config_path, config_options)

monkey/monkey_island/cc/arg_parser.py

@@ -1,15 +1,21 @@
-from dataclasses import dataclass
-
-from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH
+from monkey_island.cc.server_utils.consts import (
+    DEFAULT_SERVER_CONFIG_PATH,
+    DEFAULT_SHOULD_SETUP_ONLY,
+)
 
 
-@dataclass
-class IslandArgs:
-    setup_only: bool
-    server_config: str
+class IslandCmdArgs:
+    setup_only: bool = DEFAULT_SHOULD_SETUP_ONLY
+    server_config_path: str = DEFAULT_SERVER_CONFIG_PATH
+
+    def __init__(self, setup_only: None, server_config_path: None):
+        if setup_only:
+            self.setup_only = setup_only
+        if server_config_path:
+            self.server_config_path = server_config_path
 
 
-def parse_cli_args() -> IslandArgs:
+def parse_cli_args() -> IslandCmdArgs:
     import argparse
 
     parser = argparse.ArgumentParser(
@@ -25,11 +31,8 @@ def parse_cli_args() -> IslandArgs:
         "compiling/packaging Islands.",
     )
     parser.add_argument(
-        "--server-config",
-        action="store",
-        help="The path to the server configuration file.",
-        default=DEFAULT_SERVER_CONFIG_PATH,
+        "--server-config", action="store", help="The path to the server configuration file."
     )
     args = parser.parse_args()
 
-    return IslandArgs(args.setup_only, args.server_config)
+    return IslandCmdArgs(args.setup_only, args.server_config)

monkey/monkey_island/cc/main.py

@@ -9,6 +9,8 @@ from threading import Thread
 # "monkey_island." work.
 from gevent.pywsgi import WSGIServer
 
+from monkey_island.setup.island_config_options import IslandConfigOptions
+
 MONKEY_ISLAND_DIR_BASE_PATH = str(Path(__file__).parent.parent)
 if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path:
     sys.path.insert(0, MONKEY_ISLAND_DIR_BASE_PATH)
@@ -22,28 +24,22 @@ from common.version import get_version  # noqa: E402
 from monkey_island.cc.app import init_app  # noqa: E402
 from monkey_island.cc.database import get_db_version  # noqa: E402
 from monkey_island.cc.database import is_db_server_up  # noqa: E402
+from monkey_island.cc.mongo_setup import init_collections, launch_mongodb  # noqa: E402
 from monkey_island.cc.resources.monkey_download import MonkeyDownload  # noqa: E402
 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer  # noqa: E402
-from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH  # noqa: E402
 from monkey_island.cc.server_utils.encryptor import initialize_encryptor  # noqa: E402
 from monkey_island.cc.services.initialize import initialize_services  # noqa: E402
 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list  # noqa: E402
 from monkey_island.cc.services.utils.network_utils import local_ip_addresses  # noqa: E402
-from monkey_island.cc.setup import setup  # noqa: E402
 
 MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0"
 
 
-def main(
-    data_dir,
-    should_setup_only=False,
-    server_config_filename=DEFAULT_SERVER_CONFIG_PATH,
-):
-    logger.info("Starting bootloader server")
+def main(setup_only: bool, server_config_path: str, config_options: IslandConfigOptions):
 
-    env_singleton.initialize_from_file(server_config_filename)
-    initialize_encryptor(data_dir)
-    initialize_services(data_dir)
+    env_singleton.initialize_from_file(server_config_path)
+    initialize_encryptor(config_options.data_dir)
+    initialize_services(config_options.data_dir)
 
     mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url())
     bootloader_server_thread = Thread(
@@ -51,11 +47,13 @@ def main(
     )
 
     bootloader_server_thread.start()
-    start_island_server(should_setup_only)
+    start_island_server(setup_only, config_options)
     bootloader_server_thread.join()
 
 
-def start_island_server(should_setup_only):
+def start_island_server(should_setup_only, config_options: IslandConfigOptions):
+    if config_options.start_mongodb:
+        launch_mongodb()
     mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url())
     wait_for_mongo_db_server(mongo_url)
     assert_mongo_db_version(mongo_url)
@@ -66,7 +64,7 @@ def start_island_server(should_setup_only):
     crt_path = str(Path(MONKEY_ISLAND_ABS_PATH, "cc", "server.crt"))
     key_path = str(Path(MONKEY_ISLAND_ABS_PATH, "cc", "server.key"))
 
-    setup()
+    init_collections()
 
     if should_setup_only:
         logger.warning("Setup only flag passed. Exiting.")

monkey/monkey_island/cc/mongo_setup.py

@@ -9,7 +9,12 @@ from monkey_island.cc.services.attack.mitre_api_interface import MitreApiInterfa
 logger = logging.getLogger(__name__)
 
 
-def setup():
+def launch_mongodb():
+    # TODO: Implement the launch of mongodb process
+    pass
+
+
+def init_collections():
     logger.info("Setting up the Monkey Island, this might take a while...")
     try_store_mitigations_on_mongo()
 

monkey/monkey_island/cc/server_config.json.develop

@@ -3,5 +3,8 @@
   "environment": {
     "server_config": "password",
     "deployment": "develop"
+  },
+  "mongodb": {
+        "start_mongodb": true
   }
 }

monkey/monkey_island/cc/server_utils/consts.py

@@ -5,6 +5,7 @@ __author__ = "itay.mizeretz"
 MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island")
 DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5
 
+# TODO move setup consts
 DEFAULT_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json")
 
 DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join(
@@ -12,3 +13,6 @@ DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join(
 )
 
 DEFAULT_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc")
+DEFAULT_LOG_LEVEL = "INFO"
+DEFAULT_START_MONGO_DB = True
+DEFAULT_SHOULD_SETUP_ONLY = False

monkey/monkey_island/config_file_parser.py

@@ -0,0 +1,19 @@
+import json
+from os.path import isfile
+
+from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH
+from monkey_island.setup.island_config_options import IslandConfigOptions
+
+
+def load_island_config_from_file(server_config_path: str) -> IslandConfigOptions:
+    config_contents = read_config_file(server_config_path)
+    return IslandConfigOptions(config_contents)
+
+
+def read_config_file(server_config_path: str) -> dict:
+    if not server_config_path or not isfile(server_config_path):
+        server_config_path = DEFAULT_SERVER_CONFIG_PATH
+    with open(server_config_path, "r") as f:
+        config_content = f.read()
+        config = json.loads(config_content)
+        return config

monkey/monkey_island/config_loader.py

@@ -1,25 +0,0 @@
-import json
-import os
-
-from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR
-
-DEFAULT_LOG_LEVEL = "INFO"
-
-
-def load_server_config_from_file(server_config_path):
-    with open(server_config_path, "r") as f:
-        config_content = f.read()
-        config = json.loads(config_content)
-        add_default_values_to_config(config)
-
-        return config
-
-
-def add_default_values_to_config(config):
-    config["data_dir"] = os.path.abspath(
-        os.path.expanduser(os.path.expandvars(config.get("data_dir", DEFAULT_DATA_DIR)))
-    )
-
-    config.setdefault("log_level", DEFAULT_LOG_LEVEL)
-
-    return config

monkey/monkey_island/setup/island_config_options.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from monkey_island.cc.server_utils.consts import (
+    DEFAULT_DATA_DIR,
+    DEFAULT_LOG_LEVEL,
+    DEFAULT_START_MONGO_DB,
+)
+
+
+class IslandConfigOptions:
+    def __init__(self, config_contents: dict):
+        self.data_dir = config_contents.get("data_dir", DEFAULT_DATA_DIR)
+
+        self.log_level = config_contents.get("log_level", DEFAULT_LOG_LEVEL)
+
+        self.start_mongodb = config_contents.get(
+            "mongodb", {"start_mongodb": DEFAULT_START_MONGO_DB}
+        ).get("start_mongodb", DEFAULT_START_MONGO_DB)

monkey/tests/data_for_tests/server_configs/server_config_empty.json

@@ -0,0 +1,2 @@
+{
+}

monkey/tests/unit_tests/monkey_island/conftest.py

@@ -9,5 +9,10 @@ def server_configs_dir(data_for_tests_dir):
 
 
 @pytest.fixture(scope="module")
-def test_server_config(server_configs_dir):
-    return os.path.join(server_configs_dir, "test_server_config.json")
+def server_config_init_only(server_configs_dir):
+    return os.path.join(server_configs_dir, "server_config_init_only.json")
+
+
+@pytest.fixture(scope="module")
+def server_config_empty(server_configs_dir):
+    return os.path.join(server_configs_dir, "server_config_empty.json")

monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py

@@ -0,0 +1,39 @@
+from monkey_island.cc.server_utils.consts import (
+    DEFAULT_DATA_DIR,
+    DEFAULT_LOG_LEVEL,
+    DEFAULT_START_MONGO_DB,
+)
+from monkey_island.setup.island_config_options import IslandConfigOptions
+
+TEST_CONFIG_FILE_CONTENTS_SPECIFIED = {
+    "data_dir": "/tmp",
+    "log_level": "test",
+    "mongodb": {"start_mongodb": False},
+}
+
+TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED = {}
+
+TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO = {"mongodb": {}}
+
+
+def test_island_config_options__data_dir():
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED)
+    assert options.data_dir == "/tmp"
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED)
+    assert options.data_dir == DEFAULT_DATA_DIR
+
+
+def test_island_config_options__log_level():
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED)
+    assert options.log_level == "test"
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED)
+    assert options.log_level == DEFAULT_LOG_LEVEL
+
+
+def test_island_config_options__mongodb():
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED)
+    assert not options.start_mongodb
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED)
+    assert options.start_mongodb == DEFAULT_START_MONGO_DB
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO)
+    assert options.start_mongodb == DEFAULT_START_MONGO_DB

monkey/tests/unit_tests/monkey_island/test_config_file_parser.py

@@ -0,0 +1,16 @@
+from monkey_island import config_file_parser
+from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_LOG_LEVEL
+
+
+def test_load_server_config_from_file(server_config_init_only):
+    config = config_file_parser.load_island_config_from_file(server_config_init_only)
+
+    assert config.data_dir == "~/.monkey_island"
+    assert config.log_level == "NOTICE"
+
+
+def test_load_server_config_from_file_empty_file(monkeypatch, server_config_empty):
+    config = config_file_parser.load_island_config_from_file(server_config_empty)
+
+    assert config.data_dir == DEFAULT_DATA_DIR
+    assert config.log_level == DEFAULT_LOG_LEVEL

monkey/tests/unit_tests/monkey_island/test_config_loader.py

@@ -1,27 +0,0 @@
-import os
-
-from monkey_island import config_loader
-from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR
-
-
-def test_load_server_config_from_file(test_server_config, mock_home_env):
-    config = config_loader.load_server_config_from_file(test_server_config)
-
-    assert config["data_dir"] == os.path.join(mock_home_env, ".monkey_island")
-    assert config["log_level"] == "NOTICE"
-
-
-def test_default_log_level():
-    test_config = {}
-    config = config_loader.add_default_values_to_config(test_config)
-
-    assert "log_level" in config
-    assert config["log_level"] == "INFO"
-
-
-def test_default_data_dir(mock_home_env):
-    test_config = {}
-    config = config_loader.add_default_values_to_config(test_config)
-
-    assert "data_dir" in config
-    assert config["data_dir"] == DEFAULT_DATA_DIR

whitelist.py

@@ -165,7 +165,7 @@ ALIBABA  # unused variable (monkey/common/cloud/environment_names.py:10)
 IBM  # unused variable (monkey/common/cloud/environment_names.py:11)
 DigitalOcean  # unused variable (monkey/common/cloud/environment_names.py:12)
 _.aws_info  # unused attribute (monkey/monkey_island/cc/environment/aws.py:13)
-
+build_from_config_file_contents  # unused method 'build_from_config_file_contents' (\monkey_island\setup\island_config_options.py:18)
 
 # these are not needed for it to work, but may be useful extra information to understand what's going on
 WINDOWS_PBA_TYPE  # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23)