Merged develop into data-dir-on-island-init

CHANGELOG.md

@@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Authentication mechanism to use bcrypt on server side. #1139
 - `server_config.json` puts environment config options in a separate section
   named "environment". #1161
+- BlackBox tests can now register if they are ran on a fresh installation. #1180
 - Improved the structure of unit tests by scoping fixtures only to relevant modules
   instead of having a one huge fixture file, improved and renamed the directory
   structure of unit tests and unit test infrastructure. #1178

appimage/server_config.json.standard

@@ -4,5 +4,8 @@
   "environment": {
     "server_config": "password",
     "deployment": "standard"
+  },
+  "mongodb": {
+        "start_mongodb": true
   }
 }

envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py

@@ -12,6 +12,10 @@ NO_AUTH_CREDS = "1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()"
 LOGGER = logging.getLogger(__name__)
 
 
+class AuthenticationFailedError(Exception):
+    pass
+
+
 # noinspection PyArgumentList
 class MonkeyIslandRequests(object):
     def __init__(self, server_address):
@@ -43,6 +47,9 @@ class MonkeyIslandRequests(object):
     def try_get_jwt_from_server(self):
         try:
             return self.get_jwt_from_server()
+        except AuthenticationFailedError:
+            self.try_set_island_to_no_password()
+            return self.get_jwt_from_server()
         except requests.ConnectionError as err:
             LOGGER.error(
                 "Unable to connect to island, aborting! Error information: {}. Server: {}".format(
@@ -51,6 +58,21 @@ class MonkeyIslandRequests(object):
             )
             assert False
 
+    def get_jwt_from_server(self):
+        resp = requests.post(  # noqa: DUO123
+            self.addr + "api/auth",
+            json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS},
+            verify=False,
+        )
+        if resp.status_code == 401:
+            raise AuthenticationFailedError
+        return resp.json()["access_token"]
+
+    def try_set_island_to_no_password(self):
+        requests.patch(  # noqa: DUO123
+            self.addr + "api/environment", json={"server_config": "standard"}, verify=False
+        )
+
     class _Decorators:
         @classmethod
         def refresh_jwt_token(cls, request_function):
@@ -62,14 +84,6 @@ class MonkeyIslandRequests(object):
 
             return request_function_wrapper
 
-    def get_jwt_from_server(self):
-        resp = requests.post(  # noqa: DUO123
-            self.addr + "api/auth",
-            json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS},
-            verify=False,
-        )
-        return resp.json()["access_token"]
-
     @_Decorators.refresh_jwt_token
     def get(self, url, data=None):
         return requests.get(  # noqa: DUO123

envs/monkey_zoo/blackbox/test_blackbox.py

@@ -98,7 +98,15 @@ def wait_machine_bootup():
 
 @pytest.fixture(scope="class")
 def island_client(island, quick_performance_tests):
+    client_established = False
+    try:
         island_client_object = MonkeyIslandClient(island)
+        client_established = island_client_object.get_api_status()
+    except Exception:
+        logging.exception("Got an exception while trying to establish connection to the Island.")
+    finally:
+        if not client_established:
+            pytest.exit("BB tests couldn't establish communication to the island.")
     if not quick_performance_tests:
         island_client_object.reset_env()
     yield island_client_object
@@ -158,10 +166,6 @@ class TestMonkeyBlackbox:
     def get_log_dir_path():
         return os.path.abspath(LOG_DIR_PATH)
 
-    def test_server_online(self, island_client):
-        if not island_client.get_api_status():
-            pytest.exit("BB tests couldn't reach the Island server, quiting.")
-
     def test_ssh_exploiter(self, island_client):
         TestMonkeyBlackbox.run_exploitation_test(island_client, Ssh, "SSH_exploiter_and_keys")
 

monkey/monkey_island.py

@@ -20,7 +20,7 @@ if "__main__" == __name__:
         else:
             config, server_config_path = setup_default_config()
 
-        setup_logging(config["data_dir"], config["log_level"])
+        setup_logging(config.data_dir, config.log_level)
 
     except OSError as ex:
         print(f"Error opening server config file: {ex}")
@@ -32,4 +32,4 @@ if "__main__" == __name__:
 
     from monkey_island.cc.main import main  # noqa: E402
 
-    main(config["data_dir"], island_args.setup_only, server_config_path)
+    main(island_args.setup_only, island_args.server_config_path, config)

monkey/monkey_island/cc/arg_parser.py

@@ -1,13 +1,21 @@
-from dataclasses import dataclass
+from monkey_island.cc.server_utils.consts import (
+    DEFAULT_SERVER_CONFIG_PATH,
+    DEFAULT_SHOULD_SETUP_ONLY,
+)
 
 
-@dataclass
+class IslandCmdArgs:
-class IslandArgs:
+    setup_only: bool = DEFAULT_SHOULD_SETUP_ONLY
-    setup_only: bool
+    server_config_path: str = DEFAULT_SERVER_CONFIG_PATH
-    server_config: str
+
+    def __init__(self, setup_only: None, server_config_path: None):
+        if setup_only:
+            self.setup_only = setup_only
+        if server_config_path:
+            self.server_config_path = server_config_path
 
 
-def parse_cli_args() -> IslandArgs:
+def parse_cli_args() -> IslandCmdArgs:
     import argparse
 
     parser = argparse.ArgumentParser(
@@ -27,4 +35,4 @@ def parse_cli_args() -> IslandArgs:
     )
     args = parser.parse_args()
 
-    return IslandArgs(args.setup_only, args.server_config)
+    return IslandCmdArgs(args.setup_only, args.server_config)

monkey/monkey_island/cc/environment/server_config_handler.py

@@ -3,11 +3,10 @@ import os
 from pathlib import Path
 
 from monkey_island.cc.server_utils.consts import (
-    DEFAULT_DATA_DIR,
     DEFAULT_DEVELOP_SERVER_CONFIG_PATH,
-    DEFAULT_LOG_LEVEL,
     DEFAULT_SERVER_CONFIG_PATH,
 )
+from monkey_island.setup.island_config_options import IslandConfigOptions
 
 
 def create_default_server_config_file() -> None:
@@ -20,20 +19,9 @@ def write_default_server_config_to_file(path: str) -> None:
     Path(path).write_text(default_config)
 
 
-def load_server_config_from_file(server_config_path):
+def load_server_config_from_file(server_config_path) -> IslandConfigOptions:
     with open(server_config_path, "r") as f:
         config_content = f.read()
         config = json.loads(config_content)
-        add_default_values_to_config(config)
 
-        return config
+        return IslandConfigOptions(config)
-
-
-def add_default_values_to_config(config):
-    config["data_dir"] = os.path.abspath(
-        os.path.expanduser(os.path.expandvars(config.get("data_dir", DEFAULT_DATA_DIR)))
-    )
-
-    config.setdefault("log_level", DEFAULT_LOG_LEVEL)
-
-    return config

monkey/monkey_island/cc/main.py

@@ -9,6 +9,8 @@ from threading import Thread
 # "monkey_island." work.
 from gevent.pywsgi import WSGIServer
 
+from monkey_island.setup.island_config_options import IslandConfigOptions
+
 MONKEY_ISLAND_DIR_BASE_PATH = str(Path(__file__).parent.parent)
 if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path:
     sys.path.insert(0, MONKEY_ISLAND_DIR_BASE_PATH)
@@ -22,28 +24,22 @@ from common.version import get_version  # noqa: E402
 from monkey_island.cc.app import init_app  # noqa: E402
 from monkey_island.cc.database import get_db_version  # noqa: E402
 from monkey_island.cc.database import is_db_server_up  # noqa: E402
+from monkey_island.cc.mongo_setup import init_collections, launch_mongodb  # noqa: E402
 from monkey_island.cc.resources.monkey_download import MonkeyDownload  # noqa: E402
 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer  # noqa: E402
-from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH  # noqa: E402
 from monkey_island.cc.server_utils.encryptor import initialize_encryptor  # noqa: E402
 from monkey_island.cc.services.initialize import initialize_services  # noqa: E402
 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list  # noqa: E402
 from monkey_island.cc.services.utils.network_utils import local_ip_addresses  # noqa: E402
-from monkey_island.cc.setup import setup  # noqa: E402
 
 MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0"
 
 
-def main(
+def main(setup_only: bool, server_config_path: str, config_options: IslandConfigOptions):
-    data_dir,
-    should_setup_only=False,
-    server_config_filename=DEFAULT_SERVER_CONFIG_PATH,
-):
-    logger.info("Starting bootloader server")
 
-    env_singleton.initialize_from_file(server_config_filename)
+    env_singleton.initialize_from_file(server_config_path)
-    initialize_encryptor(data_dir)
+    initialize_encryptor(config_options.data_dir)
-    initialize_services(data_dir)
+    initialize_services(config_options.data_dir)
 
     mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url())
     bootloader_server_thread = Thread(
@@ -51,11 +47,13 @@ def main(
     )
 
     bootloader_server_thread.start()
-    start_island_server(should_setup_only)
+    start_island_server(setup_only, config_options)
     bootloader_server_thread.join()
 
 
-def start_island_server(should_setup_only):
+def start_island_server(should_setup_only, config_options: IslandConfigOptions):
+    if config_options.start_mongodb:
+        launch_mongodb()
     mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url())
     wait_for_mongo_db_server(mongo_url)
     assert_mongo_db_version(mongo_url)
@@ -66,7 +64,7 @@ def start_island_server(should_setup_only):
     crt_path = str(Path(MONKEY_ISLAND_ABS_PATH, "cc", "server.crt"))
     key_path = str(Path(MONKEY_ISLAND_ABS_PATH, "cc", "server.key"))
 
-    setup()
+    init_collections()
 
     if should_setup_only:
         logger.warning("Setup only flag passed. Exiting.")

monkey/monkey_island/cc/mongo_setup.py

@@ -9,7 +9,12 @@ from monkey_island.cc.services.attack.mitre_api_interface import MitreApiInterfa
 logger = logging.getLogger(__name__)
 
 
-def setup():
+def launch_mongodb():
+    # TODO: Implement the launch of mongodb process
+    pass
+
+
+def init_collections():
     logger.info("Setting up the Monkey Island, this might take a while...")
     try_store_mitigations_on_mongo()
 

monkey/monkey_island/cc/server_config.json.develop

@@ -3,5 +3,8 @@
   "environment": {
     "server_config": "password",
     "deployment": "develop"
+  },
+  "mongodb": {
+        "start_mongodb": true
   }
 }

monkey/monkey_island/cc/server_utils/consts.py

@@ -14,8 +14,6 @@ def get_default_data_dir() -> str:
 
 SERVER_CONFIG_FILENAME = "server_config.json"
 
-DEFAULT_LOG_LEVEL = "INFO"
-
 MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island")
 
 DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir())
@@ -29,3 +27,7 @@ DEFAULT_SERVER_CONFIG_PATH = os.path.expandvars(
 DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join(
     MONKEY_ISLAND_ABS_PATH, "cc", f"{SERVER_CONFIG_FILENAME}.develop"
 )
+
+DEFAULT_LOG_LEVEL = "INFO"
+DEFAULT_START_MONGO_DB = True
+DEFAULT_SHOULD_SETUP_ONLY = False

monkey/monkey_island/config_file_parser.py

@@ -0,0 +1,19 @@
+import json
+from os.path import isfile
+
+from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH
+from monkey_island.setup.island_config_options import IslandConfigOptions
+
+
+def load_island_config_from_file(server_config_path: str) -> IslandConfigOptions:
+    config_contents = read_config_file(server_config_path)
+    return IslandConfigOptions(config_contents)
+
+
+def read_config_file(server_config_path: str) -> dict:
+    if not server_config_path or not isfile(server_config_path):
+        server_config_path = DEFAULT_SERVER_CONFIG_PATH
+    with open(server_config_path, "r") as f:
+        config_content = f.read()
+        config = json.loads(config_content)
+        return config

monkey/monkey_island/setup/config_setup.py

@@ -4,16 +4,18 @@ from typing import Tuple
 from monkey_island.cc.environment import server_config_handler
 from monkey_island.cc.environment.data_dir_generator import create_data_dir  # noqa: E402
 from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH
+from monkey_island.setup.island_config_options import IslandConfigOptions
 
 
-def setup_config_by_cmd_arg(server_config_path) -> Tuple[dict, str]:
+def setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, str]:
     server_config_path = os.path.expandvars(os.path.expanduser(server_config_path))
     config = server_config_handler.load_server_config_from_file(server_config_path)
+
     create_data_dir(config["data_dir"], create_parent_dirs=True)
     return config, server_config_path
 
 
-def setup_default_config() -> Tuple[dict, str]:
+def setup_default_config() -> Tuple[IslandConfigOptions, str]:
     server_config_path = DEFAULT_SERVER_CONFIG_PATH
     create_data_dir(DEFAULT_DATA_DIR, create_parent_dirs=False)
     server_config_handler.create_default_server_config_file()

monkey/monkey_island/setup/island_config_options.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from monkey_island.cc.server_utils.consts import (
+    DEFAULT_DATA_DIR,
+    DEFAULT_LOG_LEVEL,
+    DEFAULT_START_MONGO_DB,
+)
+
+
+class IslandConfigOptions:
+    def __init__(self, config_contents: dict):
+        self.data_dir = config_contents.get("data_dir", DEFAULT_DATA_DIR)
+
+        self.log_level = config_contents.get("log_level", DEFAULT_LOG_LEVEL)
+
+        self.start_mongodb = config_contents.get(
+            "mongodb", {"start_mongodb": DEFAULT_START_MONGO_DB}
+        ).get("start_mongodb", DEFAULT_START_MONGO_DB)

monkey/tests/data_for_tests/server_configs/server_config_empty.json

@@ -0,0 +1,2 @@
+{
+}

monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py

@@ -1,27 +0,0 @@
-import os
-
-from monkey_island.cc.environment import server_config_handler
-from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR
-
-
-def test_load_server_config_from_file(test_server_config, mock_home_env):
-    config = server_config_handler.load_server_config_from_file(test_server_config)
-
-    assert config["data_dir"] == os.path.join(mock_home_env, ".monkey_island")
-    assert config["log_level"] == "NOTICE"
-
-
-def test_default_log_level():
-    test_config = {}
-    config = server_config_handler.add_default_values_to_config(test_config)
-
-    assert "log_level" in config
-    assert config["log_level"] == "INFO"
-
-
-def test_default_data_dir(mock_home_env):
-    test_config = {}
-    config = server_config_handler.add_default_values_to_config(test_config)
-
-    assert "data_dir" in config
-    assert config["data_dir"] == DEFAULT_DATA_DIR

monkey/tests/unit_tests/monkey_island/conftest.py

@@ -9,5 +9,10 @@ def server_configs_dir(data_for_tests_dir):
 
 
 @pytest.fixture(scope="module")
-def test_server_config(server_configs_dir):
+def server_config_init_only(server_configs_dir):
-    return os.path.join(server_configs_dir, "test_server_config.json")
+    return os.path.join(server_configs_dir, "server_config_init_only.json")
+
+
+@pytest.fixture(scope="module")
+def server_config_empty(server_configs_dir):
+    return os.path.join(server_configs_dir, "server_config_empty.json")

monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py

@@ -0,0 +1,39 @@
+from monkey_island.cc.server_utils.consts import (
+    DEFAULT_DATA_DIR,
+    DEFAULT_LOG_LEVEL,
+    DEFAULT_START_MONGO_DB,
+)
+from monkey_island.setup.island_config_options import IslandConfigOptions
+
+TEST_CONFIG_FILE_CONTENTS_SPECIFIED = {
+    "data_dir": "/tmp",
+    "log_level": "test",
+    "mongodb": {"start_mongodb": False},
+}
+
+TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED = {}
+
+TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO = {"mongodb": {}}
+
+
+def test_island_config_options__data_dir():
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED)
+    assert options.data_dir == "/tmp"
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED)
+    assert options.data_dir == DEFAULT_DATA_DIR
+
+
+def test_island_config_options__log_level():
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED)
+    assert options.log_level == "test"
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED)
+    assert options.log_level == DEFAULT_LOG_LEVEL
+
+
+def test_island_config_options__mongodb():
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED)
+    assert not options.start_mongodb
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED)
+    assert options.start_mongodb == DEFAULT_START_MONGO_DB
+    options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO)
+    assert options.start_mongodb == DEFAULT_START_MONGO_DB

monkey/tests/unit_tests/monkey_island/test_config_file_parser.py

@@ -0,0 +1,16 @@
+from monkey_island import config_file_parser
+from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_LOG_LEVEL
+
+
+def test_load_server_config_from_file(server_config_init_only):
+    config = config_file_parser.load_island_config_from_file(server_config_init_only)
+
+    assert config.data_dir == "~/.monkey_island"
+    assert config.log_level == "NOTICE"
+
+
+def test_load_server_config_from_file_empty_file(monkeypatch, server_config_empty):
+    config = config_file_parser.load_island_config_from_file(server_config_empty)
+
+    assert config.data_dir == DEFAULT_DATA_DIR
+    assert config.log_level == DEFAULT_LOG_LEVEL

whitelist.py

@@ -165,7 +165,7 @@ ALIBABA  # unused variable (monkey/common/cloud/environment_names.py:10)
 IBM  # unused variable (monkey/common/cloud/environment_names.py:11)
 DigitalOcean  # unused variable (monkey/common/cloud/environment_names.py:12)
 _.aws_info  # unused attribute (monkey/monkey_island/cc/environment/aws.py:13)
-
+build_from_config_file_contents  # unused method 'build_from_config_file_contents' (\monkey_island\setup\island_config_options.py:18)
 
 # these are not needed for it to work, but may be useful extra information to understand what's going on
 WINDOWS_PBA_TYPE  # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23)