diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cd76b3cc..c6e3bfc45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Authentication mechanism to use bcrypt on server side. #1139 - `server_config.json` puts environment config options in a separate section named "environment". #1161 +- BlackBox tests can now register if they are ran on a fresh installation. #1180 - Improved the structure of unit tests by scoping fixtures only to relevant modules instead of having a one huge fixture file, improved and renamed the directory structure of unit tests and unit test infrastructure. #1178 diff --git a/appimage/server_config.json.standard b/appimage/server_config.json.standard index 8c894b849..af975a9e0 100644 --- a/appimage/server_config.json.standard +++ b/appimage/server_config.json.standard @@ -4,5 +4,8 @@ "environment": { "server_config": "password", "deployment": "standard" + }, + "mongodb": { + "start_mongodb": true } } diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py index f7be1b3cf..8e8392b9e 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py @@ -12,6 +12,10 @@ NO_AUTH_CREDS = "1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()" LOGGER = logging.getLogger(__name__) +class AuthenticationFailedError(Exception): + pass + + # noinspection PyArgumentList class MonkeyIslandRequests(object): def __init__(self, server_address): @@ -43,6 +47,9 @@ class MonkeyIslandRequests(object): def try_get_jwt_from_server(self): try: return self.get_jwt_from_server() + except AuthenticationFailedError: + self.try_set_island_to_no_password() + return self.get_jwt_from_server() except requests.ConnectionError as err: LOGGER.error( "Unable to connect to island, aborting! Error information: {}. Server: {}".format( @@ -51,6 +58,21 @@ class MonkeyIslandRequests(object): ) assert False + def get_jwt_from_server(self): + resp = requests.post( # noqa: DUO123 + self.addr + "api/auth", + json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS}, + verify=False, + ) + if resp.status_code == 401: + raise AuthenticationFailedError + return resp.json()["access_token"] + + def try_set_island_to_no_password(self): + requests.patch( # noqa: DUO123 + self.addr + "api/environment", json={"server_config": "standard"}, verify=False + ) + class _Decorators: @classmethod def refresh_jwt_token(cls, request_function): @@ -62,14 +84,6 @@ class MonkeyIslandRequests(object): return request_function_wrapper - def get_jwt_from_server(self): - resp = requests.post( # noqa: DUO123 - self.addr + "api/auth", - json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS}, - verify=False, - ) - return resp.json()["access_token"] - @_Decorators.refresh_jwt_token def get(self, url, data=None): return requests.get( # noqa: DUO123 diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 20f495151..5ee5f63c7 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -98,7 +98,15 @@ def wait_machine_bootup(): @pytest.fixture(scope="class") def island_client(island, quick_performance_tests): - island_client_object = MonkeyIslandClient(island) + client_established = False + try: + island_client_object = MonkeyIslandClient(island) + client_established = island_client_object.get_api_status() + except Exception: + logging.exception("Got an exception while trying to establish connection to the Island.") + finally: + if not client_established: + pytest.exit("BB tests couldn't establish communication to the island.") if not quick_performance_tests: island_client_object.reset_env() yield island_client_object @@ -158,10 +166,6 @@ class TestMonkeyBlackbox: def get_log_dir_path(): return os.path.abspath(LOG_DIR_PATH) - def test_server_online(self, island_client): - if not island_client.get_api_status(): - pytest.exit("BB tests couldn't reach the Island server, quiting.") - def test_ssh_exploiter(self, island_client): TestMonkeyBlackbox.run_exploitation_test(island_client, Ssh, "SSH_exploiter_and_keys") diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index e3d8348d1..200dd9b1f 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -20,7 +20,7 @@ if "__main__" == __name__: else: config, server_config_path = setup_default_config() - setup_logging(config["data_dir"], config["log_level"]) + setup_logging(config.data_dir, config.log_level) except OSError as ex: print(f"Error opening server config file: {ex}") @@ -32,4 +32,4 @@ if "__main__" == __name__: from monkey_island.cc.main import main # noqa: E402 - main(config["data_dir"], island_args.setup_only, server_config_path) + main(island_args.setup_only, island_args.server_config_path, config) diff --git a/monkey/monkey_island/cc/arg_parser.py b/monkey/monkey_island/cc/arg_parser.py index 982776230..457ffbac2 100644 --- a/monkey/monkey_island/cc/arg_parser.py +++ b/monkey/monkey_island/cc/arg_parser.py @@ -1,13 +1,21 @@ -from dataclasses import dataclass +from monkey_island.cc.server_utils.consts import ( + DEFAULT_SERVER_CONFIG_PATH, + DEFAULT_SHOULD_SETUP_ONLY, +) -@dataclass -class IslandArgs: - setup_only: bool - server_config: str +class IslandCmdArgs: + setup_only: bool = DEFAULT_SHOULD_SETUP_ONLY + server_config_path: str = DEFAULT_SERVER_CONFIG_PATH + + def __init__(self, setup_only: None, server_config_path: None): + if setup_only: + self.setup_only = setup_only + if server_config_path: + self.server_config_path = server_config_path -def parse_cli_args() -> IslandArgs: +def parse_cli_args() -> IslandCmdArgs: import argparse parser = argparse.ArgumentParser( @@ -27,4 +35,4 @@ def parse_cli_args() -> IslandArgs: ) args = parser.parse_args() - return IslandArgs(args.setup_only, args.server_config) + return IslandCmdArgs(args.setup_only, args.server_config) diff --git a/monkey/monkey_island/cc/environment/server_config_handler.py b/monkey/monkey_island/cc/environment/server_config_handler.py index 35cc91f3e..fc5938694 100644 --- a/monkey/monkey_island/cc/environment/server_config_handler.py +++ b/monkey/monkey_island/cc/environment/server_config_handler.py @@ -3,11 +3,10 @@ import os from pathlib import Path from monkey_island.cc.server_utils.consts import ( - DEFAULT_DATA_DIR, DEFAULT_DEVELOP_SERVER_CONFIG_PATH, - DEFAULT_LOG_LEVEL, DEFAULT_SERVER_CONFIG_PATH, ) +from monkey_island.setup.island_config_options import IslandConfigOptions def create_default_server_config_file() -> None: @@ -20,20 +19,9 @@ def write_default_server_config_to_file(path: str) -> None: Path(path).write_text(default_config) -def load_server_config_from_file(server_config_path): +def load_server_config_from_file(server_config_path) -> IslandConfigOptions: with open(server_config_path, "r") as f: config_content = f.read() config = json.loads(config_content) - add_default_values_to_config(config) - return config - - -def add_default_values_to_config(config): - config["data_dir"] = os.path.abspath( - os.path.expanduser(os.path.expandvars(config.get("data_dir", DEFAULT_DATA_DIR))) - ) - - config.setdefault("log_level", DEFAULT_LOG_LEVEL) - - return config + return IslandConfigOptions(config) diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index cf56144ed..21b118447 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -9,6 +9,8 @@ from threading import Thread # "monkey_island." work. from gevent.pywsgi import WSGIServer +from monkey_island.setup.island_config_options import IslandConfigOptions + MONKEY_ISLAND_DIR_BASE_PATH = str(Path(__file__).parent.parent) if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path: sys.path.insert(0, MONKEY_ISLAND_DIR_BASE_PATH) @@ -22,28 +24,22 @@ from common.version import get_version # noqa: E402 from monkey_island.cc.app import init_app # noqa: E402 from monkey_island.cc.database import get_db_version # noqa: E402 from monkey_island.cc.database import is_db_server_up # noqa: E402 +from monkey_island.cc.mongo_setup import init_collections, launch_mongodb # noqa: E402 from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 -from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH # noqa: E402 from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402 from monkey_island.cc.services.initialize import initialize_services # noqa: E402 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 -from monkey_island.cc.setup import setup # noqa: E402 MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" -def main( - data_dir, - should_setup_only=False, - server_config_filename=DEFAULT_SERVER_CONFIG_PATH, -): - logger.info("Starting bootloader server") +def main(setup_only: bool, server_config_path: str, config_options: IslandConfigOptions): - env_singleton.initialize_from_file(server_config_filename) - initialize_encryptor(data_dir) - initialize_services(data_dir) + env_singleton.initialize_from_file(server_config_path) + initialize_encryptor(config_options.data_dir) + initialize_services(config_options.data_dir) mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url()) bootloader_server_thread = Thread( @@ -51,11 +47,13 @@ def main( ) bootloader_server_thread.start() - start_island_server(should_setup_only) + start_island_server(setup_only, config_options) bootloader_server_thread.join() -def start_island_server(should_setup_only): +def start_island_server(should_setup_only, config_options: IslandConfigOptions): + if config_options.start_mongodb: + launch_mongodb() mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url()) wait_for_mongo_db_server(mongo_url) assert_mongo_db_version(mongo_url) @@ -66,7 +64,7 @@ def start_island_server(should_setup_only): crt_path = str(Path(MONKEY_ISLAND_ABS_PATH, "cc", "server.crt")) key_path = str(Path(MONKEY_ISLAND_ABS_PATH, "cc", "server.key")) - setup() + init_collections() if should_setup_only: logger.warning("Setup only flag passed. Exiting.") diff --git a/monkey/monkey_island/cc/setup.py b/monkey/monkey_island/cc/mongo_setup.py similarity index 94% rename from monkey/monkey_island/cc/setup.py rename to monkey/monkey_island/cc/mongo_setup.py index a03c554be..74cb29fc2 100644 --- a/monkey/monkey_island/cc/setup.py +++ b/monkey/monkey_island/cc/mongo_setup.py @@ -9,7 +9,12 @@ from monkey_island.cc.services.attack.mitre_api_interface import MitreApiInterfa logger = logging.getLogger(__name__) -def setup(): +def launch_mongodb(): + # TODO: Implement the launch of mongodb process + pass + + +def init_collections(): logger.info("Setting up the Monkey Island, this might take a while...") try_store_mitigations_on_mongo() diff --git a/monkey/monkey_island/cc/server_config.json.develop b/monkey/monkey_island/cc/server_config.json.develop index fe9e2687f..5d8dc85aa 100644 --- a/monkey/monkey_island/cc/server_config.json.develop +++ b/monkey/monkey_island/cc/server_config.json.develop @@ -3,5 +3,8 @@ "environment": { "server_config": "password", "deployment": "develop" + }, + "mongodb": { + "start_mongodb": true } } diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index a62bb98ab..4178928e2 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -14,8 +14,6 @@ def get_default_data_dir() -> str: SERVER_CONFIG_FILENAME = "server_config.json" -DEFAULT_LOG_LEVEL = "INFO" - MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island") DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir()) @@ -29,3 +27,7 @@ DEFAULT_SERVER_CONFIG_PATH = os.path.expandvars( DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( MONKEY_ISLAND_ABS_PATH, "cc", f"{SERVER_CONFIG_FILENAME}.develop" ) + +DEFAULT_LOG_LEVEL = "INFO" +DEFAULT_START_MONGO_DB = True +DEFAULT_SHOULD_SETUP_ONLY = False diff --git a/monkey/monkey_island/config_file_parser.py b/monkey/monkey_island/config_file_parser.py new file mode 100644 index 000000000..830ae6720 --- /dev/null +++ b/monkey/monkey_island/config_file_parser.py @@ -0,0 +1,19 @@ +import json +from os.path import isfile + +from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH +from monkey_island.setup.island_config_options import IslandConfigOptions + + +def load_island_config_from_file(server_config_path: str) -> IslandConfigOptions: + config_contents = read_config_file(server_config_path) + return IslandConfigOptions(config_contents) + + +def read_config_file(server_config_path: str) -> dict: + if not server_config_path or not isfile(server_config_path): + server_config_path = DEFAULT_SERVER_CONFIG_PATH + with open(server_config_path, "r") as f: + config_content = f.read() + config = json.loads(config_content) + return config diff --git a/monkey/monkey_island/setup/__init__.py b/monkey/monkey_island/setup/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/monkey_island/setup/config_setup.py b/monkey/monkey_island/setup/config_setup.py index 05f410fa8..12f073f64 100644 --- a/monkey/monkey_island/setup/config_setup.py +++ b/monkey/monkey_island/setup/config_setup.py @@ -4,16 +4,18 @@ from typing import Tuple from monkey_island.cc.environment import server_config_handler from monkey_island.cc.environment.data_dir_generator import create_data_dir # noqa: E402 from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH +from monkey_island.setup.island_config_options import IslandConfigOptions -def setup_config_by_cmd_arg(server_config_path) -> Tuple[dict, str]: +def setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, str]: server_config_path = os.path.expandvars(os.path.expanduser(server_config_path)) config = server_config_handler.load_server_config_from_file(server_config_path) + create_data_dir(config["data_dir"], create_parent_dirs=True) return config, server_config_path -def setup_default_config() -> Tuple[dict, str]: +def setup_default_config() -> Tuple[IslandConfigOptions, str]: server_config_path = DEFAULT_SERVER_CONFIG_PATH create_data_dir(DEFAULT_DATA_DIR, create_parent_dirs=False) server_config_handler.create_default_server_config_file() diff --git a/monkey/monkey_island/setup/island_config_options.py b/monkey/monkey_island/setup/island_config_options.py new file mode 100644 index 000000000..bf1c06e1b --- /dev/null +++ b/monkey/monkey_island/setup/island_config_options.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from monkey_island.cc.server_utils.consts import ( + DEFAULT_DATA_DIR, + DEFAULT_LOG_LEVEL, + DEFAULT_START_MONGO_DB, +) + + +class IslandConfigOptions: + def __init__(self, config_contents: dict): + self.data_dir = config_contents.get("data_dir", DEFAULT_DATA_DIR) + + self.log_level = config_contents.get("log_level", DEFAULT_LOG_LEVEL) + + self.start_mongodb = config_contents.get( + "mongodb", {"start_mongodb": DEFAULT_START_MONGO_DB} + ).get("start_mongodb", DEFAULT_START_MONGO_DB) diff --git a/monkey/tests/data_for_tests/server_configs/server_config_empty.json b/monkey/tests/data_for_tests/server_configs/server_config_empty.json new file mode 100644 index 000000000..2c63c0851 --- /dev/null +++ b/monkey/tests/data_for_tests/server_configs/server_config_empty.json @@ -0,0 +1,2 @@ +{ +} diff --git a/monkey/tests/data_for_tests/server_configs/test_server_config.json b/monkey/tests/data_for_tests/server_configs/server_config_init_only.json similarity index 100% rename from monkey/tests/data_for_tests/server_configs/test_server_config.json rename to monkey/tests/data_for_tests/server_configs/server_config_init_only.json diff --git a/monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py b/monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py index acd89d84f..e69de29bb 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py +++ b/monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py @@ -1,27 +0,0 @@ -import os - -from monkey_island.cc.environment import server_config_handler -from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR - - -def test_load_server_config_from_file(test_server_config, mock_home_env): - config = server_config_handler.load_server_config_from_file(test_server_config) - - assert config["data_dir"] == os.path.join(mock_home_env, ".monkey_island") - assert config["log_level"] == "NOTICE" - - -def test_default_log_level(): - test_config = {} - config = server_config_handler.add_default_values_to_config(test_config) - - assert "log_level" in config - assert config["log_level"] == "INFO" - - -def test_default_data_dir(mock_home_env): - test_config = {} - config = server_config_handler.add_default_values_to_config(test_config) - - assert "data_dir" in config - assert config["data_dir"] == DEFAULT_DATA_DIR diff --git a/monkey/tests/unit_tests/monkey_island/conftest.py b/monkey/tests/unit_tests/monkey_island/conftest.py index 4b7149595..4d0d48e1f 100644 --- a/monkey/tests/unit_tests/monkey_island/conftest.py +++ b/monkey/tests/unit_tests/monkey_island/conftest.py @@ -9,5 +9,10 @@ def server_configs_dir(data_for_tests_dir): @pytest.fixture(scope="module") -def test_server_config(server_configs_dir): - return os.path.join(server_configs_dir, "test_server_config.json") +def server_config_init_only(server_configs_dir): + return os.path.join(server_configs_dir, "server_config_init_only.json") + + +@pytest.fixture(scope="module") +def server_config_empty(server_configs_dir): + return os.path.join(server_configs_dir, "server_config_empty.json") diff --git a/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py b/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py new file mode 100644 index 000000000..9175bc274 --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/setup/test_island_config_options.py @@ -0,0 +1,39 @@ +from monkey_island.cc.server_utils.consts import ( + DEFAULT_DATA_DIR, + DEFAULT_LOG_LEVEL, + DEFAULT_START_MONGO_DB, +) +from monkey_island.setup.island_config_options import IslandConfigOptions + +TEST_CONFIG_FILE_CONTENTS_SPECIFIED = { + "data_dir": "/tmp", + "log_level": "test", + "mongodb": {"start_mongodb": False}, +} + +TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED = {} + +TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO = {"mongodb": {}} + + +def test_island_config_options__data_dir(): + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED) + assert options.data_dir == "/tmp" + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED) + assert options.data_dir == DEFAULT_DATA_DIR + + +def test_island_config_options__log_level(): + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED) + assert options.log_level == "test" + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED) + assert options.log_level == DEFAULT_LOG_LEVEL + + +def test_island_config_options__mongodb(): + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED) + assert not options.start_mongodb + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED) + assert options.start_mongodb == DEFAULT_START_MONGO_DB + options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO) + assert options.start_mongodb == DEFAULT_START_MONGO_DB diff --git a/monkey/tests/unit_tests/monkey_island/test_config_file_parser.py b/monkey/tests/unit_tests/monkey_island/test_config_file_parser.py new file mode 100644 index 000000000..db377e6e7 --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/test_config_file_parser.py @@ -0,0 +1,16 @@ +from monkey_island import config_file_parser +from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_LOG_LEVEL + + +def test_load_server_config_from_file(server_config_init_only): + config = config_file_parser.load_island_config_from_file(server_config_init_only) + + assert config.data_dir == "~/.monkey_island" + assert config.log_level == "NOTICE" + + +def test_load_server_config_from_file_empty_file(monkeypatch, server_config_empty): + config = config_file_parser.load_island_config_from_file(server_config_empty) + + assert config.data_dir == DEFAULT_DATA_DIR + assert config.log_level == DEFAULT_LOG_LEVEL diff --git a/whitelist.py b/whitelist.py index bd147220a..51d4c22b8 100644 --- a/whitelist.py +++ b/whitelist.py @@ -165,7 +165,7 @@ ALIBABA # unused variable (monkey/common/cloud/environment_names.py:10) IBM # unused variable (monkey/common/cloud/environment_names.py:11) DigitalOcean # unused variable (monkey/common/cloud/environment_names.py:12) _.aws_info # unused attribute (monkey/monkey_island/cc/environment/aws.py:13) - +build_from_config_file_contents # unused method 'build_from_config_file_contents' (\monkey_island\setup\island_config_options.py:18) # these are not needed for it to work, but may be useful extra information to understand what's going on WINDOWS_PBA_TYPE # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23)