Merged develop into data-dir-on-island-init
This commit is contained in:
commit
7d1c5dd908
|
@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
|||
- Authentication mechanism to use bcrypt on server side. #1139
|
||||
- `server_config.json` puts environment config options in a separate section
|
||||
named "environment". #1161
|
||||
- BlackBox tests can now register if they are ran on a fresh installation. #1180
|
||||
- Improved the structure of unit tests by scoping fixtures only to relevant modules
|
||||
instead of having a one huge fixture file, improved and renamed the directory
|
||||
structure of unit tests and unit test infrastructure. #1178
|
||||
|
|
|
@ -4,5 +4,8 @@
|
|||
"environment": {
|
||||
"server_config": "password",
|
||||
"deployment": "standard"
|
||||
},
|
||||
"mongodb": {
|
||||
"start_mongodb": true
|
||||
}
|
||||
}
|
||||
|
|
|
@ -12,6 +12,10 @@ NO_AUTH_CREDS = "1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()"
|
|||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AuthenticationFailedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# noinspection PyArgumentList
|
||||
class MonkeyIslandRequests(object):
|
||||
def __init__(self, server_address):
|
||||
|
@ -43,6 +47,9 @@ class MonkeyIslandRequests(object):
|
|||
def try_get_jwt_from_server(self):
|
||||
try:
|
||||
return self.get_jwt_from_server()
|
||||
except AuthenticationFailedError:
|
||||
self.try_set_island_to_no_password()
|
||||
return self.get_jwt_from_server()
|
||||
except requests.ConnectionError as err:
|
||||
LOGGER.error(
|
||||
"Unable to connect to island, aborting! Error information: {}. Server: {}".format(
|
||||
|
@ -51,6 +58,21 @@ class MonkeyIslandRequests(object):
|
|||
)
|
||||
assert False
|
||||
|
||||
def get_jwt_from_server(self):
|
||||
resp = requests.post( # noqa: DUO123
|
||||
self.addr + "api/auth",
|
||||
json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS},
|
||||
verify=False,
|
||||
)
|
||||
if resp.status_code == 401:
|
||||
raise AuthenticationFailedError
|
||||
return resp.json()["access_token"]
|
||||
|
||||
def try_set_island_to_no_password(self):
|
||||
requests.patch( # noqa: DUO123
|
||||
self.addr + "api/environment", json={"server_config": "standard"}, verify=False
|
||||
)
|
||||
|
||||
class _Decorators:
|
||||
@classmethod
|
||||
def refresh_jwt_token(cls, request_function):
|
||||
|
@ -62,14 +84,6 @@ class MonkeyIslandRequests(object):
|
|||
|
||||
return request_function_wrapper
|
||||
|
||||
def get_jwt_from_server(self):
|
||||
resp = requests.post( # noqa: DUO123
|
||||
self.addr + "api/auth",
|
||||
json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS},
|
||||
verify=False,
|
||||
)
|
||||
return resp.json()["access_token"]
|
||||
|
||||
@_Decorators.refresh_jwt_token
|
||||
def get(self, url, data=None):
|
||||
return requests.get( # noqa: DUO123
|
||||
|
|
|
@ -98,7 +98,15 @@ def wait_machine_bootup():
|
|||
|
||||
@pytest.fixture(scope="class")
|
||||
def island_client(island, quick_performance_tests):
|
||||
client_established = False
|
||||
try:
|
||||
island_client_object = MonkeyIslandClient(island)
|
||||
client_established = island_client_object.get_api_status()
|
||||
except Exception:
|
||||
logging.exception("Got an exception while trying to establish connection to the Island.")
|
||||
finally:
|
||||
if not client_established:
|
||||
pytest.exit("BB tests couldn't establish communication to the island.")
|
||||
if not quick_performance_tests:
|
||||
island_client_object.reset_env()
|
||||
yield island_client_object
|
||||
|
@ -158,10 +166,6 @@ class TestMonkeyBlackbox:
|
|||
def get_log_dir_path():
|
||||
return os.path.abspath(LOG_DIR_PATH)
|
||||
|
||||
def test_server_online(self, island_client):
|
||||
if not island_client.get_api_status():
|
||||
pytest.exit("BB tests couldn't reach the Island server, quiting.")
|
||||
|
||||
def test_ssh_exploiter(self, island_client):
|
||||
TestMonkeyBlackbox.run_exploitation_test(island_client, Ssh, "SSH_exploiter_and_keys")
|
||||
|
||||
|
|
|
@ -20,7 +20,7 @@ if "__main__" == __name__:
|
|||
else:
|
||||
config, server_config_path = setup_default_config()
|
||||
|
||||
setup_logging(config["data_dir"], config["log_level"])
|
||||
setup_logging(config.data_dir, config.log_level)
|
||||
|
||||
except OSError as ex:
|
||||
print(f"Error opening server config file: {ex}")
|
||||
|
@ -32,4 +32,4 @@ if "__main__" == __name__:
|
|||
|
||||
from monkey_island.cc.main import main # noqa: E402
|
||||
|
||||
main(config["data_dir"], island_args.setup_only, server_config_path)
|
||||
main(island_args.setup_only, island_args.server_config_path, config)
|
||||
|
|
|
@ -1,13 +1,21 @@
|
|||
from dataclasses import dataclass
|
||||
from monkey_island.cc.server_utils.consts import (
|
||||
DEFAULT_SERVER_CONFIG_PATH,
|
||||
DEFAULT_SHOULD_SETUP_ONLY,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class IslandArgs:
|
||||
setup_only: bool
|
||||
server_config: str
|
||||
class IslandCmdArgs:
|
||||
setup_only: bool = DEFAULT_SHOULD_SETUP_ONLY
|
||||
server_config_path: str = DEFAULT_SERVER_CONFIG_PATH
|
||||
|
||||
def __init__(self, setup_only: None, server_config_path: None):
|
||||
if setup_only:
|
||||
self.setup_only = setup_only
|
||||
if server_config_path:
|
||||
self.server_config_path = server_config_path
|
||||
|
||||
|
||||
def parse_cli_args() -> IslandArgs:
|
||||
def parse_cli_args() -> IslandCmdArgs:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
|
@ -27,4 +35,4 @@ def parse_cli_args() -> IslandArgs:
|
|||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
return IslandArgs(args.setup_only, args.server_config)
|
||||
return IslandCmdArgs(args.setup_only, args.server_config)
|
||||
|
|
|
@ -3,11 +3,10 @@ import os
|
|||
from pathlib import Path
|
||||
|
||||
from monkey_island.cc.server_utils.consts import (
|
||||
DEFAULT_DATA_DIR,
|
||||
DEFAULT_DEVELOP_SERVER_CONFIG_PATH,
|
||||
DEFAULT_LOG_LEVEL,
|
||||
DEFAULT_SERVER_CONFIG_PATH,
|
||||
)
|
||||
from monkey_island.setup.island_config_options import IslandConfigOptions
|
||||
|
||||
|
||||
def create_default_server_config_file() -> None:
|
||||
|
@ -20,20 +19,9 @@ def write_default_server_config_to_file(path: str) -> None:
|
|||
Path(path).write_text(default_config)
|
||||
|
||||
|
||||
def load_server_config_from_file(server_config_path):
|
||||
def load_server_config_from_file(server_config_path) -> IslandConfigOptions:
|
||||
with open(server_config_path, "r") as f:
|
||||
config_content = f.read()
|
||||
config = json.loads(config_content)
|
||||
add_default_values_to_config(config)
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def add_default_values_to_config(config):
|
||||
config["data_dir"] = os.path.abspath(
|
||||
os.path.expanduser(os.path.expandvars(config.get("data_dir", DEFAULT_DATA_DIR)))
|
||||
)
|
||||
|
||||
config.setdefault("log_level", DEFAULT_LOG_LEVEL)
|
||||
|
||||
return config
|
||||
return IslandConfigOptions(config)
|
||||
|
|
|
@ -9,6 +9,8 @@ from threading import Thread
|
|||
# "monkey_island." work.
|
||||
from gevent.pywsgi import WSGIServer
|
||||
|
||||
from monkey_island.setup.island_config_options import IslandConfigOptions
|
||||
|
||||
MONKEY_ISLAND_DIR_BASE_PATH = str(Path(__file__).parent.parent)
|
||||
if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path:
|
||||
sys.path.insert(0, MONKEY_ISLAND_DIR_BASE_PATH)
|
||||
|
@ -22,28 +24,22 @@ from common.version import get_version # noqa: E402
|
|||
from monkey_island.cc.app import init_app # noqa: E402
|
||||
from monkey_island.cc.database import get_db_version # noqa: E402
|
||||
from monkey_island.cc.database import is_db_server_up # noqa: E402
|
||||
from monkey_island.cc.mongo_setup import init_collections, launch_mongodb # noqa: E402
|
||||
from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402
|
||||
from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402
|
||||
from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH # noqa: E402
|
||||
from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402
|
||||
from monkey_island.cc.services.initialize import initialize_services # noqa: E402
|
||||
from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402
|
||||
from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402
|
||||
from monkey_island.cc.setup import setup # noqa: E402
|
||||
|
||||
MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0"
|
||||
|
||||
|
||||
def main(
|
||||
data_dir,
|
||||
should_setup_only=False,
|
||||
server_config_filename=DEFAULT_SERVER_CONFIG_PATH,
|
||||
):
|
||||
logger.info("Starting bootloader server")
|
||||
def main(setup_only: bool, server_config_path: str, config_options: IslandConfigOptions):
|
||||
|
||||
env_singleton.initialize_from_file(server_config_filename)
|
||||
initialize_encryptor(data_dir)
|
||||
initialize_services(data_dir)
|
||||
env_singleton.initialize_from_file(server_config_path)
|
||||
initialize_encryptor(config_options.data_dir)
|
||||
initialize_services(config_options.data_dir)
|
||||
|
||||
mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url())
|
||||
bootloader_server_thread = Thread(
|
||||
|
@ -51,11 +47,13 @@ def main(
|
|||
)
|
||||
|
||||
bootloader_server_thread.start()
|
||||
start_island_server(should_setup_only)
|
||||
start_island_server(setup_only, config_options)
|
||||
bootloader_server_thread.join()
|
||||
|
||||
|
||||
def start_island_server(should_setup_only):
|
||||
def start_island_server(should_setup_only, config_options: IslandConfigOptions):
|
||||
if config_options.start_mongodb:
|
||||
launch_mongodb()
|
||||
mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url())
|
||||
wait_for_mongo_db_server(mongo_url)
|
||||
assert_mongo_db_version(mongo_url)
|
||||
|
@ -66,7 +64,7 @@ def start_island_server(should_setup_only):
|
|||
crt_path = str(Path(MONKEY_ISLAND_ABS_PATH, "cc", "server.crt"))
|
||||
key_path = str(Path(MONKEY_ISLAND_ABS_PATH, "cc", "server.key"))
|
||||
|
||||
setup()
|
||||
init_collections()
|
||||
|
||||
if should_setup_only:
|
||||
logger.warning("Setup only flag passed. Exiting.")
|
||||
|
|
|
@ -9,7 +9,12 @@ from monkey_island.cc.services.attack.mitre_api_interface import MitreApiInterfa
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def setup():
|
||||
def launch_mongodb():
|
||||
# TODO: Implement the launch of mongodb process
|
||||
pass
|
||||
|
||||
|
||||
def init_collections():
|
||||
logger.info("Setting up the Monkey Island, this might take a while...")
|
||||
try_store_mitigations_on_mongo()
|
||||
|
|
@ -3,5 +3,8 @@
|
|||
"environment": {
|
||||
"server_config": "password",
|
||||
"deployment": "develop"
|
||||
},
|
||||
"mongodb": {
|
||||
"start_mongodb": true
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,8 +14,6 @@ def get_default_data_dir() -> str:
|
|||
|
||||
SERVER_CONFIG_FILENAME = "server_config.json"
|
||||
|
||||
DEFAULT_LOG_LEVEL = "INFO"
|
||||
|
||||
MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island")
|
||||
|
||||
DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir())
|
||||
|
@ -29,3 +27,7 @@ DEFAULT_SERVER_CONFIG_PATH = os.path.expandvars(
|
|||
DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join(
|
||||
MONKEY_ISLAND_ABS_PATH, "cc", f"{SERVER_CONFIG_FILENAME}.develop"
|
||||
)
|
||||
|
||||
DEFAULT_LOG_LEVEL = "INFO"
|
||||
DEFAULT_START_MONGO_DB = True
|
||||
DEFAULT_SHOULD_SETUP_ONLY = False
|
||||
|
|
|
@ -0,0 +1,19 @@
|
|||
import json
|
||||
from os.path import isfile
|
||||
|
||||
from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH
|
||||
from monkey_island.setup.island_config_options import IslandConfigOptions
|
||||
|
||||
|
||||
def load_island_config_from_file(server_config_path: str) -> IslandConfigOptions:
|
||||
config_contents = read_config_file(server_config_path)
|
||||
return IslandConfigOptions(config_contents)
|
||||
|
||||
|
||||
def read_config_file(server_config_path: str) -> dict:
|
||||
if not server_config_path or not isfile(server_config_path):
|
||||
server_config_path = DEFAULT_SERVER_CONFIG_PATH
|
||||
with open(server_config_path, "r") as f:
|
||||
config_content = f.read()
|
||||
config = json.loads(config_content)
|
||||
return config
|
|
@ -4,16 +4,18 @@ from typing import Tuple
|
|||
from monkey_island.cc.environment import server_config_handler
|
||||
from monkey_island.cc.environment.data_dir_generator import create_data_dir # noqa: E402
|
||||
from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH
|
||||
from monkey_island.setup.island_config_options import IslandConfigOptions
|
||||
|
||||
|
||||
def setup_config_by_cmd_arg(server_config_path) -> Tuple[dict, str]:
|
||||
def setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, str]:
|
||||
server_config_path = os.path.expandvars(os.path.expanduser(server_config_path))
|
||||
config = server_config_handler.load_server_config_from_file(server_config_path)
|
||||
|
||||
create_data_dir(config["data_dir"], create_parent_dirs=True)
|
||||
return config, server_config_path
|
||||
|
||||
|
||||
def setup_default_config() -> Tuple[dict, str]:
|
||||
def setup_default_config() -> Tuple[IslandConfigOptions, str]:
|
||||
server_config_path = DEFAULT_SERVER_CONFIG_PATH
|
||||
create_data_dir(DEFAULT_DATA_DIR, create_parent_dirs=False)
|
||||
server_config_handler.create_default_server_config_file()
|
||||
|
|
|
@ -0,0 +1,18 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from monkey_island.cc.server_utils.consts import (
|
||||
DEFAULT_DATA_DIR,
|
||||
DEFAULT_LOG_LEVEL,
|
||||
DEFAULT_START_MONGO_DB,
|
||||
)
|
||||
|
||||
|
||||
class IslandConfigOptions:
|
||||
def __init__(self, config_contents: dict):
|
||||
self.data_dir = config_contents.get("data_dir", DEFAULT_DATA_DIR)
|
||||
|
||||
self.log_level = config_contents.get("log_level", DEFAULT_LOG_LEVEL)
|
||||
|
||||
self.start_mongodb = config_contents.get(
|
||||
"mongodb", {"start_mongodb": DEFAULT_START_MONGO_DB}
|
||||
).get("start_mongodb", DEFAULT_START_MONGO_DB)
|
|
@ -0,0 +1,2 @@
|
|||
{
|
||||
}
|
|
@ -1,27 +0,0 @@
|
|||
import os
|
||||
|
||||
from monkey_island.cc.environment import server_config_handler
|
||||
from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR
|
||||
|
||||
|
||||
def test_load_server_config_from_file(test_server_config, mock_home_env):
|
||||
config = server_config_handler.load_server_config_from_file(test_server_config)
|
||||
|
||||
assert config["data_dir"] == os.path.join(mock_home_env, ".monkey_island")
|
||||
assert config["log_level"] == "NOTICE"
|
||||
|
||||
|
||||
def test_default_log_level():
|
||||
test_config = {}
|
||||
config = server_config_handler.add_default_values_to_config(test_config)
|
||||
|
||||
assert "log_level" in config
|
||||
assert config["log_level"] == "INFO"
|
||||
|
||||
|
||||
def test_default_data_dir(mock_home_env):
|
||||
test_config = {}
|
||||
config = server_config_handler.add_default_values_to_config(test_config)
|
||||
|
||||
assert "data_dir" in config
|
||||
assert config["data_dir"] == DEFAULT_DATA_DIR
|
|
@ -9,5 +9,10 @@ def server_configs_dir(data_for_tests_dir):
|
|||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def test_server_config(server_configs_dir):
|
||||
return os.path.join(server_configs_dir, "test_server_config.json")
|
||||
def server_config_init_only(server_configs_dir):
|
||||
return os.path.join(server_configs_dir, "server_config_init_only.json")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_config_empty(server_configs_dir):
|
||||
return os.path.join(server_configs_dir, "server_config_empty.json")
|
||||
|
|
|
@ -0,0 +1,39 @@
|
|||
from monkey_island.cc.server_utils.consts import (
|
||||
DEFAULT_DATA_DIR,
|
||||
DEFAULT_LOG_LEVEL,
|
||||
DEFAULT_START_MONGO_DB,
|
||||
)
|
||||
from monkey_island.setup.island_config_options import IslandConfigOptions
|
||||
|
||||
TEST_CONFIG_FILE_CONTENTS_SPECIFIED = {
|
||||
"data_dir": "/tmp",
|
||||
"log_level": "test",
|
||||
"mongodb": {"start_mongodb": False},
|
||||
}
|
||||
|
||||
TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED = {}
|
||||
|
||||
TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO = {"mongodb": {}}
|
||||
|
||||
|
||||
def test_island_config_options__data_dir():
|
||||
options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED)
|
||||
assert options.data_dir == "/tmp"
|
||||
options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED)
|
||||
assert options.data_dir == DEFAULT_DATA_DIR
|
||||
|
||||
|
||||
def test_island_config_options__log_level():
|
||||
options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED)
|
||||
assert options.log_level == "test"
|
||||
options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED)
|
||||
assert options.log_level == DEFAULT_LOG_LEVEL
|
||||
|
||||
|
||||
def test_island_config_options__mongodb():
|
||||
options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_SPECIFIED)
|
||||
assert not options.start_mongodb
|
||||
options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_UNSPECIFIED)
|
||||
assert options.start_mongodb == DEFAULT_START_MONGO_DB
|
||||
options = IslandConfigOptions(TEST_CONFIG_FILE_CONTENTS_NO_STARTMONGO)
|
||||
assert options.start_mongodb == DEFAULT_START_MONGO_DB
|
|
@ -0,0 +1,16 @@
|
|||
from monkey_island import config_file_parser
|
||||
from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_LOG_LEVEL
|
||||
|
||||
|
||||
def test_load_server_config_from_file(server_config_init_only):
|
||||
config = config_file_parser.load_island_config_from_file(server_config_init_only)
|
||||
|
||||
assert config.data_dir == "~/.monkey_island"
|
||||
assert config.log_level == "NOTICE"
|
||||
|
||||
|
||||
def test_load_server_config_from_file_empty_file(monkeypatch, server_config_empty):
|
||||
config = config_file_parser.load_island_config_from_file(server_config_empty)
|
||||
|
||||
assert config.data_dir == DEFAULT_DATA_DIR
|
||||
assert config.log_level == DEFAULT_LOG_LEVEL
|
|
@ -165,7 +165,7 @@ ALIBABA # unused variable (monkey/common/cloud/environment_names.py:10)
|
|||
IBM # unused variable (monkey/common/cloud/environment_names.py:11)
|
||||
DigitalOcean # unused variable (monkey/common/cloud/environment_names.py:12)
|
||||
_.aws_info # unused attribute (monkey/monkey_island/cc/environment/aws.py:13)
|
||||
|
||||
build_from_config_file_contents # unused method 'build_from_config_file_contents' (\monkey_island\setup\island_config_options.py:18)
|
||||
|
||||
# these are not needed for it to work, but may be useful extra information to understand what's going on
|
||||
WINDOWS_PBA_TYPE # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23)
|
||||
|
|
Loading…
Reference in New Issue