diff --git a/CHANGELOG.md b/CHANGELOG.md index aa2a96cb0..c6e3bfc45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Improved the structure of unit tests by scoping fixtures only to relevant modules instead of having a one huge fixture file, improved and renamed the directory structure of unit tests and unit test infrastructure. #1178 +- Create/check data directory on Island init. #1170 ### Removed - Relevant dead code as reported by Vulture. #1149 diff --git a/appimage/run_appimage.sh b/appimage/run_appimage.sh index 837ef5d3a..d31b41843 100644 --- a/appimage/run_appimage.sh +++ b/appimage/run_appimage.sh @@ -3,20 +3,12 @@ PYTHON_CMD="$APPDIR"/opt/python3.7/bin/python3.7 DOT_MONKEY="$HOME"/.monkey_island/ -configure_default_server() { - if [ ! -f "$DOT_MONKEY"/server_config.json ]; then - cp "$APPDIR"/usr/src/monkey_island/cc/server_config.json.standard "$DOT_MONKEY"/server_config.json - fi -} - # shellcheck disable=SC2174 mkdir --mode=0700 --parents "$DOT_MONKEY" DB_DIR="$DOT_MONKEY"/db mkdir --parents "$DB_DIR" -configure_default_server - cd "$APPDIR"/usr/src || exit 1 ./monkey_island/bin/mongodb/bin/mongod --dbpath "$DB_DIR" & ${PYTHON_CMD} ./monkey_island.py --server-config "$DOT_MONKEY"/server_config.json diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index d723a6923..82b930a3e 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -1,22 +1,25 @@ -from gevent import monkey as gevent_monkey +# This import patches other imports and needs to be first +import monkey_island.setup.gevent_setup # noqa: F401 isort:skip +import json + +import monkey_island.cc.environment.environment_singleton as env_singleton from monkey_island.cc.arg_parser import parse_cli_args -from monkey_island.config_file_parser import load_island_config_from_file - -gevent_monkey.patch_all() - -import json # noqa: E402 - -from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E402 +from monkey_island.cc.server_utils.island_logger import setup_logging +from monkey_island.setup.config_setup import setup_config_by_cmd_arg, setup_default_config if "__main__" == __name__: island_args = parse_cli_args() + # This is here in order to catch EVERYTHING, some functions are being called on + # imports, so the log init needs to be first. try: - # This is here in order to catch EVERYTHING, some functions are being called on - # imports, so the log init needs to be first. - config_options = load_island_config_from_file(island_args.server_config_path) - setup_logging(config_options.data_dir, config_options.log_level) + if island_args.server_config_path: + config, server_config_path = setup_config_by_cmd_arg(island_args.server_config_path) + else: + config, server_config_path = setup_default_config() + + setup_logging(config.data_dir, config.log_level) except OSError as ex: print(f"Error opening server config file: {ex}") @@ -26,6 +29,11 @@ if "__main__" == __name__: print(f"Error loading server config: {ex}") exit(1) + # We need to initialize environment singleton before importing main, + # because main imports modules from monkey_island/cc/models and models need a connection to the + # mongodb. Mongodb connection parameters are initialized in environment singleton. + env_singleton.initialize_from_file(server_config_path) + from monkey_island.cc.main import main # noqa: E402 - main(island_args.setup_only, island_args.server_config_path, config_options) + main(island_args.setup_only, config) diff --git a/monkey/monkey_island/cc/arg_parser.py b/monkey/monkey_island/cc/arg_parser.py index 457ffbac2..617658080 100644 --- a/monkey/monkey_island/cc/arg_parser.py +++ b/monkey/monkey_island/cc/arg_parser.py @@ -1,18 +1,10 @@ -from monkey_island.cc.server_utils.consts import ( - DEFAULT_SERVER_CONFIG_PATH, - DEFAULT_SHOULD_SETUP_ONLY, -) +from dataclasses import dataclass +@dataclass class IslandCmdArgs: - setup_only: bool = DEFAULT_SHOULD_SETUP_ONLY - server_config_path: str = DEFAULT_SERVER_CONFIG_PATH - - def __init__(self, setup_only: None, server_config_path: None): - if setup_only: - self.setup_only = setup_only - if server_config_path: - self.server_config_path = server_config_path + setup_only: bool + server_config_path: str def parse_cli_args() -> IslandCmdArgs: diff --git a/monkey/monkey_island/cc/environment/data_dir_generator.py b/monkey/monkey_island/cc/environment/data_dir_generator.py new file mode 100644 index 000000000..58e16d4b7 --- /dev/null +++ b/monkey/monkey_island/cc/environment/data_dir_generator.py @@ -0,0 +1,30 @@ +import logging +import os + +from monkey_island.cc.environment.utils import is_windows_os +from monkey_island.cc.environment.windows_permissions import set_full_folder_access + +LOG = logging.getLogger(__name__) + + +def create_data_dir(data_dir: str, create_parent_dirs: bool) -> None: + if not os.path.isdir(data_dir): + try: + if create_parent_dirs: + os.makedirs(data_dir, mode=0o700) + else: + os.mkdir(data_dir, mode=0o700) + except Exception as ex: + LOG.error( + f'Could not create data directory at "{data_dir}" (maybe `$HOME` could not be ' + f"resolved?): {str(ex)}" + ) + + if is_windows_os(): # `mode=0o700` doesn't work on Windows + try: + set_full_folder_access(folder_path=data_dir) + except Exception as ex: + LOG.error( + f'Data directory was created at "{data_dir}" but permissions could not be ' + f"set successfully: {str(ex)}" + ) diff --git a/monkey/monkey_island/cc/environment/environment_config.py b/monkey/monkey_island/cc/environment/environment_config.py index 6f4626c9e..b013bdcf3 100644 --- a/monkey/monkey_island/cc/environment/environment_config.py +++ b/monkey/monkey_island/cc/environment/environment_config.py @@ -2,10 +2,8 @@ from __future__ import annotations import json import os -from pathlib import Path from typing import Dict, List -import monkey_island.cc.environment.server_config_generator as server_config_generator from monkey_island.cc.environment.user_creds import UserCreds from monkey_island.cc.resources.auth.auth_user import User from monkey_island.cc.resources.auth.user_store import UserStore @@ -24,8 +22,6 @@ class EnvironmentConfig: def _load_from_file(self, file_path): file_path = os.path.expanduser(file_path) - if not Path(file_path).is_file(): - server_config_generator.create_default_config_file(file_path) with open(file_path, "r") as f: config_content = f.read() diff --git a/monkey/monkey_island/cc/environment/environment_singleton.py b/monkey/monkey_island/cc/environment/environment_singleton.py index e7e316ac5..f1a6a2a39 100644 --- a/monkey/monkey_island/cc/environment/environment_singleton.py +++ b/monkey/monkey_island/cc/environment/environment_singleton.py @@ -2,7 +2,6 @@ import logging import monkey_island.cc.resources.auth.user_store as user_store from monkey_island.cc.environment import EnvironmentConfig, aws, password, standard -from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH __author__ = "itay.mizeretz" @@ -48,8 +47,3 @@ def initialize_from_file(file_path): except Exception: logger.error("Failed initializing environment", exc_info=True) raise - - -# TODO: This is only needed so that unit tests pass. After PR #848 is merged, we may be -# able to remove this line. -initialize_from_file(DEFAULT_SERVER_CONFIG_PATH) diff --git a/monkey/monkey_island/cc/environment/server_config_generator.py b/monkey/monkey_island/cc/environment/server_config_generator.py deleted file mode 100644 index 211b745c5..000000000 --- a/monkey/monkey_island/cc/environment/server_config_generator.py +++ /dev/null @@ -1,8 +0,0 @@ -from pathlib import Path - -from monkey_island.cc.server_utils.consts import DEFAULT_DEVELOP_SERVER_CONFIG_PATH - - -def create_default_config_file(path): - default_config = Path(DEFAULT_DEVELOP_SERVER_CONFIG_PATH).read_text() - Path(path).write_text(default_config) diff --git a/monkey/monkey_island/cc/environment/server_config_handler.py b/monkey/monkey_island/cc/environment/server_config_handler.py new file mode 100644 index 000000000..fc5938694 --- /dev/null +++ b/monkey/monkey_island/cc/environment/server_config_handler.py @@ -0,0 +1,27 @@ +import json +import os +from pathlib import Path + +from monkey_island.cc.server_utils.consts import ( + DEFAULT_DEVELOP_SERVER_CONFIG_PATH, + DEFAULT_SERVER_CONFIG_PATH, +) +from monkey_island.setup.island_config_options import IslandConfigOptions + + +def create_default_server_config_file() -> None: + if not os.path.isfile(DEFAULT_SERVER_CONFIG_PATH): + write_default_server_config_to_file(DEFAULT_SERVER_CONFIG_PATH) + + +def write_default_server_config_to_file(path: str) -> None: + default_config = Path(DEFAULT_DEVELOP_SERVER_CONFIG_PATH).read_text() + Path(path).write_text(default_config) + + +def load_server_config_from_file(server_config_path) -> IslandConfigOptions: + with open(server_config_path, "r") as f: + config_content = f.read() + config = json.loads(config_content) + + return IslandConfigOptions(config) diff --git a/monkey/monkey_island/cc/environment/utils.py b/monkey/monkey_island/cc/environment/utils.py new file mode 100644 index 000000000..cbb8a1d6f --- /dev/null +++ b/monkey/monkey_island/cc/environment/utils.py @@ -0,0 +1,5 @@ +import sys + + +def is_windows_os() -> bool: + return sys.platform.startswith("win") diff --git a/monkey/monkey_island/cc/environment/windows_permissions.py b/monkey/monkey_island/cc/environment/windows_permissions.py new file mode 100644 index 000000000..d17947a2e --- /dev/null +++ b/monkey/monkey_island/cc/environment/windows_permissions.py @@ -0,0 +1,34 @@ +from monkey_island.cc.environment.utils import is_windows_os + +if is_windows_os(): + import ntsecuritycon + import win32api + import win32con + import win32security + + +def set_full_folder_access(folder_path: str) -> None: + user = get_user_pySID_object() + + security_descriptor = win32security.GetFileSecurity( + folder_path, win32security.DACL_SECURITY_INFORMATION + ) + dacl = win32security.ACL() + dacl.AddAccessAllowedAce( + win32security.ACL_REVISION, + ntsecuritycon.FILE_ALL_ACCESS, + user, + ) + security_descriptor.SetSecurityDescriptorDacl(1, dacl, 0) + win32security.SetFileSecurity( + folder_path, win32security.DACL_SECURITY_INFORMATION, security_descriptor + ) + + +def get_user_pySID_object(): + # get current user's name + username = win32api.GetUserNameEx(win32con.NameSamCompatible) + # pySID object for the current user + user, _, _ = win32security.LookupAccountName("", username) + + return user diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index 21b118447..df015863b 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -35,9 +35,7 @@ from monkey_island.cc.services.utils.network_utils import local_ip_addresses # MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" -def main(setup_only: bool, server_config_path: str, config_options: IslandConfigOptions): - - env_singleton.initialize_from_file(server_config_path) +def main(setup_only: bool, config_options: IslandConfigOptions): initialize_encryptor(config_options.data_dir) initialize_services(config_options.data_dir) @@ -122,7 +120,3 @@ def assert_mongo_db_version(mongo_url): sys.exit(-1) else: logger.info("Mongo DB version OK. Got {0}".format(str(server_version))) - - -if __name__ == "__main__": - main() diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index 602d815c4..d787a59ef 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -1,17 +1,18 @@ from mongoengine import connect -import monkey_island.cc.environment.environment_singleton as env_singleton +import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 -from .command_control_channel import CommandControlChannel # noqa: F401 +from .command_control_channel import CommandControlChannel # noqa: F401, E402 # Order of importing matters here, for registering the embedded and referenced documents before # using them. -from .config import Config # noqa: F401 -from .creds import Creds # noqa: F401 -from .monkey import Monkey # noqa: F401 -from .monkey_ttl import MonkeyTtl # noqa: F401 -from .pba_results import PbaResults # noqa: F401 +from .config import Config # noqa: F401, E402 +from .creds import Creds # noqa: F401, E402 +from .monkey import Monkey # noqa: F401, E402 +from .monkey_ttl import MonkeyTtl # noqa: F401, E402 +from .pba_results import PbaResults # noqa: F401, E402 +# TODO refactor into explicit call when implementing mongodb startup connect( db=env_singleton.env.mongo_db_name, host=env_singleton.env.mongo_db_host, diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index c25822c6f..bc99b4394 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -1,18 +1,32 @@ import os +from monkey_island.cc.environment.utils import is_windows_os + __author__ = "itay.mizeretz" + +def get_default_data_dir() -> str: + if is_windows_os(): + return r"%AppData%\monkey_island" + else: + return r"$HOME/.monkey_island" + + +SERVER_CONFIG_FILENAME = "server_config.json" + MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island") + +DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir()) + DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5 -# TODO move setup consts -DEFAULT_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json") - -DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( - MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json.develop" +DEFAULT_SERVER_CONFIG_PATH = os.path.expandvars( + os.path.join(DEFAULT_DATA_DIR, SERVER_CONFIG_FILENAME) +) + +DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join( + MONKEY_ISLAND_ABS_PATH, "cc", f"{SERVER_CONFIG_FILENAME}.develop" ) -DEFAULT_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc") DEFAULT_LOG_LEVEL = "INFO" DEFAULT_START_MONGO_DB = True -DEFAULT_SHOULD_SETUP_ONLY = False diff --git a/monkey/monkey_island/setup/config_setup.py b/monkey/monkey_island/setup/config_setup.py new file mode 100644 index 000000000..5c9625ac4 --- /dev/null +++ b/monkey/monkey_island/setup/config_setup.py @@ -0,0 +1,23 @@ +import os +from typing import Tuple + +from monkey_island.cc.environment import server_config_handler +from monkey_island.cc.environment.data_dir_generator import create_data_dir # noqa: E402 +from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH +from monkey_island.setup.island_config_options import IslandConfigOptions + + +def setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, str]: + server_config_path = os.path.expandvars(os.path.expanduser(server_config_path)) + config = server_config_handler.load_server_config_from_file(server_config_path) + + create_data_dir(config.data_dir, create_parent_dirs=True) + return config, server_config_path + + +def setup_default_config() -> Tuple[IslandConfigOptions, str]: + server_config_path = DEFAULT_SERVER_CONFIG_PATH + create_data_dir(DEFAULT_DATA_DIR, create_parent_dirs=False) + server_config_handler.create_default_server_config_file() + config = server_config_handler.load_server_config_from_file(server_config_path) + return config, server_config_path diff --git a/monkey/monkey_island/setup/gevent_setup.py b/monkey/monkey_island/setup/gevent_setup.py new file mode 100644 index 000000000..9fa2b47f9 --- /dev/null +++ b/monkey/monkey_island/setup/gevent_setup.py @@ -0,0 +1,6 @@ +from gevent import monkey as gevent_monkey + +# We need to monkeypatch before any other imports to +# make standard libraries compatible with gevent. +# http://www.gevent.org/api/gevent.monkey.html +gevent_monkey.patch_all() diff --git a/monkey/tests/unit_tests/monkey_island/cc/conftest.py b/monkey/tests/unit_tests/monkey_island/cc/conftest.py index af35c9b25..4bccfd862 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/conftest.py +++ b/monkey/tests/unit_tests/monkey_island/cc/conftest.py @@ -1,3 +1,11 @@ +import monkey_island.cc.environment.environment_singleton as env_singleton +from monkey_island.cc.environment.testing import TestingEnvironment + +# Mock environment singleton because it contains mongodb parameters +# needed for model tests. See monkey/monkey_island/cc/models/__init__.py +env_config = {} +env_singleton.env = TestingEnvironment(env_config) + # Without these imports pytests can't use fixtures, # because they are not found -from tests.unit_tests.monkey_island.cc.mongomock_fixtures import * # noqa: F401,F403 +from tests.unit_tests.monkey_island.cc.mongomock_fixtures import * # noqa: F401,F403,E402 diff --git a/monkey/tests/unit_tests/monkey_island/cc/environment/__init__.py b/monkey/tests/unit_tests/monkey_island/cc/environment/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/tests/unit_tests/monkey_island/cc/environment/test_environment_config.py b/monkey/tests/unit_tests/monkey_island/cc/environment/test_environment_config.py index 481e871ae..0e3efda04 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/environment/test_environment_config.py +++ b/monkey/tests/unit_tests/monkey_island/cc/environment/test_environment_config.py @@ -93,15 +93,3 @@ def test_get_users(standard_with_credentials): assert users[0].id == 1 assert users[0].username == "test" assert users[0].secret == "abcdef" - - -def test_generate_default_file(config_file): - environment_config = EnvironmentConfig(config_file) - - assert os.path.isfile(config_file) - - assert environment_config.server_config == "password" - assert environment_config.deployment == "develop" - assert environment_config.user_creds.username == "" - assert environment_config.user_creds.password_hash == "" - assert environment_config.aws is None diff --git a/monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py b/monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/tests/unit_tests/monkey_island/cc/test_consts.py b/monkey/tests/unit_tests/monkey_island/cc/test_consts.py index eebb7414f..993ddaa64 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/test_consts.py +++ b/monkey/tests/unit_tests/monkey_island/cc/test_consts.py @@ -5,8 +5,8 @@ import monkey_island.cc.server_utils.consts as consts def test_default_server_config_file_path(): if platform.system() == "Windows": - server_file_path = consts.MONKEY_ISLAND_ABS_PATH + r"\cc\server_config.json" + server_file_path = f"{consts.DEFAULT_DATA_DIR}\\{consts.SERVER_CONFIG_FILENAME}" else: - server_file_path = consts.MONKEY_ISLAND_ABS_PATH + "/cc/server_config.json" + server_file_path = f"{consts.DEFAULT_DATA_DIR}/{consts.SERVER_CONFIG_FILENAME}" assert consts.DEFAULT_SERVER_CONFIG_PATH == server_file_path