Merge pull request #1170 from guardicore/data-dir-on-island-init

Create data directory on Island initialisation

CHANGELOG.md

@@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Improved the structure of unit tests by scoping fixtures only to relevant modules
   instead of having a one huge fixture file, improved and renamed the directory
   structure of unit tests and unit test infrastructure. #1178
+- Create/check data directory on Island init. #1170
 
 ### Removed
 - Relevant dead code as reported by Vulture. #1149

appimage/run_appimage.sh

@@ -3,20 +3,12 @@
 PYTHON_CMD="$APPDIR"/opt/python3.7/bin/python3.7
 DOT_MONKEY="$HOME"/.monkey_island/
 
-configure_default_server() {
-    if [ ! -f "$DOT_MONKEY"/server_config.json ]; then
-        cp "$APPDIR"/usr/src/monkey_island/cc/server_config.json.standard "$DOT_MONKEY"/server_config.json
-    fi
-}
-
 # shellcheck disable=SC2174
 mkdir --mode=0700 --parents "$DOT_MONKEY"
 
 DB_DIR="$DOT_MONKEY"/db
 mkdir --parents "$DB_DIR"
 
-configure_default_server
-
 cd "$APPDIR"/usr/src || exit 1
 ./monkey_island/bin/mongodb/bin/mongod --dbpath "$DB_DIR" &
 ${PYTHON_CMD} ./monkey_island.py --server-config "$DOT_MONKEY"/server_config.json

monkey/monkey_island.py

@@ -1,22 +1,25 @@
-from gevent import monkey as gevent_monkey
+# This import patches other imports and needs to be first
+import monkey_island.setup.gevent_setup  # noqa: F401 isort:skip
 
+import json
+
+import monkey_island.cc.environment.environment_singleton as env_singleton
 from monkey_island.cc.arg_parser import parse_cli_args
-from monkey_island.config_file_parser import load_island_config_from_file
+from monkey_island.cc.server_utils.island_logger import setup_logging
-
+from monkey_island.setup.config_setup import setup_config_by_cmd_arg, setup_default_config
-gevent_monkey.patch_all()
-
-import json  # noqa: E402
-
-from monkey_island.cc.server_utils.island_logger import setup_logging  # noqa: E402
 
 if "__main__" == __name__:
     island_args = parse_cli_args()
 
+    # This is here in order to catch EVERYTHING, some functions are being called on
+    # imports, so the log init needs to be first.
     try:
-        # This is here in order to catch EVERYTHING, some functions are being called on
+        if island_args.server_config_path:
-        # imports, so the log init needs to be first.
+            config, server_config_path = setup_config_by_cmd_arg(island_args.server_config_path)
-        config_options = load_island_config_from_file(island_args.server_config_path)
+        else:
-        setup_logging(config_options.data_dir, config_options.log_level)
+            config, server_config_path = setup_default_config()
+
+        setup_logging(config.data_dir, config.log_level)
 
     except OSError as ex:
         print(f"Error opening server config file: {ex}")
@@ -26,6 +29,11 @@ if "__main__" == __name__:
         print(f"Error loading server config: {ex}")
         exit(1)
 
+    # We need to initialize environment singleton before importing main,
+    # because main imports modules from monkey_island/cc/models and models need a connection to the
+    # mongodb. Mongodb connection parameters are initialized in environment singleton.
+    env_singleton.initialize_from_file(server_config_path)
+
     from monkey_island.cc.main import main  # noqa: E402
 
-    main(island_args.setup_only, island_args.server_config_path, config_options)
+    main(island_args.setup_only, config)

monkey/monkey_island/cc/arg_parser.py

@@ -1,18 +1,10 @@
-from monkey_island.cc.server_utils.consts import (
+from dataclasses import dataclass
-    DEFAULT_SERVER_CONFIG_PATH,
-    DEFAULT_SHOULD_SETUP_ONLY,
-)
 
 
+@dataclass
 class IslandCmdArgs:
-    setup_only: bool = DEFAULT_SHOULD_SETUP_ONLY
+    setup_only: bool
-    server_config_path: str = DEFAULT_SERVER_CONFIG_PATH
+    server_config_path: str
-
-    def __init__(self, setup_only: None, server_config_path: None):
-        if setup_only:
-            self.setup_only = setup_only
-        if server_config_path:
-            self.server_config_path = server_config_path
 
 
 def parse_cli_args() -> IslandCmdArgs:

monkey/monkey_island/cc/environment/data_dir_generator.py

@@ -0,0 +1,30 @@
+import logging
+import os
+
+from monkey_island.cc.environment.utils import is_windows_os
+from monkey_island.cc.environment.windows_permissions import set_full_folder_access
+
+LOG = logging.getLogger(__name__)
+
+
+def create_data_dir(data_dir: str, create_parent_dirs: bool) -> None:
+    if not os.path.isdir(data_dir):
+        try:
+            if create_parent_dirs:
+                os.makedirs(data_dir, mode=0o700)
+            else:
+                os.mkdir(data_dir, mode=0o700)
+        except Exception as ex:
+            LOG.error(
+                f'Could not create data directory at "{data_dir}" (maybe `$HOME` could not be '
+                f"resolved?): {str(ex)}"
+            )
+
+        if is_windows_os():  # `mode=0o700` doesn't work on Windows
+            try:
+                set_full_folder_access(folder_path=data_dir)
+            except Exception as ex:
+                LOG.error(
+                    f'Data directory was created at "{data_dir}" but permissions could not be '
+                    f"set successfully: {str(ex)}"
+                )

monkey/monkey_island/cc/environment/environment_config.py

@@ -2,10 +2,8 @@ from __future__ import annotations
 
 import json
 import os
-from pathlib import Path
 from typing import Dict, List
 
-import monkey_island.cc.environment.server_config_generator as server_config_generator
 from monkey_island.cc.environment.user_creds import UserCreds
 from monkey_island.cc.resources.auth.auth_user import User
 from monkey_island.cc.resources.auth.user_store import UserStore
@@ -24,8 +22,6 @@ class EnvironmentConfig:
     def _load_from_file(self, file_path):
         file_path = os.path.expanduser(file_path)
 
-        if not Path(file_path).is_file():
-            server_config_generator.create_default_config_file(file_path)
         with open(file_path, "r") as f:
             config_content = f.read()
 

monkey/monkey_island/cc/environment/environment_singleton.py

@@ -2,7 +2,6 @@ import logging
 
 import monkey_island.cc.resources.auth.user_store as user_store
 from monkey_island.cc.environment import EnvironmentConfig, aws, password, standard
-from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH
 
 __author__ = "itay.mizeretz"
 
@@ -48,8 +47,3 @@ def initialize_from_file(file_path):
     except Exception:
         logger.error("Failed initializing environment", exc_info=True)
         raise
-
-
-# TODO: This is only needed so that unit tests pass. After PR #848 is merged, we may be
-# able to remove this line.
-initialize_from_file(DEFAULT_SERVER_CONFIG_PATH)

monkey/monkey_island/cc/environment/server_config_generator.py

@@ -1,8 +0,0 @@
-from pathlib import Path
-
-from monkey_island.cc.server_utils.consts import DEFAULT_DEVELOP_SERVER_CONFIG_PATH
-
-
-def create_default_config_file(path):
-    default_config = Path(DEFAULT_DEVELOP_SERVER_CONFIG_PATH).read_text()
-    Path(path).write_text(default_config)

monkey/monkey_island/cc/environment/server_config_handler.py

@@ -0,0 +1,27 @@
+import json
+import os
+from pathlib import Path
+
+from monkey_island.cc.server_utils.consts import (
+    DEFAULT_DEVELOP_SERVER_CONFIG_PATH,
+    DEFAULT_SERVER_CONFIG_PATH,
+)
+from monkey_island.setup.island_config_options import IslandConfigOptions
+
+
+def create_default_server_config_file() -> None:
+    if not os.path.isfile(DEFAULT_SERVER_CONFIG_PATH):
+        write_default_server_config_to_file(DEFAULT_SERVER_CONFIG_PATH)
+
+
+def write_default_server_config_to_file(path: str) -> None:
+    default_config = Path(DEFAULT_DEVELOP_SERVER_CONFIG_PATH).read_text()
+    Path(path).write_text(default_config)
+
+
+def load_server_config_from_file(server_config_path) -> IslandConfigOptions:
+    with open(server_config_path, "r") as f:
+        config_content = f.read()
+        config = json.loads(config_content)
+
+        return IslandConfigOptions(config)

monkey/monkey_island/cc/environment/utils.py

@@ -0,0 +1,5 @@
+import sys
+
+
+def is_windows_os() -> bool:
+    return sys.platform.startswith("win")

monkey/monkey_island/cc/environment/windows_permissions.py

@@ -0,0 +1,34 @@
+from monkey_island.cc.environment.utils import is_windows_os
+
+if is_windows_os():
+    import ntsecuritycon
+    import win32api
+    import win32con
+    import win32security
+
+
+def set_full_folder_access(folder_path: str) -> None:
+    user = get_user_pySID_object()
+
+    security_descriptor = win32security.GetFileSecurity(
+        folder_path, win32security.DACL_SECURITY_INFORMATION
+    )
+    dacl = win32security.ACL()
+    dacl.AddAccessAllowedAce(
+        win32security.ACL_REVISION,
+        ntsecuritycon.FILE_ALL_ACCESS,
+        user,
+    )
+    security_descriptor.SetSecurityDescriptorDacl(1, dacl, 0)
+    win32security.SetFileSecurity(
+        folder_path, win32security.DACL_SECURITY_INFORMATION, security_descriptor
+    )
+
+
+def get_user_pySID_object():
+    # get current user's name
+    username = win32api.GetUserNameEx(win32con.NameSamCompatible)
+    # pySID object for the current user
+    user, _, _ = win32security.LookupAccountName("", username)
+
+    return user

monkey/monkey_island/cc/main.py

@@ -35,9 +35,7 @@ from monkey_island.cc.services.utils.network_utils import local_ip_addresses  #
 MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0"
 
 
-def main(setup_only: bool, server_config_path: str, config_options: IslandConfigOptions):
+def main(setup_only: bool, config_options: IslandConfigOptions):
-
-    env_singleton.initialize_from_file(server_config_path)
     initialize_encryptor(config_options.data_dir)
     initialize_services(config_options.data_dir)
 
@@ -122,7 +120,3 @@ def assert_mongo_db_version(mongo_url):
         sys.exit(-1)
     else:
         logger.info("Mongo DB version OK. Got {0}".format(str(server_version)))
-
-
-if __name__ == "__main__":
-    main()

monkey/monkey_island/cc/models/__init__.py

@@ -1,17 +1,18 @@
 from mongoengine import connect
 
-import monkey_island.cc.environment.environment_singleton as env_singleton
+import monkey_island.cc.environment.environment_singleton as env_singleton  # noqa: E402
 
-from .command_control_channel import CommandControlChannel  # noqa: F401
+from .command_control_channel import CommandControlChannel  # noqa: F401, E402
 
 # Order of importing matters here, for registering the embedded and referenced documents before
 # using them.
-from .config import Config  # noqa: F401
+from .config import Config  # noqa: F401, E402
-from .creds import Creds  # noqa: F401
+from .creds import Creds  # noqa: F401, E402
-from .monkey import Monkey  # noqa: F401
+from .monkey import Monkey  # noqa: F401, E402
-from .monkey_ttl import MonkeyTtl  # noqa: F401
+from .monkey_ttl import MonkeyTtl  # noqa: F401, E402
-from .pba_results import PbaResults  # noqa: F401
+from .pba_results import PbaResults  # noqa: F401, E402
 
+# TODO refactor into explicit call when implementing mongodb startup
 connect(
     db=env_singleton.env.mongo_db_name,
     host=env_singleton.env.mongo_db_host,

monkey/monkey_island/cc/server_utils/consts.py

@@ -1,18 +1,32 @@
 import os
 
+from monkey_island.cc.environment.utils import is_windows_os
+
 __author__ = "itay.mizeretz"
 
+
+def get_default_data_dir() -> str:
+    if is_windows_os():
+        return r"%AppData%\monkey_island"
+    else:
+        return r"$HOME/.monkey_island"
+
+
+SERVER_CONFIG_FILENAME = "server_config.json"
+
 MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island")
+
+DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir())
+
 DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5
 
-# TODO move setup consts
+DEFAULT_SERVER_CONFIG_PATH = os.path.expandvars(
-DEFAULT_SERVER_CONFIG_PATH = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json")
+    os.path.join(DEFAULT_DATA_DIR, SERVER_CONFIG_FILENAME)
-
+)
-DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join(
+
-    MONKEY_ISLAND_ABS_PATH, "cc", "server_config.json.develop"
+DEFAULT_DEVELOP_SERVER_CONFIG_PATH = os.path.join(
+    MONKEY_ISLAND_ABS_PATH, "cc", f"{SERVER_CONFIG_FILENAME}.develop"
 )
 
-DEFAULT_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc")
 DEFAULT_LOG_LEVEL = "INFO"
 DEFAULT_START_MONGO_DB = True
-DEFAULT_SHOULD_SETUP_ONLY = False

monkey/monkey_island/setup/config_setup.py

@@ -0,0 +1,23 @@
+import os
+from typing import Tuple
+
+from monkey_island.cc.environment import server_config_handler
+from monkey_island.cc.environment.data_dir_generator import create_data_dir  # noqa: E402
+from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH
+from monkey_island.setup.island_config_options import IslandConfigOptions
+
+
+def setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, str]:
+    server_config_path = os.path.expandvars(os.path.expanduser(server_config_path))
+    config = server_config_handler.load_server_config_from_file(server_config_path)
+
+    create_data_dir(config.data_dir, create_parent_dirs=True)
+    return config, server_config_path
+
+
+def setup_default_config() -> Tuple[IslandConfigOptions, str]:
+    server_config_path = DEFAULT_SERVER_CONFIG_PATH
+    create_data_dir(DEFAULT_DATA_DIR, create_parent_dirs=False)
+    server_config_handler.create_default_server_config_file()
+    config = server_config_handler.load_server_config_from_file(server_config_path)
+    return config, server_config_path

monkey/monkey_island/setup/gevent_setup.py

@@ -0,0 +1,6 @@
+from gevent import monkey as gevent_monkey
+
+# We need to monkeypatch before any other imports to
+# make standard libraries compatible with gevent.
+# http://www.gevent.org/api/gevent.monkey.html
+gevent_monkey.patch_all()

monkey/tests/unit_tests/monkey_island/cc/conftest.py

@@ -1,3 +1,11 @@
+import monkey_island.cc.environment.environment_singleton as env_singleton
+from monkey_island.cc.environment.testing import TestingEnvironment
+
+# Mock environment singleton because it contains mongodb parameters
+# needed for model tests. See monkey/monkey_island/cc/models/__init__.py
+env_config = {}
+env_singleton.env = TestingEnvironment(env_config)
+
 # Without these imports pytests can't use fixtures,
 # because they are not found
-from tests.unit_tests.monkey_island.cc.mongomock_fixtures import *  # noqa: F401,F403
+from tests.unit_tests.monkey_island.cc.mongomock_fixtures import *  # noqa: F401,F403,E402

monkey/tests/unit_tests/monkey_island/cc/environment/test_environment_config.py

@@ -93,15 +93,3 @@ def test_get_users(standard_with_credentials):
     assert users[0].id == 1
     assert users[0].username == "test"
     assert users[0].secret == "abcdef"
-
-
-def test_generate_default_file(config_file):
-    environment_config = EnvironmentConfig(config_file)
-
-    assert os.path.isfile(config_file)
-
-    assert environment_config.server_config == "password"
-    assert environment_config.deployment == "develop"
-    assert environment_config.user_creds.username == ""
-    assert environment_config.user_creds.password_hash == ""
-    assert environment_config.aws is None

monkey/tests/unit_tests/monkey_island/cc/test_consts.py

@@ -5,8 +5,8 @@ import monkey_island.cc.server_utils.consts as consts
 
 def test_default_server_config_file_path():
     if platform.system() == "Windows":
-        server_file_path = consts.MONKEY_ISLAND_ABS_PATH + r"\cc\server_config.json"
+        server_file_path = f"{consts.DEFAULT_DATA_DIR}\\{consts.SERVER_CONFIG_FILENAME}"
     else:
-        server_file_path = consts.MONKEY_ISLAND_ABS_PATH + "/cc/server_config.json"
+        server_file_path = f"{consts.DEFAULT_DATA_DIR}/{consts.SERVER_CONFIG_FILENAME}"
 
     assert consts.DEFAULT_SERVER_CONFIG_PATH == server_file_path